[llvm] [NVPTX] Cleanup support for atomic loads/stores (PR #201468)
Akshay Deodhar via llvm-commits
llvm-commits at lists.llvm.org
Wed Jun 3 15:48:13 PDT 2026
https://github.com/akshayrdeodhar created https://github.com/llvm/llvm-project/pull/201468
- Adds exhaustive python-scripted testing for codegen for loads and stores in NVPTX, for all valid combinations of datatype x ordering x scope x addrspace x volatility, in the style of atomicrmw/cmpxchg/fence.
- Migrates the emulation of seq_cst loads to AtomicExpandPass using bracketInstructionWithFences. For this, updates AtomicExpandPass to set load/store ordering after a fence split. Cleans up associated helper functions.
- Emulates formerly-unsupported atomic loads and stores using fences in AtomicExpandPass. This leads to all possible memory orders being emulated even for SM60, using membar as a fence.
- Relaxes the lowering of seq_cst stores to fence.sc; st.relaxed instead of fence.sc; st.release.
- Updates NVPTXAtomicLower to downgrade atomic local loads/stores to nonatomic, in a fashion similar to atomicrmw.
- Note that volatile loads are currently being lowered incorrectly- will be fixed in a followup PR.
>From 29110d56a58e89340c473935c03af9ff5762f131 Mon Sep 17 00:00:00 2001
From: Akshay Deodhar <adeodhar at nvidia.com>
Date: Mon, 18 May 2026 22:47:44 +0000
Subject: [PATCH] [NVPTX] Cleanup support for atomic loads/stores
---
llvm/lib/CodeGen/AtomicExpandPass.cpp | 4 +-
llvm/lib/Target/NVPTX/NVPTXAtomicLower.cpp | 31 +-
llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp | 58 +-
llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.h | 2 +-
llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp | 124 +-
llvm/test/CodeGen/NVPTX/lit.local.cfg | 2 +-
llvm/test/CodeGen/NVPTX/load-sm60.ll | 20340 ++++++++++++++++
llvm/test/CodeGen/NVPTX/load-sm70.ll | 19956 +++++++++++++++
llvm/test/CodeGen/NVPTX/load-sm90.ll | 22986 ++++++++++++++++++
llvm/test/CodeGen/NVPTX/load-store-sm-70.ll | 360 +-
llvm/test/CodeGen/NVPTX/load-store-sm-90.ll | 120 +-
llvm/test/CodeGen/NVPTX/load.py | 194 +
llvm/test/CodeGen/NVPTX/store-sm60.ll | 19360 +++++++++++++++
llvm/test/CodeGen/NVPTX/store-sm70.ll | 19168 +++++++++++++++
llvm/test/CodeGen/NVPTX/store-sm90.ll | 21950 +++++++++++++++++
llvm/test/CodeGen/NVPTX/store.py | 206 +
llvm/test/lit.cfg.py | 6 +-
17 files changed, 124547 insertions(+), 320 deletions(-)
create mode 100644 llvm/test/CodeGen/NVPTX/load-sm60.ll
create mode 100644 llvm/test/CodeGen/NVPTX/load-sm70.ll
create mode 100644 llvm/test/CodeGen/NVPTX/load-sm90.ll
create mode 100644 llvm/test/CodeGen/NVPTX/load.py
create mode 100644 llvm/test/CodeGen/NVPTX/store-sm60.ll
create mode 100644 llvm/test/CodeGen/NVPTX/store-sm70.ll
create mode 100644 llvm/test/CodeGen/NVPTX/store-sm90.ll
create mode 100644 llvm/test/CodeGen/NVPTX/store.py
diff --git a/llvm/lib/CodeGen/AtomicExpandPass.cpp b/llvm/lib/CodeGen/AtomicExpandPass.cpp
index 960d2492c2856..13ec35ca3f1cb 100644
--- a/llvm/lib/CodeGen/AtomicExpandPass.cpp
+++ b/llvm/lib/CodeGen/AtomicExpandPass.cpp
@@ -350,7 +350,7 @@ bool AtomicExpandImpl::processAtomicInstr(Instruction *I) {
}
MadeChange |= tryInsertFencesForAtomic(
- LI, isAcquireOrStronger(LI->getOrdering()), AtomicOrdering::Monotonic);
+ LI, isAcquireOrStronger(LI->getOrdering()), TLI->atomicOperationOrderAfterFenceSplit(LI));
MadeChange |= tryExpandAtomicLoad(LI);
return MadeChange;
@@ -373,7 +373,7 @@ bool AtomicExpandImpl::processAtomicInstr(Instruction *I) {
}
MadeChange |= tryInsertFencesForAtomic(
- SI, isReleaseOrStronger(SI->getOrdering()), AtomicOrdering::Monotonic);
+ SI, isReleaseOrStronger(SI->getOrdering()), TLI->atomicOperationOrderAfterFenceSplit(SI));
MadeChange |= tryExpandAtomicStore(SI);
return MadeChange;
diff --git a/llvm/lib/Target/NVPTX/NVPTXAtomicLower.cpp b/llvm/lib/Target/NVPTX/NVPTXAtomicLower.cpp
index 918daf6c04ecd..badb4ebb9f9fc 100644
--- a/llvm/lib/Target/NVPTX/NVPTXAtomicLower.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXAtomicLower.cpp
@@ -42,15 +42,36 @@ class NVPTXAtomicLower : public FunctionPass {
} // namespace
bool NVPTXAtomicLower::runOnFunction(Function &F) {
- SmallVector<AtomicRMWInst *> LocalMemoryAtomics;
- for (Instruction &I : instructions(F))
- if (AtomicRMWInst *RMWI = dyn_cast<AtomicRMWInst>(&I))
+ SmallVector<AtomicRMWInst *> LocalRMWs;
+ SmallVector<LoadInst *> LocalAtomicLoads;
+ SmallVector<StoreInst *> LocalAtomicStores;
+ // TODO: Handle cmpxchg
+ for (Instruction &I : instructions(F)) {
+ if (auto *RMWI = dyn_cast<AtomicRMWInst>(&I)) {
if (RMWI->getPointerAddressSpace() == ADDRESS_SPACE_LOCAL)
- LocalMemoryAtomics.push_back(RMWI);
+ LocalRMWs.push_back(RMWI);
+ } else if (auto *LI = dyn_cast<LoadInst>(&I)) {
+ if (LI->isAtomic() &&
+ LI->getPointerAddressSpace() == ADDRESS_SPACE_LOCAL)
+ LocalAtomicLoads.push_back(LI);
+ } else if (auto *SI = dyn_cast<StoreInst>(&I)) {
+ if (SI->isAtomic() &&
+ SI->getPointerAddressSpace() == ADDRESS_SPACE_LOCAL)
+ LocalAtomicStores.push_back(SI);
+ }
+ }
bool Changed = false;
- for (AtomicRMWInst *RMWI : LocalMemoryAtomics)
+ for (AtomicRMWInst *RMWI : LocalRMWs)
Changed |= lowerAtomicRMWInst(RMWI);
+ for (LoadInst *LI : LocalAtomicLoads) {
+ LI->setAtomic(AtomicOrdering::NotAtomic);
+ Changed = true;
+ }
+ for (StoreInst *SI : LocalAtomicStores) {
+ SI->setAtomic(AtomicOrdering::NotAtomic);
+ Changed = true;
+ }
return Changed;
}
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp b/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp
index ede1deb5400b0..7b16de900d11f 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp
@@ -551,15 +551,8 @@ NVPTX::Scope NVPTXDAGToDAGISel::getAtomicScope(const MemSDNode *N) const {
namespace {
-struct OperationOrderings {
- NVPTX::Ordering InstructionOrdering, FenceOrdering;
- OperationOrderings(NVPTX::Ordering IO = NVPTX::Ordering::NotAtomic,
- NVPTX::Ordering FO = NVPTX::Ordering::NotAtomic)
- : InstructionOrdering(IO), FenceOrdering(FO) {}
-};
-
-static OperationOrderings
-getOperationOrderings(MemSDNode *N, const NVPTXSubtarget *Subtarget) {
+static NVPTX::Ordering
+getInstructionOrdering(MemSDNode *N, const NVPTXSubtarget *Subtarget) {
AtomicOrdering Ordering = N->getSuccessOrdering();
auto CodeAddrSpace = NVPTXDAGToDAGISel::getAddrSpace(N);
@@ -730,18 +723,8 @@ getOperationOrderings(MemSDNode *N, const NVPTXSubtarget *Subtarget) {
//
// This sets the ordering of the fence to SequentiallyConsistent, and
// sets the corresponding ordering for the instruction.
- NVPTX::Ordering InstrOrder;
- if (N->readMem())
- InstrOrder = NVPTX::Ordering::Acquire;
- else if (N->writeMem())
- InstrOrder = NVPTX::Ordering::Release;
- else
- report_fatal_error(
- formatv("NVPTX does not support SequentiallyConsistent Ordering on "
- "read-modify-writes yet: {}",
- N->getOperationName()));
- return OperationOrderings(InstrOrder,
- NVPTX::Ordering::SequentiallyConsistent);
+ llvm_unreachable("seq_cst ordering should not reach this point, "
+ "is emulated in AtomicExpandPass");
}
}
report_fatal_error(
@@ -888,14 +871,11 @@ static unsigned int getFenceOp(NVPTX::Ordering O, NVPTX::Scope S,
llvm_unreachable("unhandled ordering");
}
-// Returns Memory Order and Scope of a memory instruction, and
-// inserts any fence before the instruction that's required to
-// implement its memory ordering.
+// Returns Memory Order and Scope of a memory instruction.
std::pair<NVPTX::Ordering, NVPTX::Scope>
-NVPTXDAGToDAGISel::insertMemoryInstructionFence(SDLoc DL, SDValue &Chain,
- MemSDNode *N) {
- auto [InstructionOrdering, FenceOrdering] =
- getOperationOrderings(N, Subtarget);
+NVPTXDAGToDAGISel::getMemInstOrderingScope(MemSDNode *N) {
+ auto InstructionOrdering =
+ getInstructionOrdering(N, Subtarget);
auto Scope = getOperationScope(N, InstructionOrdering);
// Singlethread scope has no inter-thread synchronization requirements, so
@@ -907,20 +887,6 @@ NVPTXDAGToDAGISel::insertMemoryInstructionFence(SDLoc DL, SDValue &Chain,
InstructionOrdering != NVPTX::Ordering::Volatile)
return {NVPTX::Ordering::NotAtomic, Scope};
- // If a fence is required before the operation, insert it:
- switch (NVPTX::Ordering(FenceOrdering)) {
- case NVPTX::Ordering::NotAtomic:
- break;
- case NVPTX::Ordering::SequentiallyConsistent: {
- auto Op = getFenceOp(FenceOrdering, Scope, Subtarget);
- Chain = SDValue(CurDAG->getMachineNode(Op, DL, MVT::Other, Chain), 0);
- break;
- }
- default:
- report_fatal_error(
- formatv("Unexpected fence ordering: \"{}\".",
- OrderingToString(NVPTX::Ordering(FenceOrdering))));
- }
return {InstructionOrdering, Scope};
}
@@ -1124,7 +1090,7 @@ bool NVPTXDAGToDAGISel::tryLoad(SDNode *N) {
SDLoc DL(LD);
SDValue Chain = N->getOperand(0);
- const auto [Ordering, Scope] = insertMemoryInstructionFence(DL, Chain, LD);
+ const auto [Ordering, Scope] = getMemInstOrderingScope(LD);
const unsigned FromTypeWidth = LD->getMemoryVT().getSizeInBits();
@@ -1203,7 +1169,7 @@ bool NVPTXDAGToDAGISel::tryLoadVector(SDNode *N) {
const MVT EltVT = LD->getSimpleValueType(0);
SDLoc DL(LD);
SDValue Chain = LD->getChain();
- const auto [Ordering, Scope] = insertMemoryInstructionFence(DL, Chain, LD);
+ const auto [Ordering, Scope] = getMemInstOrderingScope(LD);
// Type Setting: fromType + fromTypeWidth
//
@@ -1396,7 +1362,7 @@ bool NVPTXDAGToDAGISel::tryStore(SDNode *N) {
SDLoc DL(ST);
SDValue Chain = ST->getChain();
- const auto [Ordering, Scope] = insertMemoryInstructionFence(DL, Chain, ST);
+ const auto [Ordering, Scope] = getMemInstOrderingScope(ST);
// Vector Setting
const unsigned ToTypeWidth = ST->getMemoryVT().getSizeInBits();
@@ -1447,7 +1413,7 @@ bool NVPTXDAGToDAGISel::tryStoreVector(SDNode *N) {
SDLoc DL(ST);
SDValue Chain = ST->getChain();
- const auto [Ordering, Scope] = insertMemoryInstructionFence(DL, Chain, ST);
+ const auto [Ordering, Scope] = getMemInstOrderingScope(ST);
const unsigned NumElts = getStoreVectorNumElts(ST);
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.h b/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.h
index fcb5700dcb6d4..f2d3272d1111a 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.h
+++ b/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.h
@@ -110,7 +110,7 @@ class LLVM_LIBRARY_VISIBILITY NVPTXDAGToDAGISel : public SelectionDAGISel {
// instruction, if needed to implement the instructions memory order. Required
// fences after the instruction need to be handled elsewhere.
std::pair<NVPTX::Ordering, NVPTX::Scope>
- insertMemoryInstructionFence(SDLoc DL, SDValue &Chain, MemSDNode *N);
+ getMemInstOrderingScope(MemSDNode *N);
NVPTX::Scope getOperationScope(MemSDNode *N, NVPTX::Ordering O) const;
public:
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
index e13911f87eed5..f5e39bda0d358 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
@@ -7600,11 +7600,35 @@ NVPTXTargetLowering::shouldExpandAtomicRMWInIR(const AtomicRMWInst *AI) const {
return AtomicExpansionKind::CmpXChg;
}
+// Atomic load lowering grid (rows: native memory-ordering support; columns:
+// LLVM ordering):
+//
+// seq_cst acquire
+// supported fence.sc; ld.acquire ld.acquire
+// unsupported fence.sc; ld.relaxed; fence.acquire ld.relaxed; fence.acquire
+//
+// Atomic store lowering grid (same rows; columns are the store-side orderings):
+//
+// seq_cst release
+// supported fence.sc; st.relaxed st.release
+// unsupported fence.sc; st.relaxed fence.release; st.relaxed
+//
+// "supported" = STI.hasMemoryOrdering() (sm_70+ with PTX 6.0+). monotonic and
+// weaker drop out of these grids -- they need no fences and pass through.
+// Stores never get a trailing fence here; their release-side ordering is
+// provided entirely by the leading fence (or by typed `st.release` when
+// available).
+
bool NVPTXTargetLowering::shouldInsertFencesForAtomic(
const Instruction *I) const {
- // This function returns true iff the operation is emulated using a CAS-loop,
- // or if it has the memory order seq_cst (which is not natively supported in
- // the PTX `atom` instruction).
+ // Returns true iff this op needs a leading and/or trailing fence emitted
+ // around it (see emitLeadingFence / emitTrailingFence). Fires when:
+ // - the operation is emulated using a CAS-loop, OR
+ // - it has memory order seq_cst (PTX has no seq_cst atom/ld/st), OR
+ // - it is a load that has an unsupported memory order
+ // (acquire / seq_cst on !hasMemoryOrdering targets), OR
+ // - it is a store that has an unsupported memory order
+ // (release / seq_cst on !hasMemoryOrdering targets).
//
// atomicrmw and cmpxchg instructions not efficiently supported by PTX
// are lowered to CAS emulation loops that preserve their memory order,
@@ -7617,6 +7641,9 @@ bool NVPTXTargetLowering::shouldInsertFencesForAtomic(
// and scope. Since PTX does not support seq_cst, we emulate it by lowering to
// a fence.sc followed by an atom according to the PTX atomics ABI
// https://docs.nvidia.com/cuda/ptx-writers-guide-to-interoperability/atomic-abi.html
+ //
+ // Atomic loads with a memory order not supported on the SM can be
+ // inefficiently emulated by appending a `fence.acquire` after a relaxed load.
if (auto *CI = dyn_cast<AtomicCmpXchgInst>(I))
return (cast<IntegerType>(CI->getCompareOperand()->getType())
->getBitWidth() < STI.getMinCmpXchgSizeInBits()) ||
@@ -7624,11 +7651,29 @@ bool NVPTXTargetLowering::shouldInsertFencesForAtomic(
if (auto *RI = dyn_cast<AtomicRMWInst>(I))
return shouldExpandAtomicRMWInIR(RI) == AtomicExpansionKind::CmpXChg ||
RI->getOrdering() == AtomicOrdering::SequentiallyConsistent;
+ if (auto *LI = dyn_cast<LoadInst>(I))
+ return LI->getOrdering() == AtomicOrdering::SequentiallyConsistent ||
+ (!STI.hasMemoryOrdering() && isAcquireOrStronger(LI->getOrdering()));
+ if (auto *SI = dyn_cast<StoreInst>(I))
+ return SI->getOrdering() == AtomicOrdering::SequentiallyConsistent ||
+ (!STI.hasMemoryOrdering() && isReleaseOrStronger(SI->getOrdering()));
return false;
}
AtomicOrdering NVPTXTargetLowering::atomicOperationOrderAfterFenceSplit(
const Instruction *I) const {
+ // Returns the new AtomicOrdering for the original op once AtomicExpandPass
+ // has split it into leading/trailing fences + the op itself. Per the grids
+ // above:
+ // - cmpxchg / atomicrmw at seq_cst (not CAS-emulated): Acquire
+ // ("fence.sc; atom.<op>.acquire" pattern).
+ // - load at any ordering when hasMemoryOrdering: Acquire
+ // ("fence.sc; ld.acquire" for seq_cst; the acquire load is already typed
+ // on supported targets).
+ // - everything else (CAS-emulated ops, seq_cst on !hasMemoryOrdering,
+ // release-or-stronger store on !hasMemoryOrdering, etc.): Monotonic
+ // (i.e., relaxed) -- the surrounding fences carry the ordering.
+ //
// If the operation is emulated by a CAS-loop, we lower the instruction to
// atom.<op>.relaxed, since AtomicExpandPass will insert fences for enforcing
// the correct memory ordering around the CAS loop.
@@ -7652,10 +7697,13 @@ AtomicOrdering NVPTXTargetLowering::atomicOperationOrderAfterFenceSplit(
cast<IntegerType>(CI->getCompareOperand()->getType())->getBitWidth() >=
STI.getMinCmpXchgSizeInBits())
return AtomicOrdering::Acquire;
- else if (auto *RI = dyn_cast<AtomicRMWInst>(I);
+ if (auto *RI = dyn_cast<AtomicRMWInst>(I);
RI && RI->getOrdering() == AtomicOrdering::SequentiallyConsistent &&
shouldExpandAtomicRMWInIR(RI) == AtomicExpansionKind::None)
return AtomicOrdering::Acquire;
+ if (auto *LI = dyn_cast<LoadInst>(I);
+ LI && STI.hasMemoryOrdering())
+ return AtomicOrdering::Acquire;
return AtomicOrdering::Monotonic;
}
@@ -7663,23 +7711,42 @@ AtomicOrdering NVPTXTargetLowering::atomicOperationOrderAfterFenceSplit(
Instruction *NVPTXTargetLowering::emitLeadingFence(IRBuilderBase &Builder,
Instruction *Inst,
AtomicOrdering Ord) const {
+ // Emits the leading fence per the grids above:
+ // - cmpxchg / atomicrmw: fence.sc iff Ord == seq_cst, else fence.release
+ // if Ord is release-or-stronger.
+ // - load: fence.sc iff Ord == seq_cst.
+ // - store: same shape as cmpxchg/rmw (fence.sc or fence.release based on
+ // Ord being seq_cst vs release-or-stronger).
+ // Returns nullptr if no leading fence is needed for this Inst/Ord.
+ //
// prerequisite: shouldInsertFencesForAtomic() should have returned `true` for
// `Inst` before its memory order was modified. We cannot enforce this with an
// assert, because AtomicExpandPass will have modified the memory order
// between the initial call to shouldInsertFencesForAtomic() and the call to
// this function.
- if (!isa<AtomicCmpXchgInst>(Inst) && !isa<AtomicRMWInst>(Inst))
- return TargetLoweringBase::emitLeadingFence(Builder, Inst, Ord);
-
- // Specialize for cmpxchg and atomicrmw
auto SSID = getAtomicSyncScopeID(Inst);
assert(SSID.has_value() && "Expected an atomic operation");
-
- if (isReleaseOrStronger(Ord))
- return Builder.CreateFence(Ord == AtomicOrdering::SequentiallyConsistent
- ? AtomicOrdering::SequentiallyConsistent
- : AtomicOrdering::Release,
- SSID.value());
+ if (isa<AtomicCmpXchgInst>(Inst) || isa<AtomicRMWInst>(Inst)) {
+ // Specialize for cmpxchg and atomicrmw
+ if (isReleaseOrStronger(Ord))
+ return Builder.CreateFence(Ord == AtomicOrdering::SequentiallyConsistent
+ ? AtomicOrdering::SequentiallyConsistent
+ : AtomicOrdering::Release,
+ SSID.value());
+ } else if (auto *LI = dyn_cast<LoadInst>(Inst)) {
+ // volatile operations have sys scope by default
+ SSID = LI->isVolatile() ? Builder.getContext().getOrInsertSyncScopeID("") : SSID;
+ if (Ord == AtomicOrdering::SequentiallyConsistent)
+ return Builder.CreateFence(AtomicOrdering::SequentiallyConsistent, SSID.value());
+ } else if (auto *SI = dyn_cast<StoreInst>(Inst)) {
+ // volatile operations have sys scope by default
+ SSID = SI->isVolatile() ? Builder.getContext().getOrInsertSyncScopeID("") : SSID;
+ if (isReleaseOrStronger(Ord))
+ return Builder.CreateFence(Ord == AtomicOrdering::SequentiallyConsistent
+ ? AtomicOrdering::SequentiallyConsistent
+ : AtomicOrdering::Release,
+ SSID.value());
+ }
return nullptr;
}
@@ -7687,25 +7754,34 @@ Instruction *NVPTXTargetLowering::emitLeadingFence(IRBuilderBase &Builder,
Instruction *NVPTXTargetLowering::emitTrailingFence(IRBuilderBase &Builder,
Instruction *Inst,
AtomicOrdering Ord) const {
+ // Emits the trailing fence per the grids above:
+ // - cmpxchg / atomicrmw: fence.acquire iff Ord is acquire-or-stronger AND
+ // the op is being CAS-emulated (i.e., the inner CAS uses relaxed).
+ // - load: fence.acquire iff !hasMemoryOrdering (covers both the acquire
+ // and seq_cst cells of the load grid).
+ // - store: never (release-side ordering for stores comes from the leading
+ // fence or the typed `st.release` instruction).
+ // Returns nullptr if no trailing fence is needed for this Inst/Ord.
+ //
// prerequisite: shouldInsertFencesForAtomic() should have returned `true` for
// `Inst` before its memory order was modified. See `emitLeadingFence` for why
// this cannot be enforced with an assert. Specialize for cmpxchg and
// atomicrmw
auto *CI = dyn_cast<AtomicCmpXchgInst>(Inst);
auto *RI = dyn_cast<AtomicRMWInst>(Inst);
- if (!CI && !RI)
- return TargetLoweringBase::emitTrailingFence(Builder, Inst, Ord);
-
auto SSID = getAtomicSyncScopeID(Inst);
assert(SSID.has_value() && "Expected an atomic operation");
+ if (CI || RI) {
+ bool IsEmulated =
+ CI ? cast<IntegerType>(CI->getCompareOperand()->getType())
+ ->getBitWidth() < STI.getMinCmpXchgSizeInBits()
+ : shouldExpandAtomicRMWInIR(RI) == AtomicExpansionKind::CmpXChg;
- bool IsEmulated =
- CI ? cast<IntegerType>(CI->getCompareOperand()->getType())
- ->getBitWidth() < STI.getMinCmpXchgSizeInBits()
- : shouldExpandAtomicRMWInIR(RI) == AtomicExpansionKind::CmpXChg;
-
- if (isAcquireOrStronger(Ord) && IsEmulated)
- return Builder.CreateFence(AtomicOrdering::Acquire, SSID.value());
+ if (isAcquireOrStronger(Ord) && IsEmulated)
+ return Builder.CreateFence(AtomicOrdering::Acquire, SSID.value());
+ } else if (isa<LoadInst>(Inst) && !STI.hasMemoryOrdering()) {
+ return Builder.CreateFence(AtomicOrdering::Acquire, SSID.value());
+ }
return nullptr;
}
diff --git a/llvm/test/CodeGen/NVPTX/lit.local.cfg b/llvm/test/CodeGen/NVPTX/lit.local.cfg
index dede5b354bb85..344d17adc5594 100644
--- a/llvm/test/CodeGen/NVPTX/lit.local.cfg
+++ b/llvm/test/CodeGen/NVPTX/lit.local.cfg
@@ -1,4 +1,4 @@
if not "NVPTX" in config.root.targets:
config.unsupported = True
config.suffixes.add(".py")
-config.excludes = ["fence.py", "cmpxchg.py", "atomicrmw.py"]
+config.excludes = ["fence.py", "cmpxchg.py", "atomicrmw.py", "load.py", "store.py"]
diff --git a/llvm/test/CodeGen/NVPTX/load-sm60.ll b/llvm/test/CodeGen/NVPTX/load-sm60.ll
new file mode 100644
index 0000000000000..e39d262da43c8
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/load-sm60.ll
@@ -0,0 +1,20340 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_60 -mattr=+ptx50 | FileCheck %s
+; RUN: %if ptxas-sm_60 && ptxas-isa-5.0 %{ llc < %s -march=nvptx64 -mcpu=sm_60 -mattr=+ptx50 | %ptxas-verify -arch=sm_60 %}
+
+define i8 @notatomic_i8_generic(ptr %addr) {
+; CHECK-LABEL: notatomic_i8_generic(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i8_generic_param_0];
+; CHECK-NEXT: ld.b8 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load i8, ptr %addr
+ ret i8 %retval
+}
+
+define i8 @notatomic_i8_generic_volatile(ptr %addr) {
+; CHECK-LABEL: notatomic_i8_generic_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i8_generic_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b8 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load volatile i8, ptr %addr
+ ret i8 %retval
+}
+
+define i16 @notatomic_i16_generic(ptr %addr) {
+; CHECK-LABEL: notatomic_i16_generic(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i16_generic_param_0];
+; CHECK-NEXT: ld.b16 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load i16, ptr %addr
+ ret i16 %retval
+}
+
+define i16 @notatomic_i16_generic_volatile(ptr %addr) {
+; CHECK-LABEL: notatomic_i16_generic_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i16_generic_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b16 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load volatile i16, ptr %addr
+ ret i16 %retval
+}
+
+define i32 @notatomic_i32_generic(ptr %addr) {
+; CHECK-LABEL: notatomic_i32_generic(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i32_generic_param_0];
+; CHECK-NEXT: ld.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load i32, ptr %addr
+ ret i32 %retval
+}
+
+define i32 @notatomic_i32_generic_volatile(ptr %addr) {
+; CHECK-LABEL: notatomic_i32_generic_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i32_generic_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load volatile i32, ptr %addr
+ ret i32 %retval
+}
+
+define i64 @notatomic_i64_generic(ptr %addr) {
+; CHECK-LABEL: notatomic_i64_generic(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i64_generic_param_0];
+; CHECK-NEXT: ld.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load i64, ptr %addr
+ ret i64 %retval
+}
+
+define i64 @notatomic_i64_generic_volatile(ptr %addr) {
+; CHECK-LABEL: notatomic_i64_generic_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i64_generic_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load volatile i64, ptr %addr
+ ret i64 %retval
+}
+
+define half @notatomic_half_generic(ptr %addr) {
+; CHECK-LABEL: notatomic_half_generic(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_half_generic_param_0];
+; CHECK-NEXT: ld.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load half, ptr %addr
+ ret half %retval
+}
+
+define half @notatomic_half_generic_volatile(ptr %addr) {
+; CHECK-LABEL: notatomic_half_generic_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_half_generic_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load volatile half, ptr %addr
+ ret half %retval
+}
+
+define bfloat @notatomic_bfloat_generic(ptr %addr) {
+; CHECK-LABEL: notatomic_bfloat_generic(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_bfloat_generic_param_0];
+; CHECK-NEXT: ld.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load bfloat, ptr %addr
+ ret bfloat %retval
+}
+
+define bfloat @notatomic_bfloat_generic_volatile(ptr %addr) {
+; CHECK-LABEL: notatomic_bfloat_generic_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_bfloat_generic_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load volatile bfloat, ptr %addr
+ ret bfloat %retval
+}
+
+define float @notatomic_float_generic(ptr %addr) {
+; CHECK-LABEL: notatomic_float_generic(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_float_generic_param_0];
+; CHECK-NEXT: ld.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load float, ptr %addr
+ ret float %retval
+}
+
+define float @notatomic_float_generic_volatile(ptr %addr) {
+; CHECK-LABEL: notatomic_float_generic_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_float_generic_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load volatile float, ptr %addr
+ ret float %retval
+}
+
+define double @notatomic_double_generic(ptr %addr) {
+; CHECK-LABEL: notatomic_double_generic(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_double_generic_param_0];
+; CHECK-NEXT: ld.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load double, ptr %addr
+ ret double %retval
+}
+
+define double @notatomic_double_generic_volatile(ptr %addr) {
+; CHECK-LABEL: notatomic_double_generic_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_double_generic_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load volatile double, ptr %addr
+ ret double %retval
+}
+
+define i8 @notatomic_i8_global(ptr addrspace(1) %addr) {
+; CHECK-LABEL: notatomic_i8_global(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i8_global_param_0];
+; CHECK-NEXT: ld.global.b8 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load i8, ptr addrspace(1) %addr
+ ret i8 %retval
+}
+
+define i8 @notatomic_i8_global_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: notatomic_i8_global_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i8_global_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b8 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load volatile i8, ptr addrspace(1) %addr
+ ret i8 %retval
+}
+
+define i16 @notatomic_i16_global(ptr addrspace(1) %addr) {
+; CHECK-LABEL: notatomic_i16_global(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i16_global_param_0];
+; CHECK-NEXT: ld.global.b16 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load i16, ptr addrspace(1) %addr
+ ret i16 %retval
+}
+
+define i16 @notatomic_i16_global_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: notatomic_i16_global_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i16_global_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b16 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load volatile i16, ptr addrspace(1) %addr
+ ret i16 %retval
+}
+
+define i32 @notatomic_i32_global(ptr addrspace(1) %addr) {
+; CHECK-LABEL: notatomic_i32_global(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i32_global_param_0];
+; CHECK-NEXT: ld.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load i32, ptr addrspace(1) %addr
+ ret i32 %retval
+}
+
+define i32 @notatomic_i32_global_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: notatomic_i32_global_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i32_global_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load volatile i32, ptr addrspace(1) %addr
+ ret i32 %retval
+}
+
+define i64 @notatomic_i64_global(ptr addrspace(1) %addr) {
+; CHECK-LABEL: notatomic_i64_global(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i64_global_param_0];
+; CHECK-NEXT: ld.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load i64, ptr addrspace(1) %addr
+ ret i64 %retval
+}
+
+define i64 @notatomic_i64_global_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: notatomic_i64_global_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i64_global_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load volatile i64, ptr addrspace(1) %addr
+ ret i64 %retval
+}
+
+define half @notatomic_half_global(ptr addrspace(1) %addr) {
+; CHECK-LABEL: notatomic_half_global(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_half_global_param_0];
+; CHECK-NEXT: ld.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load half, ptr addrspace(1) %addr
+ ret half %retval
+}
+
+define half @notatomic_half_global_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: notatomic_half_global_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_half_global_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load volatile half, ptr addrspace(1) %addr
+ ret half %retval
+}
+
+define bfloat @notatomic_bfloat_global(ptr addrspace(1) %addr) {
+; CHECK-LABEL: notatomic_bfloat_global(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_bfloat_global_param_0];
+; CHECK-NEXT: ld.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load bfloat, ptr addrspace(1) %addr
+ ret bfloat %retval
+}
+
+define bfloat @notatomic_bfloat_global_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: notatomic_bfloat_global_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_bfloat_global_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load volatile bfloat, ptr addrspace(1) %addr
+ ret bfloat %retval
+}
+
+define float @notatomic_float_global(ptr addrspace(1) %addr) {
+; CHECK-LABEL: notatomic_float_global(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_float_global_param_0];
+; CHECK-NEXT: ld.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load float, ptr addrspace(1) %addr
+ ret float %retval
+}
+
+define float @notatomic_float_global_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: notatomic_float_global_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_float_global_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load volatile float, ptr addrspace(1) %addr
+ ret float %retval
+}
+
+define double @notatomic_double_global(ptr addrspace(1) %addr) {
+; CHECK-LABEL: notatomic_double_global(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_double_global_param_0];
+; CHECK-NEXT: ld.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load double, ptr addrspace(1) %addr
+ ret double %retval
+}
+
+define double @notatomic_double_global_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: notatomic_double_global_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_double_global_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load volatile double, ptr addrspace(1) %addr
+ ret double %retval
+}
+
+define i8 @notatomic_i8_shared(ptr addrspace(3) %addr) {
+; CHECK-LABEL: notatomic_i8_shared(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i8_shared_param_0];
+; CHECK-NEXT: ld.shared.b8 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load i8, ptr addrspace(3) %addr
+ ret i8 %retval
+}
+
+define i8 @notatomic_i8_shared_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: notatomic_i8_shared_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i8_shared_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b8 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load volatile i8, ptr addrspace(3) %addr
+ ret i8 %retval
+}
+
+define i16 @notatomic_i16_shared(ptr addrspace(3) %addr) {
+; CHECK-LABEL: notatomic_i16_shared(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i16_shared_param_0];
+; CHECK-NEXT: ld.shared.b16 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load i16, ptr addrspace(3) %addr
+ ret i16 %retval
+}
+
+define i16 @notatomic_i16_shared_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: notatomic_i16_shared_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i16_shared_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load volatile i16, ptr addrspace(3) %addr
+ ret i16 %retval
+}
+
+define i32 @notatomic_i32_shared(ptr addrspace(3) %addr) {
+; CHECK-LABEL: notatomic_i32_shared(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i32_shared_param_0];
+; CHECK-NEXT: ld.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load i32, ptr addrspace(3) %addr
+ ret i32 %retval
+}
+
+define i32 @notatomic_i32_shared_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: notatomic_i32_shared_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i32_shared_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load volatile i32, ptr addrspace(3) %addr
+ ret i32 %retval
+}
+
+define i64 @notatomic_i64_shared(ptr addrspace(3) %addr) {
+; CHECK-LABEL: notatomic_i64_shared(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i64_shared_param_0];
+; CHECK-NEXT: ld.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load i64, ptr addrspace(3) %addr
+ ret i64 %retval
+}
+
+define i64 @notatomic_i64_shared_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: notatomic_i64_shared_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i64_shared_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load volatile i64, ptr addrspace(3) %addr
+ ret i64 %retval
+}
+
+define half @notatomic_half_shared(ptr addrspace(3) %addr) {
+; CHECK-LABEL: notatomic_half_shared(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_half_shared_param_0];
+; CHECK-NEXT: ld.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load half, ptr addrspace(3) %addr
+ ret half %retval
+}
+
+define half @notatomic_half_shared_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: notatomic_half_shared_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_half_shared_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load volatile half, ptr addrspace(3) %addr
+ ret half %retval
+}
+
+define bfloat @notatomic_bfloat_shared(ptr addrspace(3) %addr) {
+; CHECK-LABEL: notatomic_bfloat_shared(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_bfloat_shared_param_0];
+; CHECK-NEXT: ld.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load bfloat, ptr addrspace(3) %addr
+ ret bfloat %retval
+}
+
+define bfloat @notatomic_bfloat_shared_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: notatomic_bfloat_shared_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_bfloat_shared_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load volatile bfloat, ptr addrspace(3) %addr
+ ret bfloat %retval
+}
+
+define float @notatomic_float_shared(ptr addrspace(3) %addr) {
+; CHECK-LABEL: notatomic_float_shared(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_float_shared_param_0];
+; CHECK-NEXT: ld.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load float, ptr addrspace(3) %addr
+ ret float %retval
+}
+
+define float @notatomic_float_shared_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: notatomic_float_shared_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_float_shared_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load volatile float, ptr addrspace(3) %addr
+ ret float %retval
+}
+
+define double @notatomic_double_shared(ptr addrspace(3) %addr) {
+; CHECK-LABEL: notatomic_double_shared(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_double_shared_param_0];
+; CHECK-NEXT: ld.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load double, ptr addrspace(3) %addr
+ ret double %retval
+}
+
+define double @notatomic_double_shared_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: notatomic_double_shared_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_double_shared_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load volatile double, ptr addrspace(3) %addr
+ ret double %retval
+}
+
+define i8 @notatomic_i8_const(ptr addrspace(4) %addr) {
+; CHECK-LABEL: notatomic_i8_const(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i8_const_param_0];
+; CHECK-NEXT: ld.const.b8 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load i8, ptr addrspace(4) %addr
+ ret i8 %retval
+}
+
+define i8 @notatomic_i8_const_volatile(ptr addrspace(4) %addr) {
+; CHECK-LABEL: notatomic_i8_const_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i8_const_volatile_param_0];
+; CHECK-NEXT: ld.const.b8 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load volatile i8, ptr addrspace(4) %addr
+ ret i8 %retval
+}
+
+define i16 @notatomic_i16_const(ptr addrspace(4) %addr) {
+; CHECK-LABEL: notatomic_i16_const(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i16_const_param_0];
+; CHECK-NEXT: ld.const.b16 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load i16, ptr addrspace(4) %addr
+ ret i16 %retval
+}
+
+define i16 @notatomic_i16_const_volatile(ptr addrspace(4) %addr) {
+; CHECK-LABEL: notatomic_i16_const_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i16_const_volatile_param_0];
+; CHECK-NEXT: ld.const.b16 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load volatile i16, ptr addrspace(4) %addr
+ ret i16 %retval
+}
+
+define i32 @notatomic_i32_const(ptr addrspace(4) %addr) {
+; CHECK-LABEL: notatomic_i32_const(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i32_const_param_0];
+; CHECK-NEXT: ld.const.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load i32, ptr addrspace(4) %addr
+ ret i32 %retval
+}
+
+define i32 @notatomic_i32_const_volatile(ptr addrspace(4) %addr) {
+; CHECK-LABEL: notatomic_i32_const_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i32_const_volatile_param_0];
+; CHECK-NEXT: ld.const.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load volatile i32, ptr addrspace(4) %addr
+ ret i32 %retval
+}
+
+define i64 @notatomic_i64_const(ptr addrspace(4) %addr) {
+; CHECK-LABEL: notatomic_i64_const(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i64_const_param_0];
+; CHECK-NEXT: ld.const.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load i64, ptr addrspace(4) %addr
+ ret i64 %retval
+}
+
+define i64 @notatomic_i64_const_volatile(ptr addrspace(4) %addr) {
+; CHECK-LABEL: notatomic_i64_const_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i64_const_volatile_param_0];
+; CHECK-NEXT: ld.const.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load volatile i64, ptr addrspace(4) %addr
+ ret i64 %retval
+}
+
+define half @notatomic_half_const(ptr addrspace(4) %addr) {
+; CHECK-LABEL: notatomic_half_const(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_half_const_param_0];
+; CHECK-NEXT: ld.const.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load half, ptr addrspace(4) %addr
+ ret half %retval
+}
+
+define half @notatomic_half_const_volatile(ptr addrspace(4) %addr) {
+; CHECK-LABEL: notatomic_half_const_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_half_const_volatile_param_0];
+; CHECK-NEXT: ld.const.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load volatile half, ptr addrspace(4) %addr
+ ret half %retval
+}
+
+define bfloat @notatomic_bfloat_const(ptr addrspace(4) %addr) {
+; CHECK-LABEL: notatomic_bfloat_const(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_bfloat_const_param_0];
+; CHECK-NEXT: ld.const.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load bfloat, ptr addrspace(4) %addr
+ ret bfloat %retval
+}
+
+define bfloat @notatomic_bfloat_const_volatile(ptr addrspace(4) %addr) {
+; CHECK-LABEL: notatomic_bfloat_const_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_bfloat_const_volatile_param_0];
+; CHECK-NEXT: ld.const.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load volatile bfloat, ptr addrspace(4) %addr
+ ret bfloat %retval
+}
+
+define float @notatomic_float_const(ptr addrspace(4) %addr) {
+; CHECK-LABEL: notatomic_float_const(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_float_const_param_0];
+; CHECK-NEXT: ld.const.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load float, ptr addrspace(4) %addr
+ ret float %retval
+}
+
+define float @notatomic_float_const_volatile(ptr addrspace(4) %addr) {
+; CHECK-LABEL: notatomic_float_const_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_float_const_volatile_param_0];
+; CHECK-NEXT: ld.const.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load volatile float, ptr addrspace(4) %addr
+ ret float %retval
+}
+
+define double @notatomic_double_const(ptr addrspace(4) %addr) {
+; CHECK-LABEL: notatomic_double_const(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_double_const_param_0];
+; CHECK-NEXT: ld.const.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load double, ptr addrspace(4) %addr
+ ret double %retval
+}
+
+define double @notatomic_double_const_volatile(ptr addrspace(4) %addr) {
+; CHECK-LABEL: notatomic_double_const_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_double_const_volatile_param_0];
+; CHECK-NEXT: ld.const.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load volatile double, ptr addrspace(4) %addr
+ ret double %retval
+}
+
+define i8 @notatomic_i8_local() {
+; CHECK-LABEL: notatomic_i8_local(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot64[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot64;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b8 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load i8, ptr addrspace(5) %slot
+ ret i8 %retval
+}
+
+define i8 @notatomic_i8_local_volatile() {
+; CHECK-LABEL: notatomic_i8_local_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot65[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot65;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b8 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load volatile i8, ptr addrspace(5) %slot
+ ret i8 %retval
+}
+
+define i16 @notatomic_i16_local() {
+; CHECK-LABEL: notatomic_i16_local(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot66[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot66;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load i16, ptr addrspace(5) %slot
+ ret i16 %retval
+}
+
+define i16 @notatomic_i16_local_volatile() {
+; CHECK-LABEL: notatomic_i16_local_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot67[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot67;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load volatile i16, ptr addrspace(5) %slot
+ ret i16 %retval
+}
+
+define i32 @notatomic_i32_local() {
+; CHECK-LABEL: notatomic_i32_local(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot68[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot68;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load i32, ptr addrspace(5) %slot
+ ret i32 %retval
+}
+
+define i32 @notatomic_i32_local_volatile() {
+; CHECK-LABEL: notatomic_i32_local_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot69[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot69;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load volatile i32, ptr addrspace(5) %slot
+ ret i32 %retval
+}
+
+define i64 @notatomic_i64_local() {
+; CHECK-LABEL: notatomic_i64_local(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot70[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot70;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load i64, ptr addrspace(5) %slot
+ ret i64 %retval
+}
+
+define i64 @notatomic_i64_local_volatile() {
+; CHECK-LABEL: notatomic_i64_local_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot71[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot71;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load volatile i64, ptr addrspace(5) %slot
+ ret i64 %retval
+}
+
+define half @notatomic_half_local() {
+; CHECK-LABEL: notatomic_half_local(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot72[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot72;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load half, ptr addrspace(5) %slot
+ ret half %retval
+}
+
+define half @notatomic_half_local_volatile() {
+; CHECK-LABEL: notatomic_half_local_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot73[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot73;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load volatile half, ptr addrspace(5) %slot
+ ret half %retval
+}
+
+define bfloat @notatomic_bfloat_local() {
+; CHECK-LABEL: notatomic_bfloat_local(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot74[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot74;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load bfloat, ptr addrspace(5) %slot
+ ret bfloat %retval
+}
+
+define bfloat @notatomic_bfloat_local_volatile() {
+; CHECK-LABEL: notatomic_bfloat_local_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot75[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot75;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load volatile bfloat, ptr addrspace(5) %slot
+ ret bfloat %retval
+}
+
+define float @notatomic_float_local() {
+; CHECK-LABEL: notatomic_float_local(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot76[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot76;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load float, ptr addrspace(5) %slot
+ ret float %retval
+}
+
+define float @notatomic_float_local_volatile() {
+; CHECK-LABEL: notatomic_float_local_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot77[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot77;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load volatile float, ptr addrspace(5) %slot
+ ret float %retval
+}
+
+define double @notatomic_double_local() {
+; CHECK-LABEL: notatomic_double_local(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot78[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot78;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca double, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load double, ptr addrspace(5) %slot
+ ret double %retval
+}
+
+define double @notatomic_double_local_volatile() {
+; CHECK-LABEL: notatomic_double_local_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot79[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot79;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca double, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load volatile double, ptr addrspace(5) %slot
+ ret double %retval
+}
+
+define i8 @notatomic_i8_param(ptr addrspace(101) %addr) {
+; CHECK-LABEL: notatomic_i8_param(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i8_param_param_0];
+; CHECK-NEXT: ld.param.b8 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load i8, ptr addrspace(101) %addr
+ ret i8 %retval
+}
+
+define i8 @notatomic_i8_param_volatile(ptr addrspace(101) %addr) {
+; CHECK-LABEL: notatomic_i8_param_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i8_param_volatile_param_0];
+; CHECK-NEXT: ld.param.b8 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load volatile i8, ptr addrspace(101) %addr
+ ret i8 %retval
+}
+
+define i16 @notatomic_i16_param(ptr addrspace(101) %addr) {
+; CHECK-LABEL: notatomic_i16_param(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i16_param_param_0];
+; CHECK-NEXT: ld.param.b16 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load i16, ptr addrspace(101) %addr
+ ret i16 %retval
+}
+
+define i16 @notatomic_i16_param_volatile(ptr addrspace(101) %addr) {
+; CHECK-LABEL: notatomic_i16_param_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i16_param_volatile_param_0];
+; CHECK-NEXT: ld.param.b16 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load volatile i16, ptr addrspace(101) %addr
+ ret i16 %retval
+}
+
+define i32 @notatomic_i32_param(ptr addrspace(101) %addr) {
+; CHECK-LABEL: notatomic_i32_param(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i32_param_param_0];
+; CHECK-NEXT: ld.param.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load i32, ptr addrspace(101) %addr
+ ret i32 %retval
+}
+
+define i32 @notatomic_i32_param_volatile(ptr addrspace(101) %addr) {
+; CHECK-LABEL: notatomic_i32_param_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i32_param_volatile_param_0];
+; CHECK-NEXT: ld.param.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load volatile i32, ptr addrspace(101) %addr
+ ret i32 %retval
+}
+
+define i64 @notatomic_i64_param(ptr addrspace(101) %addr) {
+; CHECK-LABEL: notatomic_i64_param(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i64_param_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load i64, ptr addrspace(101) %addr
+ ret i64 %retval
+}
+
+define i64 @notatomic_i64_param_volatile(ptr addrspace(101) %addr) {
+; CHECK-LABEL: notatomic_i64_param_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i64_param_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load volatile i64, ptr addrspace(101) %addr
+ ret i64 %retval
+}
+
+define half @notatomic_half_param(ptr addrspace(101) %addr) {
+; CHECK-LABEL: notatomic_half_param(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_half_param_param_0];
+; CHECK-NEXT: ld.param.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load half, ptr addrspace(101) %addr
+ ret half %retval
+}
+
+define half @notatomic_half_param_volatile(ptr addrspace(101) %addr) {
+; CHECK-LABEL: notatomic_half_param_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_half_param_volatile_param_0];
+; CHECK-NEXT: ld.param.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load volatile half, ptr addrspace(101) %addr
+ ret half %retval
+}
+
+define bfloat @notatomic_bfloat_param(ptr addrspace(101) %addr) {
+; CHECK-LABEL: notatomic_bfloat_param(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_bfloat_param_param_0];
+; CHECK-NEXT: ld.param.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load bfloat, ptr addrspace(101) %addr
+ ret bfloat %retval
+}
+
+define bfloat @notatomic_bfloat_param_volatile(ptr addrspace(101) %addr) {
+; CHECK-LABEL: notatomic_bfloat_param_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_bfloat_param_volatile_param_0];
+; CHECK-NEXT: ld.param.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load volatile bfloat, ptr addrspace(101) %addr
+ ret bfloat %retval
+}
+
+define float @notatomic_float_param(ptr addrspace(101) %addr) {
+; CHECK-LABEL: notatomic_float_param(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_float_param_param_0];
+; CHECK-NEXT: ld.param.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load float, ptr addrspace(101) %addr
+ ret float %retval
+}
+
+define float @notatomic_float_param_volatile(ptr addrspace(101) %addr) {
+; CHECK-LABEL: notatomic_float_param_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_float_param_volatile_param_0];
+; CHECK-NEXT: ld.param.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load volatile float, ptr addrspace(101) %addr
+ ret float %retval
+}
+
+define double @notatomic_double_param(ptr addrspace(101) %addr) {
+; CHECK-LABEL: notatomic_double_param(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_double_param_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load double, ptr addrspace(101) %addr
+ ret double %retval
+}
+
+define double @notatomic_double_param_volatile(ptr addrspace(101) %addr) {
+; CHECK-LABEL: notatomic_double_param_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_double_param_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load volatile double, ptr addrspace(101) %addr
+ ret double %retval
+}
+
+define i8 @unordered_i8_generic_sys(ptr %addr) {
+; CHECK-LABEL: unordered_i8_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_generic_sys_param_0];
+; CHECK-NEXT: ld.volatile.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr %addr syncscope("") unordered, align 1
+ ret i8 %retval
+}
+
+define i8 @unordered_i8_generic_cta(ptr %addr) {
+; CHECK-LABEL: unordered_i8_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_generic_cta_param_0];
+; CHECK-NEXT: ld.volatile.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr %addr syncscope("block") unordered, align 1
+ ret i8 %retval
+}
+
+define i8 @unordered_i8_generic_cluster(ptr %addr) {
+; CHECK-LABEL: unordered_i8_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_generic_cluster_param_0];
+; CHECK-NEXT: ld.volatile.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr %addr syncscope("cluster") unordered, align 1
+ ret i8 %retval
+}
+
+define i8 @unordered_i8_generic_gpu(ptr %addr) {
+; CHECK-LABEL: unordered_i8_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_generic_gpu_param_0];
+; CHECK-NEXT: ld.volatile.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr %addr syncscope("device") unordered, align 1
+ ret i8 %retval
+}
+
+define i8 @unordered_i8_generic_sys_volatile(ptr %addr) {
+; CHECK-LABEL: unordered_i8_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr %addr syncscope("") unordered, align 1
+ ret i8 %retval
+}
+
+define i8 @unordered_i8_generic_cta_volatile(ptr %addr) {
+; CHECK-LABEL: unordered_i8_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr %addr syncscope("block") unordered, align 1
+ ret i8 %retval
+}
+
+define i8 @unordered_i8_generic_cluster_volatile(ptr %addr) {
+; CHECK-LABEL: unordered_i8_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr %addr syncscope("cluster") unordered, align 1
+ ret i8 %retval
+}
+
+define i8 @unordered_i8_generic_gpu_volatile(ptr %addr) {
+; CHECK-LABEL: unordered_i8_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr %addr syncscope("device") unordered, align 1
+ ret i8 %retval
+}
+
+define i16 @unordered_i16_generic_sys(ptr %addr) {
+; CHECK-LABEL: unordered_i16_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_generic_sys_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr %addr syncscope("") unordered, align 2
+ ret i16 %retval
+}
+
+define i16 @unordered_i16_generic_cta(ptr %addr) {
+; CHECK-LABEL: unordered_i16_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_generic_cta_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr %addr syncscope("block") unordered, align 2
+ ret i16 %retval
+}
+
+define i16 @unordered_i16_generic_cluster(ptr %addr) {
+; CHECK-LABEL: unordered_i16_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_generic_cluster_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr %addr syncscope("cluster") unordered, align 2
+ ret i16 %retval
+}
+
+define i16 @unordered_i16_generic_gpu(ptr %addr) {
+; CHECK-LABEL: unordered_i16_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_generic_gpu_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr %addr syncscope("device") unordered, align 2
+ ret i16 %retval
+}
+
+define i16 @unordered_i16_generic_sys_volatile(ptr %addr) {
+; CHECK-LABEL: unordered_i16_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr %addr syncscope("") unordered, align 2
+ ret i16 %retval
+}
+
+define i16 @unordered_i16_generic_cta_volatile(ptr %addr) {
+; CHECK-LABEL: unordered_i16_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr %addr syncscope("block") unordered, align 2
+ ret i16 %retval
+}
+
+define i16 @unordered_i16_generic_cluster_volatile(ptr %addr) {
+; CHECK-LABEL: unordered_i16_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr %addr syncscope("cluster") unordered, align 2
+ ret i16 %retval
+}
+
+define i16 @unordered_i16_generic_gpu_volatile(ptr %addr) {
+; CHECK-LABEL: unordered_i16_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr %addr syncscope("device") unordered, align 2
+ ret i16 %retval
+}
+
+define i32 @unordered_i32_generic_sys(ptr %addr) {
+; CHECK-LABEL: unordered_i32_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_generic_sys_param_0];
+; CHECK-NEXT: ld.volatile.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr %addr syncscope("") unordered, align 4
+ ret i32 %retval
+}
+
+define i32 @unordered_i32_generic_cta(ptr %addr) {
+; CHECK-LABEL: unordered_i32_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_generic_cta_param_0];
+; CHECK-NEXT: ld.volatile.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr %addr syncscope("block") unordered, align 4
+ ret i32 %retval
+}
+
+define i32 @unordered_i32_generic_cluster(ptr %addr) {
+; CHECK-LABEL: unordered_i32_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_generic_cluster_param_0];
+; CHECK-NEXT: ld.volatile.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr %addr syncscope("cluster") unordered, align 4
+ ret i32 %retval
+}
+
+define i32 @unordered_i32_generic_gpu(ptr %addr) {
+; CHECK-LABEL: unordered_i32_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_generic_gpu_param_0];
+; CHECK-NEXT: ld.volatile.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr %addr syncscope("device") unordered, align 4
+ ret i32 %retval
+}
+
+define i32 @unordered_i32_generic_sys_volatile(ptr %addr) {
+; CHECK-LABEL: unordered_i32_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr %addr syncscope("") unordered, align 4
+ ret i32 %retval
+}
+
+define i32 @unordered_i32_generic_cta_volatile(ptr %addr) {
+; CHECK-LABEL: unordered_i32_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr %addr syncscope("block") unordered, align 4
+ ret i32 %retval
+}
+
+define i32 @unordered_i32_generic_cluster_volatile(ptr %addr) {
+; CHECK-LABEL: unordered_i32_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr %addr syncscope("cluster") unordered, align 4
+ ret i32 %retval
+}
+
+define i32 @unordered_i32_generic_gpu_volatile(ptr %addr) {
+; CHECK-LABEL: unordered_i32_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr %addr syncscope("device") unordered, align 4
+ ret i32 %retval
+}
+
+define i64 @unordered_i64_generic_sys(ptr %addr) {
+; CHECK-LABEL: unordered_i64_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_generic_sys_param_0];
+; CHECK-NEXT: ld.volatile.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr %addr syncscope("") unordered, align 8
+ ret i64 %retval
+}
+
+define i64 @unordered_i64_generic_cta(ptr %addr) {
+; CHECK-LABEL: unordered_i64_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_generic_cta_param_0];
+; CHECK-NEXT: ld.volatile.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr %addr syncscope("block") unordered, align 8
+ ret i64 %retval
+}
+
+define i64 @unordered_i64_generic_cluster(ptr %addr) {
+; CHECK-LABEL: unordered_i64_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_generic_cluster_param_0];
+; CHECK-NEXT: ld.volatile.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr %addr syncscope("cluster") unordered, align 8
+ ret i64 %retval
+}
+
+define i64 @unordered_i64_generic_gpu(ptr %addr) {
+; CHECK-LABEL: unordered_i64_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_generic_gpu_param_0];
+; CHECK-NEXT: ld.volatile.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr %addr syncscope("device") unordered, align 8
+ ret i64 %retval
+}
+
+define i64 @unordered_i64_generic_sys_volatile(ptr %addr) {
+; CHECK-LABEL: unordered_i64_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr %addr syncscope("") unordered, align 8
+ ret i64 %retval
+}
+
+define i64 @unordered_i64_generic_cta_volatile(ptr %addr) {
+; CHECK-LABEL: unordered_i64_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr %addr syncscope("block") unordered, align 8
+ ret i64 %retval
+}
+
+define i64 @unordered_i64_generic_cluster_volatile(ptr %addr) {
+; CHECK-LABEL: unordered_i64_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr %addr syncscope("cluster") unordered, align 8
+ ret i64 %retval
+}
+
+define i64 @unordered_i64_generic_gpu_volatile(ptr %addr) {
+; CHECK-LABEL: unordered_i64_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr %addr syncscope("device") unordered, align 8
+ ret i64 %retval
+}
+
+define half @unordered_half_generic_sys(ptr %addr) {
+; CHECK-LABEL: unordered_half_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_generic_sys_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr %addr syncscope("") unordered, align 2
+ ret half %retval
+}
+
+define half @unordered_half_generic_cta(ptr %addr) {
+; CHECK-LABEL: unordered_half_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_generic_cta_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr %addr syncscope("block") unordered, align 2
+ ret half %retval
+}
+
+define half @unordered_half_generic_cluster(ptr %addr) {
+; CHECK-LABEL: unordered_half_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_generic_cluster_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr %addr syncscope("cluster") unordered, align 2
+ ret half %retval
+}
+
+define half @unordered_half_generic_gpu(ptr %addr) {
+; CHECK-LABEL: unordered_half_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_generic_gpu_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr %addr syncscope("device") unordered, align 2
+ ret half %retval
+}
+
+define half @unordered_half_generic_sys_volatile(ptr %addr) {
+; CHECK-LABEL: unordered_half_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr %addr syncscope("") unordered, align 2
+ ret half %retval
+}
+
+define half @unordered_half_generic_cta_volatile(ptr %addr) {
+; CHECK-LABEL: unordered_half_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr %addr syncscope("block") unordered, align 2
+ ret half %retval
+}
+
+define half @unordered_half_generic_cluster_volatile(ptr %addr) {
+; CHECK-LABEL: unordered_half_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr %addr syncscope("cluster") unordered, align 2
+ ret half %retval
+}
+
+define half @unordered_half_generic_gpu_volatile(ptr %addr) {
+; CHECK-LABEL: unordered_half_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr %addr syncscope("device") unordered, align 2
+ ret half %retval
+}
+
+define bfloat @unordered_bfloat_generic_sys(ptr %addr) {
+; CHECK-LABEL: unordered_bfloat_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_generic_sys_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr %addr syncscope("") unordered, align 2
+ ret bfloat %retval
+}
+
+define bfloat @unordered_bfloat_generic_cta(ptr %addr) {
+; CHECK-LABEL: unordered_bfloat_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_generic_cta_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr %addr syncscope("block") unordered, align 2
+ ret bfloat %retval
+}
+
+define bfloat @unordered_bfloat_generic_cluster(ptr %addr) {
+; CHECK-LABEL: unordered_bfloat_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_generic_cluster_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr %addr syncscope("cluster") unordered, align 2
+ ret bfloat %retval
+}
+
+define bfloat @unordered_bfloat_generic_gpu(ptr %addr) {
+; CHECK-LABEL: unordered_bfloat_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_generic_gpu_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr %addr syncscope("device") unordered, align 2
+ ret bfloat %retval
+}
+
+define bfloat @unordered_bfloat_generic_sys_volatile(ptr %addr) {
+; CHECK-LABEL: unordered_bfloat_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr %addr syncscope("") unordered, align 2
+ ret bfloat %retval
+}
+
+define bfloat @unordered_bfloat_generic_cta_volatile(ptr %addr) {
+; CHECK-LABEL: unordered_bfloat_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr %addr syncscope("block") unordered, align 2
+ ret bfloat %retval
+}
+
+define bfloat @unordered_bfloat_generic_cluster_volatile(ptr %addr) {
+; CHECK-LABEL: unordered_bfloat_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr %addr syncscope("cluster") unordered, align 2
+ ret bfloat %retval
+}
+
+define bfloat @unordered_bfloat_generic_gpu_volatile(ptr %addr) {
+; CHECK-LABEL: unordered_bfloat_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr %addr syncscope("device") unordered, align 2
+ ret bfloat %retval
+}
+
+define float @unordered_float_generic_sys(ptr %addr) {
+; CHECK-LABEL: unordered_float_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_generic_sys_param_0];
+; CHECK-NEXT: ld.volatile.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr %addr syncscope("") unordered, align 4
+ ret float %retval
+}
+
+define float @unordered_float_generic_cta(ptr %addr) {
+; CHECK-LABEL: unordered_float_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_generic_cta_param_0];
+; CHECK-NEXT: ld.volatile.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr %addr syncscope("block") unordered, align 4
+ ret float %retval
+}
+
+define float @unordered_float_generic_cluster(ptr %addr) {
+; CHECK-LABEL: unordered_float_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_generic_cluster_param_0];
+; CHECK-NEXT: ld.volatile.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr %addr syncscope("cluster") unordered, align 4
+ ret float %retval
+}
+
+define float @unordered_float_generic_gpu(ptr %addr) {
+; CHECK-LABEL: unordered_float_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_generic_gpu_param_0];
+; CHECK-NEXT: ld.volatile.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr %addr syncscope("device") unordered, align 4
+ ret float %retval
+}
+
+define float @unordered_float_generic_sys_volatile(ptr %addr) {
+; CHECK-LABEL: unordered_float_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr %addr syncscope("") unordered, align 4
+ ret float %retval
+}
+
+define float @unordered_float_generic_cta_volatile(ptr %addr) {
+; CHECK-LABEL: unordered_float_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr %addr syncscope("block") unordered, align 4
+ ret float %retval
+}
+
+define float @unordered_float_generic_cluster_volatile(ptr %addr) {
+; CHECK-LABEL: unordered_float_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr %addr syncscope("cluster") unordered, align 4
+ ret float %retval
+}
+
+define float @unordered_float_generic_gpu_volatile(ptr %addr) {
+; CHECK-LABEL: unordered_float_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr %addr syncscope("device") unordered, align 4
+ ret float %retval
+}
+
+define double @unordered_double_generic_sys(ptr %addr) {
+; CHECK-LABEL: unordered_double_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_generic_sys_param_0];
+; CHECK-NEXT: ld.volatile.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr %addr syncscope("") unordered, align 8
+ ret double %retval
+}
+
+define double @unordered_double_generic_cta(ptr %addr) {
+; CHECK-LABEL: unordered_double_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_generic_cta_param_0];
+; CHECK-NEXT: ld.volatile.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr %addr syncscope("block") unordered, align 8
+ ret double %retval
+}
+
+define double @unordered_double_generic_cluster(ptr %addr) {
+; CHECK-LABEL: unordered_double_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_generic_cluster_param_0];
+; CHECK-NEXT: ld.volatile.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr %addr syncscope("cluster") unordered, align 8
+ ret double %retval
+}
+
+define double @unordered_double_generic_gpu(ptr %addr) {
+; CHECK-LABEL: unordered_double_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_generic_gpu_param_0];
+; CHECK-NEXT: ld.volatile.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr %addr syncscope("device") unordered, align 8
+ ret double %retval
+}
+
+define double @unordered_double_generic_sys_volatile(ptr %addr) {
+; CHECK-LABEL: unordered_double_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr %addr syncscope("") unordered, align 8
+ ret double %retval
+}
+
+define double @unordered_double_generic_cta_volatile(ptr %addr) {
+; CHECK-LABEL: unordered_double_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr %addr syncscope("block") unordered, align 8
+ ret double %retval
+}
+
+define double @unordered_double_generic_cluster_volatile(ptr %addr) {
+; CHECK-LABEL: unordered_double_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr %addr syncscope("cluster") unordered, align 8
+ ret double %retval
+}
+
+define double @unordered_double_generic_gpu_volatile(ptr %addr) {
+; CHECK-LABEL: unordered_double_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr %addr syncscope("device") unordered, align 8
+ ret double %retval
+}
+
+define i8 @unordered_i8_global_sys(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i8_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_global_sys_param_0];
+; CHECK-NEXT: ld.volatile.global.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr addrspace(1) %addr syncscope("") unordered, align 1
+ ret i8 %retval
+}
+
+define i8 @unordered_i8_global_cta(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i8_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_global_cta_param_0];
+; CHECK-NEXT: ld.volatile.global.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr addrspace(1) %addr syncscope("block") unordered, align 1
+ ret i8 %retval
+}
+
+define i8 @unordered_i8_global_cluster(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i8_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_global_cluster_param_0];
+; CHECK-NEXT: ld.volatile.global.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr addrspace(1) %addr syncscope("cluster") unordered, align 1
+ ret i8 %retval
+}
+
+define i8 @unordered_i8_global_gpu(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i8_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_global_gpu_param_0];
+; CHECK-NEXT: ld.volatile.global.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr addrspace(1) %addr syncscope("device") unordered, align 1
+ ret i8 %retval
+}
+
+define i8 @unordered_i8_global_sys_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i8_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr addrspace(1) %addr syncscope("") unordered, align 1
+ ret i8 %retval
+}
+
+define i8 @unordered_i8_global_cta_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i8_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr addrspace(1) %addr syncscope("block") unordered, align 1
+ ret i8 %retval
+}
+
+define i8 @unordered_i8_global_cluster_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i8_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr addrspace(1) %addr syncscope("cluster") unordered, align 1
+ ret i8 %retval
+}
+
+define i8 @unordered_i8_global_gpu_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i8_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr addrspace(1) %addr syncscope("device") unordered, align 1
+ ret i8 %retval
+}
+
+define i16 @unordered_i16_global_sys(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i16_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_global_sys_param_0];
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr addrspace(1) %addr syncscope("") unordered, align 2
+ ret i16 %retval
+}
+
+define i16 @unordered_i16_global_cta(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i16_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_global_cta_param_0];
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr addrspace(1) %addr syncscope("block") unordered, align 2
+ ret i16 %retval
+}
+
+define i16 @unordered_i16_global_cluster(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i16_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_global_cluster_param_0];
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr addrspace(1) %addr syncscope("cluster") unordered, align 2
+ ret i16 %retval
+}
+
+define i16 @unordered_i16_global_gpu(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i16_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_global_gpu_param_0];
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr addrspace(1) %addr syncscope("device") unordered, align 2
+ ret i16 %retval
+}
+
+define i16 @unordered_i16_global_sys_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i16_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr addrspace(1) %addr syncscope("") unordered, align 2
+ ret i16 %retval
+}
+
+define i16 @unordered_i16_global_cta_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i16_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr addrspace(1) %addr syncscope("block") unordered, align 2
+ ret i16 %retval
+}
+
+define i16 @unordered_i16_global_cluster_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i16_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr addrspace(1) %addr syncscope("cluster") unordered, align 2
+ ret i16 %retval
+}
+
+define i16 @unordered_i16_global_gpu_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i16_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr addrspace(1) %addr syncscope("device") unordered, align 2
+ ret i16 %retval
+}
+
+define i32 @unordered_i32_global_sys(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i32_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_global_sys_param_0];
+; CHECK-NEXT: ld.volatile.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr addrspace(1) %addr syncscope("") unordered, align 4
+ ret i32 %retval
+}
+
+define i32 @unordered_i32_global_cta(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i32_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_global_cta_param_0];
+; CHECK-NEXT: ld.volatile.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr addrspace(1) %addr syncscope("block") unordered, align 4
+ ret i32 %retval
+}
+
+define i32 @unordered_i32_global_cluster(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i32_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_global_cluster_param_0];
+; CHECK-NEXT: ld.volatile.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr addrspace(1) %addr syncscope("cluster") unordered, align 4
+ ret i32 %retval
+}
+
+define i32 @unordered_i32_global_gpu(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i32_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_global_gpu_param_0];
+; CHECK-NEXT: ld.volatile.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr addrspace(1) %addr syncscope("device") unordered, align 4
+ ret i32 %retval
+}
+
+define i32 @unordered_i32_global_sys_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i32_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr addrspace(1) %addr syncscope("") unordered, align 4
+ ret i32 %retval
+}
+
+define i32 @unordered_i32_global_cta_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i32_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr addrspace(1) %addr syncscope("block") unordered, align 4
+ ret i32 %retval
+}
+
+define i32 @unordered_i32_global_cluster_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i32_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr addrspace(1) %addr syncscope("cluster") unordered, align 4
+ ret i32 %retval
+}
+
+define i32 @unordered_i32_global_gpu_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i32_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr addrspace(1) %addr syncscope("device") unordered, align 4
+ ret i32 %retval
+}
+
+define i64 @unordered_i64_global_sys(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i64_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_global_sys_param_0];
+; CHECK-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr addrspace(1) %addr syncscope("") unordered, align 8
+ ret i64 %retval
+}
+
+define i64 @unordered_i64_global_cta(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i64_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_global_cta_param_0];
+; CHECK-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr addrspace(1) %addr syncscope("block") unordered, align 8
+ ret i64 %retval
+}
+
+define i64 @unordered_i64_global_cluster(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i64_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_global_cluster_param_0];
+; CHECK-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr addrspace(1) %addr syncscope("cluster") unordered, align 8
+ ret i64 %retval
+}
+
+define i64 @unordered_i64_global_gpu(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i64_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_global_gpu_param_0];
+; CHECK-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr addrspace(1) %addr syncscope("device") unordered, align 8
+ ret i64 %retval
+}
+
+define i64 @unordered_i64_global_sys_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i64_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr addrspace(1) %addr syncscope("") unordered, align 8
+ ret i64 %retval
+}
+
+define i64 @unordered_i64_global_cta_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i64_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr addrspace(1) %addr syncscope("block") unordered, align 8
+ ret i64 %retval
+}
+
+define i64 @unordered_i64_global_cluster_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i64_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr addrspace(1) %addr syncscope("cluster") unordered, align 8
+ ret i64 %retval
+}
+
+define i64 @unordered_i64_global_gpu_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i64_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr addrspace(1) %addr syncscope("device") unordered, align 8
+ ret i64 %retval
+}
+
+define half @unordered_half_global_sys(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_half_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_global_sys_param_0];
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr addrspace(1) %addr syncscope("") unordered, align 2
+ ret half %retval
+}
+
+define half @unordered_half_global_cta(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_half_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_global_cta_param_0];
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr addrspace(1) %addr syncscope("block") unordered, align 2
+ ret half %retval
+}
+
+define half @unordered_half_global_cluster(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_half_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_global_cluster_param_0];
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr addrspace(1) %addr syncscope("cluster") unordered, align 2
+ ret half %retval
+}
+
+define half @unordered_half_global_gpu(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_half_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_global_gpu_param_0];
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr addrspace(1) %addr syncscope("device") unordered, align 2
+ ret half %retval
+}
+
+define half @unordered_half_global_sys_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_half_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr addrspace(1) %addr syncscope("") unordered, align 2
+ ret half %retval
+}
+
+define half @unordered_half_global_cta_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_half_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr addrspace(1) %addr syncscope("block") unordered, align 2
+ ret half %retval
+}
+
+define half @unordered_half_global_cluster_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_half_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr addrspace(1) %addr syncscope("cluster") unordered, align 2
+ ret half %retval
+}
+
+define half @unordered_half_global_gpu_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_half_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr addrspace(1) %addr syncscope("device") unordered, align 2
+ ret half %retval
+}
+
+define bfloat @unordered_bfloat_global_sys(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_bfloat_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_global_sys_param_0];
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr addrspace(1) %addr syncscope("") unordered, align 2
+ ret bfloat %retval
+}
+
+define bfloat @unordered_bfloat_global_cta(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_bfloat_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_global_cta_param_0];
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr addrspace(1) %addr syncscope("block") unordered, align 2
+ ret bfloat %retval
+}
+
+define bfloat @unordered_bfloat_global_cluster(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_bfloat_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_global_cluster_param_0];
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr addrspace(1) %addr syncscope("cluster") unordered, align 2
+ ret bfloat %retval
+}
+
+define bfloat @unordered_bfloat_global_gpu(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_bfloat_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_global_gpu_param_0];
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr addrspace(1) %addr syncscope("device") unordered, align 2
+ ret bfloat %retval
+}
+
+define bfloat @unordered_bfloat_global_sys_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_bfloat_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr addrspace(1) %addr syncscope("") unordered, align 2
+ ret bfloat %retval
+}
+
+define bfloat @unordered_bfloat_global_cta_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_bfloat_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr addrspace(1) %addr syncscope("block") unordered, align 2
+ ret bfloat %retval
+}
+
+define bfloat @unordered_bfloat_global_cluster_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_bfloat_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr addrspace(1) %addr syncscope("cluster") unordered, align 2
+ ret bfloat %retval
+}
+
+define bfloat @unordered_bfloat_global_gpu_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_bfloat_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr addrspace(1) %addr syncscope("device") unordered, align 2
+ ret bfloat %retval
+}
+
+define float @unordered_float_global_sys(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_float_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_global_sys_param_0];
+; CHECK-NEXT: ld.volatile.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr addrspace(1) %addr syncscope("") unordered, align 4
+ ret float %retval
+}
+
+define float @unordered_float_global_cta(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_float_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_global_cta_param_0];
+; CHECK-NEXT: ld.volatile.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr addrspace(1) %addr syncscope("block") unordered, align 4
+ ret float %retval
+}
+
+define float @unordered_float_global_cluster(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_float_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_global_cluster_param_0];
+; CHECK-NEXT: ld.volatile.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr addrspace(1) %addr syncscope("cluster") unordered, align 4
+ ret float %retval
+}
+
+define float @unordered_float_global_gpu(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_float_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_global_gpu_param_0];
+; CHECK-NEXT: ld.volatile.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr addrspace(1) %addr syncscope("device") unordered, align 4
+ ret float %retval
+}
+
+define float @unordered_float_global_sys_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_float_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr addrspace(1) %addr syncscope("") unordered, align 4
+ ret float %retval
+}
+
+define float @unordered_float_global_cta_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_float_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr addrspace(1) %addr syncscope("block") unordered, align 4
+ ret float %retval
+}
+
+define float @unordered_float_global_cluster_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_float_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr addrspace(1) %addr syncscope("cluster") unordered, align 4
+ ret float %retval
+}
+
+define float @unordered_float_global_gpu_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_float_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr addrspace(1) %addr syncscope("device") unordered, align 4
+ ret float %retval
+}
+
+define double @unordered_double_global_sys(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_double_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_global_sys_param_0];
+; CHECK-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr addrspace(1) %addr syncscope("") unordered, align 8
+ ret double %retval
+}
+
+define double @unordered_double_global_cta(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_double_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_global_cta_param_0];
+; CHECK-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr addrspace(1) %addr syncscope("block") unordered, align 8
+ ret double %retval
+}
+
+define double @unordered_double_global_cluster(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_double_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_global_cluster_param_0];
+; CHECK-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr addrspace(1) %addr syncscope("cluster") unordered, align 8
+ ret double %retval
+}
+
+define double @unordered_double_global_gpu(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_double_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_global_gpu_param_0];
+; CHECK-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr addrspace(1) %addr syncscope("device") unordered, align 8
+ ret double %retval
+}
+
+define double @unordered_double_global_sys_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_double_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr addrspace(1) %addr syncscope("") unordered, align 8
+ ret double %retval
+}
+
+define double @unordered_double_global_cta_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_double_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr addrspace(1) %addr syncscope("block") unordered, align 8
+ ret double %retval
+}
+
+define double @unordered_double_global_cluster_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_double_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr addrspace(1) %addr syncscope("cluster") unordered, align 8
+ ret double %retval
+}
+
+define double @unordered_double_global_gpu_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_double_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr addrspace(1) %addr syncscope("device") unordered, align 8
+ ret double %retval
+}
+
+define i8 @unordered_i8_shared_sys(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i8_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_shared_sys_param_0];
+; CHECK-NEXT: ld.volatile.shared.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr addrspace(3) %addr syncscope("") unordered, align 1
+ ret i8 %retval
+}
+
+define i8 @unordered_i8_shared_cta(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i8_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_shared_cta_param_0];
+; CHECK-NEXT: ld.volatile.shared.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr addrspace(3) %addr syncscope("block") unordered, align 1
+ ret i8 %retval
+}
+
+define i8 @unordered_i8_shared_cluster(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i8_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_shared_cluster_param_0];
+; CHECK-NEXT: ld.volatile.shared.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr addrspace(3) %addr syncscope("cluster") unordered, align 1
+ ret i8 %retval
+}
+
+define i8 @unordered_i8_shared_gpu(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i8_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_shared_gpu_param_0];
+; CHECK-NEXT: ld.volatile.shared.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr addrspace(3) %addr syncscope("device") unordered, align 1
+ ret i8 %retval
+}
+
+define i8 @unordered_i8_shared_sys_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i8_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr addrspace(3) %addr syncscope("") unordered, align 1
+ ret i8 %retval
+}
+
+define i8 @unordered_i8_shared_cta_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i8_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr addrspace(3) %addr syncscope("block") unordered, align 1
+ ret i8 %retval
+}
+
+define i8 @unordered_i8_shared_cluster_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i8_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr addrspace(3) %addr syncscope("cluster") unordered, align 1
+ ret i8 %retval
+}
+
+define i8 @unordered_i8_shared_gpu_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i8_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr addrspace(3) %addr syncscope("device") unordered, align 1
+ ret i8 %retval
+}
+
+define i16 @unordered_i16_shared_sys(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i16_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_shared_sys_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr addrspace(3) %addr syncscope("") unordered, align 2
+ ret i16 %retval
+}
+
+define i16 @unordered_i16_shared_cta(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i16_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_shared_cta_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr addrspace(3) %addr syncscope("block") unordered, align 2
+ ret i16 %retval
+}
+
+define i16 @unordered_i16_shared_cluster(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i16_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_shared_cluster_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr addrspace(3) %addr syncscope("cluster") unordered, align 2
+ ret i16 %retval
+}
+
+define i16 @unordered_i16_shared_gpu(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i16_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_shared_gpu_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr addrspace(3) %addr syncscope("device") unordered, align 2
+ ret i16 %retval
+}
+
+define i16 @unordered_i16_shared_sys_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i16_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr addrspace(3) %addr syncscope("") unordered, align 2
+ ret i16 %retval
+}
+
+define i16 @unordered_i16_shared_cta_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i16_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr addrspace(3) %addr syncscope("block") unordered, align 2
+ ret i16 %retval
+}
+
+define i16 @unordered_i16_shared_cluster_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i16_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr addrspace(3) %addr syncscope("cluster") unordered, align 2
+ ret i16 %retval
+}
+
+define i16 @unordered_i16_shared_gpu_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i16_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr addrspace(3) %addr syncscope("device") unordered, align 2
+ ret i16 %retval
+}
+
+define i32 @unordered_i32_shared_sys(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i32_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_shared_sys_param_0];
+; CHECK-NEXT: ld.volatile.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr addrspace(3) %addr syncscope("") unordered, align 4
+ ret i32 %retval
+}
+
+define i32 @unordered_i32_shared_cta(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i32_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_shared_cta_param_0];
+; CHECK-NEXT: ld.volatile.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr addrspace(3) %addr syncscope("block") unordered, align 4
+ ret i32 %retval
+}
+
+define i32 @unordered_i32_shared_cluster(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i32_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_shared_cluster_param_0];
+; CHECK-NEXT: ld.volatile.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr addrspace(3) %addr syncscope("cluster") unordered, align 4
+ ret i32 %retval
+}
+
+define i32 @unordered_i32_shared_gpu(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i32_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_shared_gpu_param_0];
+; CHECK-NEXT: ld.volatile.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr addrspace(3) %addr syncscope("device") unordered, align 4
+ ret i32 %retval
+}
+
+define i32 @unordered_i32_shared_sys_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i32_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr addrspace(3) %addr syncscope("") unordered, align 4
+ ret i32 %retval
+}
+
+define i32 @unordered_i32_shared_cta_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i32_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr addrspace(3) %addr syncscope("block") unordered, align 4
+ ret i32 %retval
+}
+
+define i32 @unordered_i32_shared_cluster_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i32_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr addrspace(3) %addr syncscope("cluster") unordered, align 4
+ ret i32 %retval
+}
+
+define i32 @unordered_i32_shared_gpu_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i32_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr addrspace(3) %addr syncscope("device") unordered, align 4
+ ret i32 %retval
+}
+
+define i64 @unordered_i64_shared_sys(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i64_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_shared_sys_param_0];
+; CHECK-NEXT: ld.volatile.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr addrspace(3) %addr syncscope("") unordered, align 8
+ ret i64 %retval
+}
+
+define i64 @unordered_i64_shared_cta(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i64_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_shared_cta_param_0];
+; CHECK-NEXT: ld.volatile.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr addrspace(3) %addr syncscope("block") unordered, align 8
+ ret i64 %retval
+}
+
+define i64 @unordered_i64_shared_cluster(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i64_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_shared_cluster_param_0];
+; CHECK-NEXT: ld.volatile.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr addrspace(3) %addr syncscope("cluster") unordered, align 8
+ ret i64 %retval
+}
+
+define i64 @unordered_i64_shared_gpu(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i64_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_shared_gpu_param_0];
+; CHECK-NEXT: ld.volatile.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr addrspace(3) %addr syncscope("device") unordered, align 8
+ ret i64 %retval
+}
+
+define i64 @unordered_i64_shared_sys_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i64_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr addrspace(3) %addr syncscope("") unordered, align 8
+ ret i64 %retval
+}
+
+define i64 @unordered_i64_shared_cta_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i64_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr addrspace(3) %addr syncscope("block") unordered, align 8
+ ret i64 %retval
+}
+
+define i64 @unordered_i64_shared_cluster_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i64_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr addrspace(3) %addr syncscope("cluster") unordered, align 8
+ ret i64 %retval
+}
+
+define i64 @unordered_i64_shared_gpu_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i64_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr addrspace(3) %addr syncscope("device") unordered, align 8
+ ret i64 %retval
+}
+
+define half @unordered_half_shared_sys(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_half_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_shared_sys_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr addrspace(3) %addr syncscope("") unordered, align 2
+ ret half %retval
+}
+
+define half @unordered_half_shared_cta(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_half_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_shared_cta_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr addrspace(3) %addr syncscope("block") unordered, align 2
+ ret half %retval
+}
+
+define half @unordered_half_shared_cluster(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_half_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_shared_cluster_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr addrspace(3) %addr syncscope("cluster") unordered, align 2
+ ret half %retval
+}
+
+define half @unordered_half_shared_gpu(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_half_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_shared_gpu_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr addrspace(3) %addr syncscope("device") unordered, align 2
+ ret half %retval
+}
+
+define half @unordered_half_shared_sys_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_half_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr addrspace(3) %addr syncscope("") unordered, align 2
+ ret half %retval
+}
+
+define half @unordered_half_shared_cta_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_half_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr addrspace(3) %addr syncscope("block") unordered, align 2
+ ret half %retval
+}
+
+define half @unordered_half_shared_cluster_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_half_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr addrspace(3) %addr syncscope("cluster") unordered, align 2
+ ret half %retval
+}
+
+define half @unordered_half_shared_gpu_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_half_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr addrspace(3) %addr syncscope("device") unordered, align 2
+ ret half %retval
+}
+
+define bfloat @unordered_bfloat_shared_sys(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_bfloat_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_shared_sys_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr addrspace(3) %addr syncscope("") unordered, align 2
+ ret bfloat %retval
+}
+
+define bfloat @unordered_bfloat_shared_cta(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_bfloat_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_shared_cta_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr addrspace(3) %addr syncscope("block") unordered, align 2
+ ret bfloat %retval
+}
+
+define bfloat @unordered_bfloat_shared_cluster(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_bfloat_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_shared_cluster_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr addrspace(3) %addr syncscope("cluster") unordered, align 2
+ ret bfloat %retval
+}
+
+define bfloat @unordered_bfloat_shared_gpu(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_bfloat_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_shared_gpu_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr addrspace(3) %addr syncscope("device") unordered, align 2
+ ret bfloat %retval
+}
+
+define bfloat @unordered_bfloat_shared_sys_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_bfloat_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr addrspace(3) %addr syncscope("") unordered, align 2
+ ret bfloat %retval
+}
+
+define bfloat @unordered_bfloat_shared_cta_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_bfloat_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr addrspace(3) %addr syncscope("block") unordered, align 2
+ ret bfloat %retval
+}
+
+define bfloat @unordered_bfloat_shared_cluster_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_bfloat_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr addrspace(3) %addr syncscope("cluster") unordered, align 2
+ ret bfloat %retval
+}
+
+define bfloat @unordered_bfloat_shared_gpu_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_bfloat_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr addrspace(3) %addr syncscope("device") unordered, align 2
+ ret bfloat %retval
+}
+
+define float @unordered_float_shared_sys(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_float_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_shared_sys_param_0];
+; CHECK-NEXT: ld.volatile.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr addrspace(3) %addr syncscope("") unordered, align 4
+ ret float %retval
+}
+
+define float @unordered_float_shared_cta(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_float_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_shared_cta_param_0];
+; CHECK-NEXT: ld.volatile.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr addrspace(3) %addr syncscope("block") unordered, align 4
+ ret float %retval
+}
+
+define float @unordered_float_shared_cluster(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_float_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_shared_cluster_param_0];
+; CHECK-NEXT: ld.volatile.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr addrspace(3) %addr syncscope("cluster") unordered, align 4
+ ret float %retval
+}
+
+define float @unordered_float_shared_gpu(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_float_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_shared_gpu_param_0];
+; CHECK-NEXT: ld.volatile.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr addrspace(3) %addr syncscope("device") unordered, align 4
+ ret float %retval
+}
+
+define float @unordered_float_shared_sys_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_float_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr addrspace(3) %addr syncscope("") unordered, align 4
+ ret float %retval
+}
+
+define float @unordered_float_shared_cta_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_float_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr addrspace(3) %addr syncscope("block") unordered, align 4
+ ret float %retval
+}
+
+define float @unordered_float_shared_cluster_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_float_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr addrspace(3) %addr syncscope("cluster") unordered, align 4
+ ret float %retval
+}
+
+define float @unordered_float_shared_gpu_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_float_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr addrspace(3) %addr syncscope("device") unordered, align 4
+ ret float %retval
+}
+
+define double @unordered_double_shared_sys(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_double_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_shared_sys_param_0];
+; CHECK-NEXT: ld.volatile.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr addrspace(3) %addr syncscope("") unordered, align 8
+ ret double %retval
+}
+
+define double @unordered_double_shared_cta(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_double_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_shared_cta_param_0];
+; CHECK-NEXT: ld.volatile.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr addrspace(3) %addr syncscope("block") unordered, align 8
+ ret double %retval
+}
+
+define double @unordered_double_shared_cluster(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_double_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_shared_cluster_param_0];
+; CHECK-NEXT: ld.volatile.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr addrspace(3) %addr syncscope("cluster") unordered, align 8
+ ret double %retval
+}
+
+define double @unordered_double_shared_gpu(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_double_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_shared_gpu_param_0];
+; CHECK-NEXT: ld.volatile.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr addrspace(3) %addr syncscope("device") unordered, align 8
+ ret double %retval
+}
+
+define double @unordered_double_shared_sys_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_double_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr addrspace(3) %addr syncscope("") unordered, align 8
+ ret double %retval
+}
+
+define double @unordered_double_shared_cta_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_double_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr addrspace(3) %addr syncscope("block") unordered, align 8
+ ret double %retval
+}
+
+define double @unordered_double_shared_cluster_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_double_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr addrspace(3) %addr syncscope("cluster") unordered, align 8
+ ret double %retval
+}
+
+define double @unordered_double_shared_gpu_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_double_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr addrspace(3) %addr syncscope("device") unordered, align 8
+ ret double %retval
+}
+
+define i8 @unordered_i8_local_sys() {
+; CHECK-LABEL: unordered_i8_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot288[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot288;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b8 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i8, ptr addrspace(5) %slot syncscope("") unordered, align 1
+ ret i8 %retval
+}
+
+define i8 @unordered_i8_local_cta() {
+; CHECK-LABEL: unordered_i8_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot289[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot289;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b8 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i8, ptr addrspace(5) %slot syncscope("block") unordered, align 1
+ ret i8 %retval
+}
+
+define i8 @unordered_i8_local_cluster() {
+; CHECK-LABEL: unordered_i8_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot290[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot290;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b8 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i8, ptr addrspace(5) %slot syncscope("cluster") unordered, align 1
+ ret i8 %retval
+}
+
+define i8 @unordered_i8_local_gpu() {
+; CHECK-LABEL: unordered_i8_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot291[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot291;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b8 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i8, ptr addrspace(5) %slot syncscope("device") unordered, align 1
+ ret i8 %retval
+}
+
+define i8 @unordered_i8_local_sys_volatile() {
+; CHECK-LABEL: unordered_i8_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot292[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot292;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b8 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i8, ptr addrspace(5) %slot syncscope("") unordered, align 1
+ ret i8 %retval
+}
+
+define i8 @unordered_i8_local_cta_volatile() {
+; CHECK-LABEL: unordered_i8_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot293[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot293;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b8 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i8, ptr addrspace(5) %slot syncscope("block") unordered, align 1
+ ret i8 %retval
+}
+
+define i8 @unordered_i8_local_cluster_volatile() {
+; CHECK-LABEL: unordered_i8_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot294[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot294;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b8 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i8, ptr addrspace(5) %slot syncscope("cluster") unordered, align 1
+ ret i8 %retval
+}
+
+define i8 @unordered_i8_local_gpu_volatile() {
+; CHECK-LABEL: unordered_i8_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot295[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot295;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b8 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i8, ptr addrspace(5) %slot syncscope("device") unordered, align 1
+ ret i8 %retval
+}
+
+define i16 @unordered_i16_local_sys() {
+; CHECK-LABEL: unordered_i16_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot296[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot296;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i16, ptr addrspace(5) %slot syncscope("") unordered, align 2
+ ret i16 %retval
+}
+
+define i16 @unordered_i16_local_cta() {
+; CHECK-LABEL: unordered_i16_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot297[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot297;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i16, ptr addrspace(5) %slot syncscope("block") unordered, align 2
+ ret i16 %retval
+}
+
+define i16 @unordered_i16_local_cluster() {
+; CHECK-LABEL: unordered_i16_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot298[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot298;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i16, ptr addrspace(5) %slot syncscope("cluster") unordered, align 2
+ ret i16 %retval
+}
+
+define i16 @unordered_i16_local_gpu() {
+; CHECK-LABEL: unordered_i16_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot299[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot299;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i16, ptr addrspace(5) %slot syncscope("device") unordered, align 2
+ ret i16 %retval
+}
+
+define i16 @unordered_i16_local_sys_volatile() {
+; CHECK-LABEL: unordered_i16_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot300[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot300;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i16, ptr addrspace(5) %slot syncscope("") unordered, align 2
+ ret i16 %retval
+}
+
+define i16 @unordered_i16_local_cta_volatile() {
+; CHECK-LABEL: unordered_i16_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot301[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot301;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i16, ptr addrspace(5) %slot syncscope("block") unordered, align 2
+ ret i16 %retval
+}
+
+define i16 @unordered_i16_local_cluster_volatile() {
+; CHECK-LABEL: unordered_i16_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot302[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot302;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i16, ptr addrspace(5) %slot syncscope("cluster") unordered, align 2
+ ret i16 %retval
+}
+
+define i16 @unordered_i16_local_gpu_volatile() {
+; CHECK-LABEL: unordered_i16_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot303[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot303;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i16, ptr addrspace(5) %slot syncscope("device") unordered, align 2
+ ret i16 %retval
+}
+
+define i32 @unordered_i32_local_sys() {
+; CHECK-LABEL: unordered_i32_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot304[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot304;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i32, ptr addrspace(5) %slot syncscope("") unordered, align 4
+ ret i32 %retval
+}
+
+define i32 @unordered_i32_local_cta() {
+; CHECK-LABEL: unordered_i32_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot305[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot305;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i32, ptr addrspace(5) %slot syncscope("block") unordered, align 4
+ ret i32 %retval
+}
+
+define i32 @unordered_i32_local_cluster() {
+; CHECK-LABEL: unordered_i32_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot306[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot306;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i32, ptr addrspace(5) %slot syncscope("cluster") unordered, align 4
+ ret i32 %retval
+}
+
+define i32 @unordered_i32_local_gpu() {
+; CHECK-LABEL: unordered_i32_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot307[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot307;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i32, ptr addrspace(5) %slot syncscope("device") unordered, align 4
+ ret i32 %retval
+}
+
+define i32 @unordered_i32_local_sys_volatile() {
+; CHECK-LABEL: unordered_i32_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot308[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot308;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i32, ptr addrspace(5) %slot syncscope("") unordered, align 4
+ ret i32 %retval
+}
+
+define i32 @unordered_i32_local_cta_volatile() {
+; CHECK-LABEL: unordered_i32_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot309[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot309;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i32, ptr addrspace(5) %slot syncscope("block") unordered, align 4
+ ret i32 %retval
+}
+
+define i32 @unordered_i32_local_cluster_volatile() {
+; CHECK-LABEL: unordered_i32_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot310[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot310;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i32, ptr addrspace(5) %slot syncscope("cluster") unordered, align 4
+ ret i32 %retval
+}
+
+define i32 @unordered_i32_local_gpu_volatile() {
+; CHECK-LABEL: unordered_i32_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot311[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot311;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i32, ptr addrspace(5) %slot syncscope("device") unordered, align 4
+ ret i32 %retval
+}
+
+define i64 @unordered_i64_local_sys() {
+; CHECK-LABEL: unordered_i64_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot312[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot312;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i64, ptr addrspace(5) %slot syncscope("") unordered, align 8
+ ret i64 %retval
+}
+
+define i64 @unordered_i64_local_cta() {
+; CHECK-LABEL: unordered_i64_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot313[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot313;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i64, ptr addrspace(5) %slot syncscope("block") unordered, align 8
+ ret i64 %retval
+}
+
+define i64 @unordered_i64_local_cluster() {
+; CHECK-LABEL: unordered_i64_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot314[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot314;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i64, ptr addrspace(5) %slot syncscope("cluster") unordered, align 8
+ ret i64 %retval
+}
+
+define i64 @unordered_i64_local_gpu() {
+; CHECK-LABEL: unordered_i64_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot315[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot315;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i64, ptr addrspace(5) %slot syncscope("device") unordered, align 8
+ ret i64 %retval
+}
+
+define i64 @unordered_i64_local_sys_volatile() {
+; CHECK-LABEL: unordered_i64_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot316[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot316;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i64, ptr addrspace(5) %slot syncscope("") unordered, align 8
+ ret i64 %retval
+}
+
+define i64 @unordered_i64_local_cta_volatile() {
+; CHECK-LABEL: unordered_i64_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot317[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot317;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i64, ptr addrspace(5) %slot syncscope("block") unordered, align 8
+ ret i64 %retval
+}
+
+define i64 @unordered_i64_local_cluster_volatile() {
+; CHECK-LABEL: unordered_i64_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot318[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot318;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i64, ptr addrspace(5) %slot syncscope("cluster") unordered, align 8
+ ret i64 %retval
+}
+
+define i64 @unordered_i64_local_gpu_volatile() {
+; CHECK-LABEL: unordered_i64_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot319[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot319;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i64, ptr addrspace(5) %slot syncscope("device") unordered, align 8
+ ret i64 %retval
+}
+
+define half @unordered_half_local_sys() {
+; CHECK-LABEL: unordered_half_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot320[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot320;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic half, ptr addrspace(5) %slot syncscope("") unordered, align 2
+ ret half %retval
+}
+
+define half @unordered_half_local_cta() {
+; CHECK-LABEL: unordered_half_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot321[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot321;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic half, ptr addrspace(5) %slot syncscope("block") unordered, align 2
+ ret half %retval
+}
+
+define half @unordered_half_local_cluster() {
+; CHECK-LABEL: unordered_half_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot322[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot322;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic half, ptr addrspace(5) %slot syncscope("cluster") unordered, align 2
+ ret half %retval
+}
+
+define half @unordered_half_local_gpu() {
+; CHECK-LABEL: unordered_half_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot323[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot323;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic half, ptr addrspace(5) %slot syncscope("device") unordered, align 2
+ ret half %retval
+}
+
+define half @unordered_half_local_sys_volatile() {
+; CHECK-LABEL: unordered_half_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot324[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot324;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile half, ptr addrspace(5) %slot syncscope("") unordered, align 2
+ ret half %retval
+}
+
+define half @unordered_half_local_cta_volatile() {
+; CHECK-LABEL: unordered_half_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot325[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot325;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile half, ptr addrspace(5) %slot syncscope("block") unordered, align 2
+ ret half %retval
+}
+
+define half @unordered_half_local_cluster_volatile() {
+; CHECK-LABEL: unordered_half_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot326[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot326;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile half, ptr addrspace(5) %slot syncscope("cluster") unordered, align 2
+ ret half %retval
+}
+
+define half @unordered_half_local_gpu_volatile() {
+; CHECK-LABEL: unordered_half_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot327[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot327;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile half, ptr addrspace(5) %slot syncscope("device") unordered, align 2
+ ret half %retval
+}
+
+define bfloat @unordered_bfloat_local_sys() {
+; CHECK-LABEL: unordered_bfloat_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot328[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot328;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic bfloat, ptr addrspace(5) %slot syncscope("") unordered, align 2
+ ret bfloat %retval
+}
+
+define bfloat @unordered_bfloat_local_cta() {
+; CHECK-LABEL: unordered_bfloat_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot329[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot329;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic bfloat, ptr addrspace(5) %slot syncscope("block") unordered, align 2
+ ret bfloat %retval
+}
+
+define bfloat @unordered_bfloat_local_cluster() {
+; CHECK-LABEL: unordered_bfloat_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot330[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot330;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic bfloat, ptr addrspace(5) %slot syncscope("cluster") unordered, align 2
+ ret bfloat %retval
+}
+
+define bfloat @unordered_bfloat_local_gpu() {
+; CHECK-LABEL: unordered_bfloat_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot331[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot331;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic bfloat, ptr addrspace(5) %slot syncscope("device") unordered, align 2
+ ret bfloat %retval
+}
+
+define bfloat @unordered_bfloat_local_sys_volatile() {
+; CHECK-LABEL: unordered_bfloat_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot332[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot332;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile bfloat, ptr addrspace(5) %slot syncscope("") unordered, align 2
+ ret bfloat %retval
+}
+
+define bfloat @unordered_bfloat_local_cta_volatile() {
+; CHECK-LABEL: unordered_bfloat_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot333[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot333;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile bfloat, ptr addrspace(5) %slot syncscope("block") unordered, align 2
+ ret bfloat %retval
+}
+
+define bfloat @unordered_bfloat_local_cluster_volatile() {
+; CHECK-LABEL: unordered_bfloat_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot334[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot334;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile bfloat, ptr addrspace(5) %slot syncscope("cluster") unordered, align 2
+ ret bfloat %retval
+}
+
+define bfloat @unordered_bfloat_local_gpu_volatile() {
+; CHECK-LABEL: unordered_bfloat_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot335[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot335;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile bfloat, ptr addrspace(5) %slot syncscope("device") unordered, align 2
+ ret bfloat %retval
+}
+
+define float @unordered_float_local_sys() {
+; CHECK-LABEL: unordered_float_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot336[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot336;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic float, ptr addrspace(5) %slot syncscope("") unordered, align 4
+ ret float %retval
+}
+
+define float @unordered_float_local_cta() {
+; CHECK-LABEL: unordered_float_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot337[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot337;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic float, ptr addrspace(5) %slot syncscope("block") unordered, align 4
+ ret float %retval
+}
+
+define float @unordered_float_local_cluster() {
+; CHECK-LABEL: unordered_float_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot338[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot338;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic float, ptr addrspace(5) %slot syncscope("cluster") unordered, align 4
+ ret float %retval
+}
+
+define float @unordered_float_local_gpu() {
+; CHECK-LABEL: unordered_float_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot339[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot339;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic float, ptr addrspace(5) %slot syncscope("device") unordered, align 4
+ ret float %retval
+}
+
+define float @unordered_float_local_sys_volatile() {
+; CHECK-LABEL: unordered_float_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot340[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot340;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile float, ptr addrspace(5) %slot syncscope("") unordered, align 4
+ ret float %retval
+}
+
+define float @unordered_float_local_cta_volatile() {
+; CHECK-LABEL: unordered_float_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot341[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot341;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile float, ptr addrspace(5) %slot syncscope("block") unordered, align 4
+ ret float %retval
+}
+
+define float @unordered_float_local_cluster_volatile() {
+; CHECK-LABEL: unordered_float_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot342[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot342;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile float, ptr addrspace(5) %slot syncscope("cluster") unordered, align 4
+ ret float %retval
+}
+
+define float @unordered_float_local_gpu_volatile() {
+; CHECK-LABEL: unordered_float_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot343[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot343;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile float, ptr addrspace(5) %slot syncscope("device") unordered, align 4
+ ret float %retval
+}
+
+define double @unordered_double_local_sys() {
+; CHECK-LABEL: unordered_double_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot344[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot344;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic double, ptr addrspace(5) %slot syncscope("") unordered, align 8
+ ret double %retval
+}
+
+define double @unordered_double_local_cta() {
+; CHECK-LABEL: unordered_double_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot345[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot345;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic double, ptr addrspace(5) %slot syncscope("block") unordered, align 8
+ ret double %retval
+}
+
+define double @unordered_double_local_cluster() {
+; CHECK-LABEL: unordered_double_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot346[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot346;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic double, ptr addrspace(5) %slot syncscope("cluster") unordered, align 8
+ ret double %retval
+}
+
+define double @unordered_double_local_gpu() {
+; CHECK-LABEL: unordered_double_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot347[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot347;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic double, ptr addrspace(5) %slot syncscope("device") unordered, align 8
+ ret double %retval
+}
+
+define double @unordered_double_local_sys_volatile() {
+; CHECK-LABEL: unordered_double_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot348[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot348;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile double, ptr addrspace(5) %slot syncscope("") unordered, align 8
+ ret double %retval
+}
+
+define double @unordered_double_local_cta_volatile() {
+; CHECK-LABEL: unordered_double_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot349[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot349;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile double, ptr addrspace(5) %slot syncscope("block") unordered, align 8
+ ret double %retval
+}
+
+define double @unordered_double_local_cluster_volatile() {
+; CHECK-LABEL: unordered_double_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot350[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot350;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile double, ptr addrspace(5) %slot syncscope("cluster") unordered, align 8
+ ret double %retval
+}
+
+define double @unordered_double_local_gpu_volatile() {
+; CHECK-LABEL: unordered_double_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot351[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot351;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile double, ptr addrspace(5) %slot syncscope("device") unordered, align 8
+ ret double %retval
+}
+
+define i8 @monotonic_i8_generic_sys(ptr %addr) {
+; CHECK-LABEL: monotonic_i8_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_generic_sys_param_0];
+; CHECK-NEXT: ld.volatile.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr %addr syncscope("") monotonic, align 1
+ ret i8 %retval
+}
+
+define i8 @monotonic_i8_generic_cta(ptr %addr) {
+; CHECK-LABEL: monotonic_i8_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_generic_cta_param_0];
+; CHECK-NEXT: ld.volatile.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr %addr syncscope("block") monotonic, align 1
+ ret i8 %retval
+}
+
+define i8 @monotonic_i8_generic_cluster(ptr %addr) {
+; CHECK-LABEL: monotonic_i8_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_generic_cluster_param_0];
+; CHECK-NEXT: ld.volatile.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr %addr syncscope("cluster") monotonic, align 1
+ ret i8 %retval
+}
+
+define i8 @monotonic_i8_generic_gpu(ptr %addr) {
+; CHECK-LABEL: monotonic_i8_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_generic_gpu_param_0];
+; CHECK-NEXT: ld.volatile.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr %addr syncscope("device") monotonic, align 1
+ ret i8 %retval
+}
+
+define i8 @monotonic_i8_generic_sys_volatile(ptr %addr) {
+; CHECK-LABEL: monotonic_i8_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr %addr syncscope("") monotonic, align 1
+ ret i8 %retval
+}
+
+define i8 @monotonic_i8_generic_cta_volatile(ptr %addr) {
+; CHECK-LABEL: monotonic_i8_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr %addr syncscope("block") monotonic, align 1
+ ret i8 %retval
+}
+
+define i8 @monotonic_i8_generic_cluster_volatile(ptr %addr) {
+; CHECK-LABEL: monotonic_i8_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr %addr syncscope("cluster") monotonic, align 1
+ ret i8 %retval
+}
+
+define i8 @monotonic_i8_generic_gpu_volatile(ptr %addr) {
+; CHECK-LABEL: monotonic_i8_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr %addr syncscope("device") monotonic, align 1
+ ret i8 %retval
+}
+
+define i16 @monotonic_i16_generic_sys(ptr %addr) {
+; CHECK-LABEL: monotonic_i16_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_generic_sys_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr %addr syncscope("") monotonic, align 2
+ ret i16 %retval
+}
+
+define i16 @monotonic_i16_generic_cta(ptr %addr) {
+; CHECK-LABEL: monotonic_i16_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_generic_cta_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr %addr syncscope("block") monotonic, align 2
+ ret i16 %retval
+}
+
+define i16 @monotonic_i16_generic_cluster(ptr %addr) {
+; CHECK-LABEL: monotonic_i16_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_generic_cluster_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr %addr syncscope("cluster") monotonic, align 2
+ ret i16 %retval
+}
+
+define i16 @monotonic_i16_generic_gpu(ptr %addr) {
+; CHECK-LABEL: monotonic_i16_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_generic_gpu_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr %addr syncscope("device") monotonic, align 2
+ ret i16 %retval
+}
+
+define i16 @monotonic_i16_generic_sys_volatile(ptr %addr) {
+; CHECK-LABEL: monotonic_i16_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr %addr syncscope("") monotonic, align 2
+ ret i16 %retval
+}
+
+define i16 @monotonic_i16_generic_cta_volatile(ptr %addr) {
+; CHECK-LABEL: monotonic_i16_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr %addr syncscope("block") monotonic, align 2
+ ret i16 %retval
+}
+
+define i16 @monotonic_i16_generic_cluster_volatile(ptr %addr) {
+; CHECK-LABEL: monotonic_i16_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr %addr syncscope("cluster") monotonic, align 2
+ ret i16 %retval
+}
+
+define i16 @monotonic_i16_generic_gpu_volatile(ptr %addr) {
+; CHECK-LABEL: monotonic_i16_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr %addr syncscope("device") monotonic, align 2
+ ret i16 %retval
+}
+
+define i32 @monotonic_i32_generic_sys(ptr %addr) {
+; CHECK-LABEL: monotonic_i32_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_generic_sys_param_0];
+; CHECK-NEXT: ld.volatile.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr %addr syncscope("") monotonic, align 4
+ ret i32 %retval
+}
+
+define i32 @monotonic_i32_generic_cta(ptr %addr) {
+; CHECK-LABEL: monotonic_i32_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_generic_cta_param_0];
+; CHECK-NEXT: ld.volatile.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr %addr syncscope("block") monotonic, align 4
+ ret i32 %retval
+}
+
+define i32 @monotonic_i32_generic_cluster(ptr %addr) {
+; CHECK-LABEL: monotonic_i32_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_generic_cluster_param_0];
+; CHECK-NEXT: ld.volatile.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr %addr syncscope("cluster") monotonic, align 4
+ ret i32 %retval
+}
+
+define i32 @monotonic_i32_generic_gpu(ptr %addr) {
+; CHECK-LABEL: monotonic_i32_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_generic_gpu_param_0];
+; CHECK-NEXT: ld.volatile.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr %addr syncscope("device") monotonic, align 4
+ ret i32 %retval
+}
+
+define i32 @monotonic_i32_generic_sys_volatile(ptr %addr) {
+; CHECK-LABEL: monotonic_i32_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr %addr syncscope("") monotonic, align 4
+ ret i32 %retval
+}
+
+define i32 @monotonic_i32_generic_cta_volatile(ptr %addr) {
+; CHECK-LABEL: monotonic_i32_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr %addr syncscope("block") monotonic, align 4
+ ret i32 %retval
+}
+
+define i32 @monotonic_i32_generic_cluster_volatile(ptr %addr) {
+; CHECK-LABEL: monotonic_i32_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr %addr syncscope("cluster") monotonic, align 4
+ ret i32 %retval
+}
+
+define i32 @monotonic_i32_generic_gpu_volatile(ptr %addr) {
+; CHECK-LABEL: monotonic_i32_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr %addr syncscope("device") monotonic, align 4
+ ret i32 %retval
+}
+
+define i64 @monotonic_i64_generic_sys(ptr %addr) {
+; CHECK-LABEL: monotonic_i64_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_generic_sys_param_0];
+; CHECK-NEXT: ld.volatile.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr %addr syncscope("") monotonic, align 8
+ ret i64 %retval
+}
+
+define i64 @monotonic_i64_generic_cta(ptr %addr) {
+; CHECK-LABEL: monotonic_i64_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_generic_cta_param_0];
+; CHECK-NEXT: ld.volatile.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr %addr syncscope("block") monotonic, align 8
+ ret i64 %retval
+}
+
+define i64 @monotonic_i64_generic_cluster(ptr %addr) {
+; CHECK-LABEL: monotonic_i64_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_generic_cluster_param_0];
+; CHECK-NEXT: ld.volatile.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr %addr syncscope("cluster") monotonic, align 8
+ ret i64 %retval
+}
+
+define i64 @monotonic_i64_generic_gpu(ptr %addr) {
+; CHECK-LABEL: monotonic_i64_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_generic_gpu_param_0];
+; CHECK-NEXT: ld.volatile.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr %addr syncscope("device") monotonic, align 8
+ ret i64 %retval
+}
+
+define i64 @monotonic_i64_generic_sys_volatile(ptr %addr) {
+; CHECK-LABEL: monotonic_i64_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr %addr syncscope("") monotonic, align 8
+ ret i64 %retval
+}
+
+define i64 @monotonic_i64_generic_cta_volatile(ptr %addr) {
+; CHECK-LABEL: monotonic_i64_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr %addr syncscope("block") monotonic, align 8
+ ret i64 %retval
+}
+
+define i64 @monotonic_i64_generic_cluster_volatile(ptr %addr) {
+; CHECK-LABEL: monotonic_i64_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr %addr syncscope("cluster") monotonic, align 8
+ ret i64 %retval
+}
+
+define i64 @monotonic_i64_generic_gpu_volatile(ptr %addr) {
+; CHECK-LABEL: monotonic_i64_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr %addr syncscope("device") monotonic, align 8
+ ret i64 %retval
+}
+
+define half @monotonic_half_generic_sys(ptr %addr) {
+; CHECK-LABEL: monotonic_half_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_generic_sys_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr %addr syncscope("") monotonic, align 2
+ ret half %retval
+}
+
+define half @monotonic_half_generic_cta(ptr %addr) {
+; CHECK-LABEL: monotonic_half_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_generic_cta_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr %addr syncscope("block") monotonic, align 2
+ ret half %retval
+}
+
+define half @monotonic_half_generic_cluster(ptr %addr) {
+; CHECK-LABEL: monotonic_half_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_generic_cluster_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr %addr syncscope("cluster") monotonic, align 2
+ ret half %retval
+}
+
+define half @monotonic_half_generic_gpu(ptr %addr) {
+; CHECK-LABEL: monotonic_half_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_generic_gpu_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr %addr syncscope("device") monotonic, align 2
+ ret half %retval
+}
+
+define half @monotonic_half_generic_sys_volatile(ptr %addr) {
+; CHECK-LABEL: monotonic_half_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr %addr syncscope("") monotonic, align 2
+ ret half %retval
+}
+
+define half @monotonic_half_generic_cta_volatile(ptr %addr) {
+; CHECK-LABEL: monotonic_half_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr %addr syncscope("block") monotonic, align 2
+ ret half %retval
+}
+
+define half @monotonic_half_generic_cluster_volatile(ptr %addr) {
+; CHECK-LABEL: monotonic_half_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr %addr syncscope("cluster") monotonic, align 2
+ ret half %retval
+}
+
+define half @monotonic_half_generic_gpu_volatile(ptr %addr) {
+; CHECK-LABEL: monotonic_half_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr %addr syncscope("device") monotonic, align 2
+ ret half %retval
+}
+
+define bfloat @monotonic_bfloat_generic_sys(ptr %addr) {
+; CHECK-LABEL: monotonic_bfloat_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_generic_sys_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr %addr syncscope("") monotonic, align 2
+ ret bfloat %retval
+}
+
+define bfloat @monotonic_bfloat_generic_cta(ptr %addr) {
+; CHECK-LABEL: monotonic_bfloat_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_generic_cta_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr %addr syncscope("block") monotonic, align 2
+ ret bfloat %retval
+}
+
+define bfloat @monotonic_bfloat_generic_cluster(ptr %addr) {
+; CHECK-LABEL: monotonic_bfloat_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_generic_cluster_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr %addr syncscope("cluster") monotonic, align 2
+ ret bfloat %retval
+}
+
+define bfloat @monotonic_bfloat_generic_gpu(ptr %addr) {
+; CHECK-LABEL: monotonic_bfloat_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_generic_gpu_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr %addr syncscope("device") monotonic, align 2
+ ret bfloat %retval
+}
+
+define bfloat @monotonic_bfloat_generic_sys_volatile(ptr %addr) {
+; CHECK-LABEL: monotonic_bfloat_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr %addr syncscope("") monotonic, align 2
+ ret bfloat %retval
+}
+
+define bfloat @monotonic_bfloat_generic_cta_volatile(ptr %addr) {
+; CHECK-LABEL: monotonic_bfloat_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr %addr syncscope("block") monotonic, align 2
+ ret bfloat %retval
+}
+
+define bfloat @monotonic_bfloat_generic_cluster_volatile(ptr %addr) {
+; CHECK-LABEL: monotonic_bfloat_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr %addr syncscope("cluster") monotonic, align 2
+ ret bfloat %retval
+}
+
+define bfloat @monotonic_bfloat_generic_gpu_volatile(ptr %addr) {
+; CHECK-LABEL: monotonic_bfloat_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr %addr syncscope("device") monotonic, align 2
+ ret bfloat %retval
+}
+
+define float @monotonic_float_generic_sys(ptr %addr) {
+; CHECK-LABEL: monotonic_float_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_generic_sys_param_0];
+; CHECK-NEXT: ld.volatile.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr %addr syncscope("") monotonic, align 4
+ ret float %retval
+}
+
+define float @monotonic_float_generic_cta(ptr %addr) {
+; CHECK-LABEL: monotonic_float_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_generic_cta_param_0];
+; CHECK-NEXT: ld.volatile.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr %addr syncscope("block") monotonic, align 4
+ ret float %retval
+}
+
+define float @monotonic_float_generic_cluster(ptr %addr) {
+; CHECK-LABEL: monotonic_float_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_generic_cluster_param_0];
+; CHECK-NEXT: ld.volatile.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr %addr syncscope("cluster") monotonic, align 4
+ ret float %retval
+}
+
+define float @monotonic_float_generic_gpu(ptr %addr) {
+; CHECK-LABEL: monotonic_float_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_generic_gpu_param_0];
+; CHECK-NEXT: ld.volatile.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr %addr syncscope("device") monotonic, align 4
+ ret float %retval
+}
+
+define float @monotonic_float_generic_sys_volatile(ptr %addr) {
+; CHECK-LABEL: monotonic_float_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr %addr syncscope("") monotonic, align 4
+ ret float %retval
+}
+
+define float @monotonic_float_generic_cta_volatile(ptr %addr) {
+; CHECK-LABEL: monotonic_float_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr %addr syncscope("block") monotonic, align 4
+ ret float %retval
+}
+
+define float @monotonic_float_generic_cluster_volatile(ptr %addr) {
+; CHECK-LABEL: monotonic_float_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr %addr syncscope("cluster") monotonic, align 4
+ ret float %retval
+}
+
+define float @monotonic_float_generic_gpu_volatile(ptr %addr) {
+; CHECK-LABEL: monotonic_float_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr %addr syncscope("device") monotonic, align 4
+ ret float %retval
+}
+
+define double @monotonic_double_generic_sys(ptr %addr) {
+; CHECK-LABEL: monotonic_double_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_generic_sys_param_0];
+; CHECK-NEXT: ld.volatile.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr %addr syncscope("") monotonic, align 8
+ ret double %retval
+}
+
+define double @monotonic_double_generic_cta(ptr %addr) {
+; CHECK-LABEL: monotonic_double_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_generic_cta_param_0];
+; CHECK-NEXT: ld.volatile.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr %addr syncscope("block") monotonic, align 8
+ ret double %retval
+}
+
+define double @monotonic_double_generic_cluster(ptr %addr) {
+; CHECK-LABEL: monotonic_double_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_generic_cluster_param_0];
+; CHECK-NEXT: ld.volatile.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr %addr syncscope("cluster") monotonic, align 8
+ ret double %retval
+}
+
+define double @monotonic_double_generic_gpu(ptr %addr) {
+; CHECK-LABEL: monotonic_double_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_generic_gpu_param_0];
+; CHECK-NEXT: ld.volatile.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr %addr syncscope("device") monotonic, align 8
+ ret double %retval
+}
+
+define double @monotonic_double_generic_sys_volatile(ptr %addr) {
+; CHECK-LABEL: monotonic_double_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr %addr syncscope("") monotonic, align 8
+ ret double %retval
+}
+
+define double @monotonic_double_generic_cta_volatile(ptr %addr) {
+; CHECK-LABEL: monotonic_double_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr %addr syncscope("block") monotonic, align 8
+ ret double %retval
+}
+
+define double @monotonic_double_generic_cluster_volatile(ptr %addr) {
+; CHECK-LABEL: monotonic_double_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr %addr syncscope("cluster") monotonic, align 8
+ ret double %retval
+}
+
+define double @monotonic_double_generic_gpu_volatile(ptr %addr) {
+; CHECK-LABEL: monotonic_double_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr %addr syncscope("device") monotonic, align 8
+ ret double %retval
+}
+
+define i8 @monotonic_i8_global_sys(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i8_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_global_sys_param_0];
+; CHECK-NEXT: ld.volatile.global.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr addrspace(1) %addr syncscope("") monotonic, align 1
+ ret i8 %retval
+}
+
+define i8 @monotonic_i8_global_cta(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i8_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_global_cta_param_0];
+; CHECK-NEXT: ld.volatile.global.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr addrspace(1) %addr syncscope("block") monotonic, align 1
+ ret i8 %retval
+}
+
+define i8 @monotonic_i8_global_cluster(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i8_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_global_cluster_param_0];
+; CHECK-NEXT: ld.volatile.global.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr addrspace(1) %addr syncscope("cluster") monotonic, align 1
+ ret i8 %retval
+}
+
+define i8 @monotonic_i8_global_gpu(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i8_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_global_gpu_param_0];
+; CHECK-NEXT: ld.volatile.global.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr addrspace(1) %addr syncscope("device") monotonic, align 1
+ ret i8 %retval
+}
+
+define i8 @monotonic_i8_global_sys_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i8_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr addrspace(1) %addr syncscope("") monotonic, align 1
+ ret i8 %retval
+}
+
+define i8 @monotonic_i8_global_cta_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i8_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr addrspace(1) %addr syncscope("block") monotonic, align 1
+ ret i8 %retval
+}
+
+define i8 @monotonic_i8_global_cluster_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i8_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr addrspace(1) %addr syncscope("cluster") monotonic, align 1
+ ret i8 %retval
+}
+
+define i8 @monotonic_i8_global_gpu_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i8_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr addrspace(1) %addr syncscope("device") monotonic, align 1
+ ret i8 %retval
+}
+
+define i16 @monotonic_i16_global_sys(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i16_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_global_sys_param_0];
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr addrspace(1) %addr syncscope("") monotonic, align 2
+ ret i16 %retval
+}
+
+define i16 @monotonic_i16_global_cta(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i16_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_global_cta_param_0];
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr addrspace(1) %addr syncscope("block") monotonic, align 2
+ ret i16 %retval
+}
+
+define i16 @monotonic_i16_global_cluster(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i16_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_global_cluster_param_0];
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr addrspace(1) %addr syncscope("cluster") monotonic, align 2
+ ret i16 %retval
+}
+
+define i16 @monotonic_i16_global_gpu(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i16_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_global_gpu_param_0];
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr addrspace(1) %addr syncscope("device") monotonic, align 2
+ ret i16 %retval
+}
+
+define i16 @monotonic_i16_global_sys_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i16_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr addrspace(1) %addr syncscope("") monotonic, align 2
+ ret i16 %retval
+}
+
+define i16 @monotonic_i16_global_cta_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i16_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr addrspace(1) %addr syncscope("block") monotonic, align 2
+ ret i16 %retval
+}
+
+define i16 @monotonic_i16_global_cluster_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i16_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr addrspace(1) %addr syncscope("cluster") monotonic, align 2
+ ret i16 %retval
+}
+
+define i16 @monotonic_i16_global_gpu_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i16_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr addrspace(1) %addr syncscope("device") monotonic, align 2
+ ret i16 %retval
+}
+
+define i32 @monotonic_i32_global_sys(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i32_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_global_sys_param_0];
+; CHECK-NEXT: ld.volatile.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr addrspace(1) %addr syncscope("") monotonic, align 4
+ ret i32 %retval
+}
+
+define i32 @monotonic_i32_global_cta(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i32_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_global_cta_param_0];
+; CHECK-NEXT: ld.volatile.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr addrspace(1) %addr syncscope("block") monotonic, align 4
+ ret i32 %retval
+}
+
+define i32 @monotonic_i32_global_cluster(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i32_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_global_cluster_param_0];
+; CHECK-NEXT: ld.volatile.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr addrspace(1) %addr syncscope("cluster") monotonic, align 4
+ ret i32 %retval
+}
+
+define i32 @monotonic_i32_global_gpu(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i32_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_global_gpu_param_0];
+; CHECK-NEXT: ld.volatile.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr addrspace(1) %addr syncscope("device") monotonic, align 4
+ ret i32 %retval
+}
+
+define i32 @monotonic_i32_global_sys_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i32_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr addrspace(1) %addr syncscope("") monotonic, align 4
+ ret i32 %retval
+}
+
+define i32 @monotonic_i32_global_cta_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i32_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr addrspace(1) %addr syncscope("block") monotonic, align 4
+ ret i32 %retval
+}
+
+define i32 @monotonic_i32_global_cluster_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i32_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr addrspace(1) %addr syncscope("cluster") monotonic, align 4
+ ret i32 %retval
+}
+
+define i32 @monotonic_i32_global_gpu_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i32_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr addrspace(1) %addr syncscope("device") monotonic, align 4
+ ret i32 %retval
+}
+
+define i64 @monotonic_i64_global_sys(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i64_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_global_sys_param_0];
+; CHECK-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr addrspace(1) %addr syncscope("") monotonic, align 8
+ ret i64 %retval
+}
+
+define i64 @monotonic_i64_global_cta(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i64_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_global_cta_param_0];
+; CHECK-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr addrspace(1) %addr syncscope("block") monotonic, align 8
+ ret i64 %retval
+}
+
+define i64 @monotonic_i64_global_cluster(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i64_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_global_cluster_param_0];
+; CHECK-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr addrspace(1) %addr syncscope("cluster") monotonic, align 8
+ ret i64 %retval
+}
+
+define i64 @monotonic_i64_global_gpu(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i64_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_global_gpu_param_0];
+; CHECK-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr addrspace(1) %addr syncscope("device") monotonic, align 8
+ ret i64 %retval
+}
+
+define i64 @monotonic_i64_global_sys_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i64_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr addrspace(1) %addr syncscope("") monotonic, align 8
+ ret i64 %retval
+}
+
+define i64 @monotonic_i64_global_cta_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i64_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr addrspace(1) %addr syncscope("block") monotonic, align 8
+ ret i64 %retval
+}
+
+define i64 @monotonic_i64_global_cluster_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i64_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr addrspace(1) %addr syncscope("cluster") monotonic, align 8
+ ret i64 %retval
+}
+
+define i64 @monotonic_i64_global_gpu_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i64_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr addrspace(1) %addr syncscope("device") monotonic, align 8
+ ret i64 %retval
+}
+
+define half @monotonic_half_global_sys(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_half_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_global_sys_param_0];
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr addrspace(1) %addr syncscope("") monotonic, align 2
+ ret half %retval
+}
+
+define half @monotonic_half_global_cta(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_half_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_global_cta_param_0];
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr addrspace(1) %addr syncscope("block") monotonic, align 2
+ ret half %retval
+}
+
+define half @monotonic_half_global_cluster(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_half_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_global_cluster_param_0];
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr addrspace(1) %addr syncscope("cluster") monotonic, align 2
+ ret half %retval
+}
+
+define half @monotonic_half_global_gpu(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_half_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_global_gpu_param_0];
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr addrspace(1) %addr syncscope("device") monotonic, align 2
+ ret half %retval
+}
+
+define half @monotonic_half_global_sys_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_half_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr addrspace(1) %addr syncscope("") monotonic, align 2
+ ret half %retval
+}
+
+define half @monotonic_half_global_cta_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_half_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr addrspace(1) %addr syncscope("block") monotonic, align 2
+ ret half %retval
+}
+
+define half @monotonic_half_global_cluster_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_half_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr addrspace(1) %addr syncscope("cluster") monotonic, align 2
+ ret half %retval
+}
+
+define half @monotonic_half_global_gpu_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_half_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr addrspace(1) %addr syncscope("device") monotonic, align 2
+ ret half %retval
+}
+
+define bfloat @monotonic_bfloat_global_sys(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_bfloat_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_global_sys_param_0];
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr addrspace(1) %addr syncscope("") monotonic, align 2
+ ret bfloat %retval
+}
+
+define bfloat @monotonic_bfloat_global_cta(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_bfloat_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_global_cta_param_0];
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr addrspace(1) %addr syncscope("block") monotonic, align 2
+ ret bfloat %retval
+}
+
+define bfloat @monotonic_bfloat_global_cluster(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_bfloat_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_global_cluster_param_0];
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr addrspace(1) %addr syncscope("cluster") monotonic, align 2
+ ret bfloat %retval
+}
+
+define bfloat @monotonic_bfloat_global_gpu(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_bfloat_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_global_gpu_param_0];
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr addrspace(1) %addr syncscope("device") monotonic, align 2
+ ret bfloat %retval
+}
+
+define bfloat @monotonic_bfloat_global_sys_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_bfloat_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr addrspace(1) %addr syncscope("") monotonic, align 2
+ ret bfloat %retval
+}
+
+define bfloat @monotonic_bfloat_global_cta_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_bfloat_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr addrspace(1) %addr syncscope("block") monotonic, align 2
+ ret bfloat %retval
+}
+
+define bfloat @monotonic_bfloat_global_cluster_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_bfloat_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr addrspace(1) %addr syncscope("cluster") monotonic, align 2
+ ret bfloat %retval
+}
+
+define bfloat @monotonic_bfloat_global_gpu_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_bfloat_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr addrspace(1) %addr syncscope("device") monotonic, align 2
+ ret bfloat %retval
+}
+
+define float @monotonic_float_global_sys(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_float_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_global_sys_param_0];
+; CHECK-NEXT: ld.volatile.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr addrspace(1) %addr syncscope("") monotonic, align 4
+ ret float %retval
+}
+
+define float @monotonic_float_global_cta(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_float_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_global_cta_param_0];
+; CHECK-NEXT: ld.volatile.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr addrspace(1) %addr syncscope("block") monotonic, align 4
+ ret float %retval
+}
+
+define float @monotonic_float_global_cluster(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_float_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_global_cluster_param_0];
+; CHECK-NEXT: ld.volatile.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr addrspace(1) %addr syncscope("cluster") monotonic, align 4
+ ret float %retval
+}
+
+define float @monotonic_float_global_gpu(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_float_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_global_gpu_param_0];
+; CHECK-NEXT: ld.volatile.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr addrspace(1) %addr syncscope("device") monotonic, align 4
+ ret float %retval
+}
+
+define float @monotonic_float_global_sys_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_float_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr addrspace(1) %addr syncscope("") monotonic, align 4
+ ret float %retval
+}
+
+define float @monotonic_float_global_cta_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_float_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr addrspace(1) %addr syncscope("block") monotonic, align 4
+ ret float %retval
+}
+
+define float @monotonic_float_global_cluster_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_float_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr addrspace(1) %addr syncscope("cluster") monotonic, align 4
+ ret float %retval
+}
+
+define float @monotonic_float_global_gpu_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_float_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr addrspace(1) %addr syncscope("device") monotonic, align 4
+ ret float %retval
+}
+
+define double @monotonic_double_global_sys(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_double_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_global_sys_param_0];
+; CHECK-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr addrspace(1) %addr syncscope("") monotonic, align 8
+ ret double %retval
+}
+
+define double @monotonic_double_global_cta(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_double_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_global_cta_param_0];
+; CHECK-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr addrspace(1) %addr syncscope("block") monotonic, align 8
+ ret double %retval
+}
+
+define double @monotonic_double_global_cluster(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_double_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_global_cluster_param_0];
+; CHECK-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr addrspace(1) %addr syncscope("cluster") monotonic, align 8
+ ret double %retval
+}
+
+define double @monotonic_double_global_gpu(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_double_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_global_gpu_param_0];
+; CHECK-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr addrspace(1) %addr syncscope("device") monotonic, align 8
+ ret double %retval
+}
+
+define double @monotonic_double_global_sys_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_double_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr addrspace(1) %addr syncscope("") monotonic, align 8
+ ret double %retval
+}
+
+define double @monotonic_double_global_cta_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_double_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr addrspace(1) %addr syncscope("block") monotonic, align 8
+ ret double %retval
+}
+
+define double @monotonic_double_global_cluster_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_double_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr addrspace(1) %addr syncscope("cluster") monotonic, align 8
+ ret double %retval
+}
+
+define double @monotonic_double_global_gpu_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_double_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr addrspace(1) %addr syncscope("device") monotonic, align 8
+ ret double %retval
+}
+
+define i8 @monotonic_i8_shared_sys(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i8_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_shared_sys_param_0];
+; CHECK-NEXT: ld.volatile.shared.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr addrspace(3) %addr syncscope("") monotonic, align 1
+ ret i8 %retval
+}
+
+define i8 @monotonic_i8_shared_cta(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i8_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_shared_cta_param_0];
+; CHECK-NEXT: ld.volatile.shared.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr addrspace(3) %addr syncscope("block") monotonic, align 1
+ ret i8 %retval
+}
+
+define i8 @monotonic_i8_shared_cluster(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i8_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_shared_cluster_param_0];
+; CHECK-NEXT: ld.volatile.shared.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr addrspace(3) %addr syncscope("cluster") monotonic, align 1
+ ret i8 %retval
+}
+
+define i8 @monotonic_i8_shared_gpu(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i8_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_shared_gpu_param_0];
+; CHECK-NEXT: ld.volatile.shared.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr addrspace(3) %addr syncscope("device") monotonic, align 1
+ ret i8 %retval
+}
+
+define i8 @monotonic_i8_shared_sys_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i8_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr addrspace(3) %addr syncscope("") monotonic, align 1
+ ret i8 %retval
+}
+
+define i8 @monotonic_i8_shared_cta_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i8_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr addrspace(3) %addr syncscope("block") monotonic, align 1
+ ret i8 %retval
+}
+
+define i8 @monotonic_i8_shared_cluster_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i8_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr addrspace(3) %addr syncscope("cluster") monotonic, align 1
+ ret i8 %retval
+}
+
+define i8 @monotonic_i8_shared_gpu_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i8_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr addrspace(3) %addr syncscope("device") monotonic, align 1
+ ret i8 %retval
+}
+
+define i16 @monotonic_i16_shared_sys(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i16_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_shared_sys_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr addrspace(3) %addr syncscope("") monotonic, align 2
+ ret i16 %retval
+}
+
+define i16 @monotonic_i16_shared_cta(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i16_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_shared_cta_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr addrspace(3) %addr syncscope("block") monotonic, align 2
+ ret i16 %retval
+}
+
+define i16 @monotonic_i16_shared_cluster(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i16_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_shared_cluster_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr addrspace(3) %addr syncscope("cluster") monotonic, align 2
+ ret i16 %retval
+}
+
+define i16 @monotonic_i16_shared_gpu(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i16_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_shared_gpu_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr addrspace(3) %addr syncscope("device") monotonic, align 2
+ ret i16 %retval
+}
+
+define i16 @monotonic_i16_shared_sys_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i16_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr addrspace(3) %addr syncscope("") monotonic, align 2
+ ret i16 %retval
+}
+
+define i16 @monotonic_i16_shared_cta_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i16_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr addrspace(3) %addr syncscope("block") monotonic, align 2
+ ret i16 %retval
+}
+
+define i16 @monotonic_i16_shared_cluster_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i16_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr addrspace(3) %addr syncscope("cluster") monotonic, align 2
+ ret i16 %retval
+}
+
+define i16 @monotonic_i16_shared_gpu_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i16_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr addrspace(3) %addr syncscope("device") monotonic, align 2
+ ret i16 %retval
+}
+
+define i32 @monotonic_i32_shared_sys(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i32_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_shared_sys_param_0];
+; CHECK-NEXT: ld.volatile.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr addrspace(3) %addr syncscope("") monotonic, align 4
+ ret i32 %retval
+}
+
+define i32 @monotonic_i32_shared_cta(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i32_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_shared_cta_param_0];
+; CHECK-NEXT: ld.volatile.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr addrspace(3) %addr syncscope("block") monotonic, align 4
+ ret i32 %retval
+}
+
+define i32 @monotonic_i32_shared_cluster(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i32_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_shared_cluster_param_0];
+; CHECK-NEXT: ld.volatile.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr addrspace(3) %addr syncscope("cluster") monotonic, align 4
+ ret i32 %retval
+}
+
+define i32 @monotonic_i32_shared_gpu(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i32_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_shared_gpu_param_0];
+; CHECK-NEXT: ld.volatile.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr addrspace(3) %addr syncscope("device") monotonic, align 4
+ ret i32 %retval
+}
+
+define i32 @monotonic_i32_shared_sys_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i32_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr addrspace(3) %addr syncscope("") monotonic, align 4
+ ret i32 %retval
+}
+
+define i32 @monotonic_i32_shared_cta_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i32_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr addrspace(3) %addr syncscope("block") monotonic, align 4
+ ret i32 %retval
+}
+
+define i32 @monotonic_i32_shared_cluster_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i32_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr addrspace(3) %addr syncscope("cluster") monotonic, align 4
+ ret i32 %retval
+}
+
+define i32 @monotonic_i32_shared_gpu_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i32_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr addrspace(3) %addr syncscope("device") monotonic, align 4
+ ret i32 %retval
+}
+
+define i64 @monotonic_i64_shared_sys(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i64_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_shared_sys_param_0];
+; CHECK-NEXT: ld.volatile.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr addrspace(3) %addr syncscope("") monotonic, align 8
+ ret i64 %retval
+}
+
+define i64 @monotonic_i64_shared_cta(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i64_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_shared_cta_param_0];
+; CHECK-NEXT: ld.volatile.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr addrspace(3) %addr syncscope("block") monotonic, align 8
+ ret i64 %retval
+}
+
+define i64 @monotonic_i64_shared_cluster(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i64_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_shared_cluster_param_0];
+; CHECK-NEXT: ld.volatile.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr addrspace(3) %addr syncscope("cluster") monotonic, align 8
+ ret i64 %retval
+}
+
+define i64 @monotonic_i64_shared_gpu(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i64_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_shared_gpu_param_0];
+; CHECK-NEXT: ld.volatile.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr addrspace(3) %addr syncscope("device") monotonic, align 8
+ ret i64 %retval
+}
+
+define i64 @monotonic_i64_shared_sys_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i64_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr addrspace(3) %addr syncscope("") monotonic, align 8
+ ret i64 %retval
+}
+
+define i64 @monotonic_i64_shared_cta_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i64_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr addrspace(3) %addr syncscope("block") monotonic, align 8
+ ret i64 %retval
+}
+
+define i64 @monotonic_i64_shared_cluster_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i64_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr addrspace(3) %addr syncscope("cluster") monotonic, align 8
+ ret i64 %retval
+}
+
+define i64 @monotonic_i64_shared_gpu_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i64_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr addrspace(3) %addr syncscope("device") monotonic, align 8
+ ret i64 %retval
+}
+
+define half @monotonic_half_shared_sys(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_half_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_shared_sys_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr addrspace(3) %addr syncscope("") monotonic, align 2
+ ret half %retval
+}
+
+define half @monotonic_half_shared_cta(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_half_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_shared_cta_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr addrspace(3) %addr syncscope("block") monotonic, align 2
+ ret half %retval
+}
+
+define half @monotonic_half_shared_cluster(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_half_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_shared_cluster_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr addrspace(3) %addr syncscope("cluster") monotonic, align 2
+ ret half %retval
+}
+
+define half @monotonic_half_shared_gpu(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_half_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_shared_gpu_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr addrspace(3) %addr syncscope("device") monotonic, align 2
+ ret half %retval
+}
+
+define half @monotonic_half_shared_sys_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_half_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr addrspace(3) %addr syncscope("") monotonic, align 2
+ ret half %retval
+}
+
+define half @monotonic_half_shared_cta_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_half_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr addrspace(3) %addr syncscope("block") monotonic, align 2
+ ret half %retval
+}
+
+define half @monotonic_half_shared_cluster_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_half_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr addrspace(3) %addr syncscope("cluster") monotonic, align 2
+ ret half %retval
+}
+
+define half @monotonic_half_shared_gpu_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_half_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr addrspace(3) %addr syncscope("device") monotonic, align 2
+ ret half %retval
+}
+
+define bfloat @monotonic_bfloat_shared_sys(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_bfloat_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_shared_sys_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr addrspace(3) %addr syncscope("") monotonic, align 2
+ ret bfloat %retval
+}
+
+define bfloat @monotonic_bfloat_shared_cta(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_bfloat_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_shared_cta_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr addrspace(3) %addr syncscope("block") monotonic, align 2
+ ret bfloat %retval
+}
+
+define bfloat @monotonic_bfloat_shared_cluster(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_bfloat_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_shared_cluster_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr addrspace(3) %addr syncscope("cluster") monotonic, align 2
+ ret bfloat %retval
+}
+
+define bfloat @monotonic_bfloat_shared_gpu(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_bfloat_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_shared_gpu_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr addrspace(3) %addr syncscope("device") monotonic, align 2
+ ret bfloat %retval
+}
+
+define bfloat @monotonic_bfloat_shared_sys_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_bfloat_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr addrspace(3) %addr syncscope("") monotonic, align 2
+ ret bfloat %retval
+}
+
+define bfloat @monotonic_bfloat_shared_cta_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_bfloat_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr addrspace(3) %addr syncscope("block") monotonic, align 2
+ ret bfloat %retval
+}
+
+define bfloat @monotonic_bfloat_shared_cluster_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_bfloat_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr addrspace(3) %addr syncscope("cluster") monotonic, align 2
+ ret bfloat %retval
+}
+
+define bfloat @monotonic_bfloat_shared_gpu_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_bfloat_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr addrspace(3) %addr syncscope("device") monotonic, align 2
+ ret bfloat %retval
+}
+
+define float @monotonic_float_shared_sys(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_float_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_shared_sys_param_0];
+; CHECK-NEXT: ld.volatile.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr addrspace(3) %addr syncscope("") monotonic, align 4
+ ret float %retval
+}
+
+define float @monotonic_float_shared_cta(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_float_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_shared_cta_param_0];
+; CHECK-NEXT: ld.volatile.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr addrspace(3) %addr syncscope("block") monotonic, align 4
+ ret float %retval
+}
+
+define float @monotonic_float_shared_cluster(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_float_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_shared_cluster_param_0];
+; CHECK-NEXT: ld.volatile.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr addrspace(3) %addr syncscope("cluster") monotonic, align 4
+ ret float %retval
+}
+
+define float @monotonic_float_shared_gpu(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_float_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_shared_gpu_param_0];
+; CHECK-NEXT: ld.volatile.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr addrspace(3) %addr syncscope("device") monotonic, align 4
+ ret float %retval
+}
+
+define float @monotonic_float_shared_sys_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_float_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr addrspace(3) %addr syncscope("") monotonic, align 4
+ ret float %retval
+}
+
+define float @monotonic_float_shared_cta_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_float_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr addrspace(3) %addr syncscope("block") monotonic, align 4
+ ret float %retval
+}
+
+define float @monotonic_float_shared_cluster_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_float_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr addrspace(3) %addr syncscope("cluster") monotonic, align 4
+ ret float %retval
+}
+
+define float @monotonic_float_shared_gpu_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_float_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr addrspace(3) %addr syncscope("device") monotonic, align 4
+ ret float %retval
+}
+
+define double @monotonic_double_shared_sys(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_double_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_shared_sys_param_0];
+; CHECK-NEXT: ld.volatile.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr addrspace(3) %addr syncscope("") monotonic, align 8
+ ret double %retval
+}
+
+define double @monotonic_double_shared_cta(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_double_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_shared_cta_param_0];
+; CHECK-NEXT: ld.volatile.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr addrspace(3) %addr syncscope("block") monotonic, align 8
+ ret double %retval
+}
+
+define double @monotonic_double_shared_cluster(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_double_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_shared_cluster_param_0];
+; CHECK-NEXT: ld.volatile.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr addrspace(3) %addr syncscope("cluster") monotonic, align 8
+ ret double %retval
+}
+
+define double @monotonic_double_shared_gpu(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_double_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_shared_gpu_param_0];
+; CHECK-NEXT: ld.volatile.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr addrspace(3) %addr syncscope("device") monotonic, align 8
+ ret double %retval
+}
+
+define double @monotonic_double_shared_sys_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_double_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr addrspace(3) %addr syncscope("") monotonic, align 8
+ ret double %retval
+}
+
+define double @monotonic_double_shared_cta_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_double_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr addrspace(3) %addr syncscope("block") monotonic, align 8
+ ret double %retval
+}
+
+define double @monotonic_double_shared_cluster_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_double_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr addrspace(3) %addr syncscope("cluster") monotonic, align 8
+ ret double %retval
+}
+
+define double @monotonic_double_shared_gpu_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_double_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr addrspace(3) %addr syncscope("device") monotonic, align 8
+ ret double %retval
+}
+
+define i8 @monotonic_i8_local_sys() {
+; CHECK-LABEL: monotonic_i8_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot544[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot544;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b8 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i8, ptr addrspace(5) %slot syncscope("") monotonic, align 1
+ ret i8 %retval
+}
+
+define i8 @monotonic_i8_local_cta() {
+; CHECK-LABEL: monotonic_i8_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot545[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot545;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b8 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i8, ptr addrspace(5) %slot syncscope("block") monotonic, align 1
+ ret i8 %retval
+}
+
+define i8 @monotonic_i8_local_cluster() {
+; CHECK-LABEL: monotonic_i8_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot546[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot546;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b8 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i8, ptr addrspace(5) %slot syncscope("cluster") monotonic, align 1
+ ret i8 %retval
+}
+
+define i8 @monotonic_i8_local_gpu() {
+; CHECK-LABEL: monotonic_i8_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot547[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot547;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b8 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i8, ptr addrspace(5) %slot syncscope("device") monotonic, align 1
+ ret i8 %retval
+}
+
+define i8 @monotonic_i8_local_sys_volatile() {
+; CHECK-LABEL: monotonic_i8_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot548[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot548;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b8 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i8, ptr addrspace(5) %slot syncscope("") monotonic, align 1
+ ret i8 %retval
+}
+
+define i8 @monotonic_i8_local_cta_volatile() {
+; CHECK-LABEL: monotonic_i8_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot549[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot549;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b8 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i8, ptr addrspace(5) %slot syncscope("block") monotonic, align 1
+ ret i8 %retval
+}
+
+define i8 @monotonic_i8_local_cluster_volatile() {
+; CHECK-LABEL: monotonic_i8_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot550[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot550;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b8 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i8, ptr addrspace(5) %slot syncscope("cluster") monotonic, align 1
+ ret i8 %retval
+}
+
+define i8 @monotonic_i8_local_gpu_volatile() {
+; CHECK-LABEL: monotonic_i8_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot551[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot551;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b8 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i8, ptr addrspace(5) %slot syncscope("device") monotonic, align 1
+ ret i8 %retval
+}
+
+define i16 @monotonic_i16_local_sys() {
+; CHECK-LABEL: monotonic_i16_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot552[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot552;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i16, ptr addrspace(5) %slot syncscope("") monotonic, align 2
+ ret i16 %retval
+}
+
+define i16 @monotonic_i16_local_cta() {
+; CHECK-LABEL: monotonic_i16_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot553[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot553;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i16, ptr addrspace(5) %slot syncscope("block") monotonic, align 2
+ ret i16 %retval
+}
+
+define i16 @monotonic_i16_local_cluster() {
+; CHECK-LABEL: monotonic_i16_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot554[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot554;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i16, ptr addrspace(5) %slot syncscope("cluster") monotonic, align 2
+ ret i16 %retval
+}
+
+define i16 @monotonic_i16_local_gpu() {
+; CHECK-LABEL: monotonic_i16_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot555[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot555;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i16, ptr addrspace(5) %slot syncscope("device") monotonic, align 2
+ ret i16 %retval
+}
+
+define i16 @monotonic_i16_local_sys_volatile() {
+; CHECK-LABEL: monotonic_i16_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot556[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot556;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i16, ptr addrspace(5) %slot syncscope("") monotonic, align 2
+ ret i16 %retval
+}
+
+define i16 @monotonic_i16_local_cta_volatile() {
+; CHECK-LABEL: monotonic_i16_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot557[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot557;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i16, ptr addrspace(5) %slot syncscope("block") monotonic, align 2
+ ret i16 %retval
+}
+
+define i16 @monotonic_i16_local_cluster_volatile() {
+; CHECK-LABEL: monotonic_i16_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot558[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot558;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i16, ptr addrspace(5) %slot syncscope("cluster") monotonic, align 2
+ ret i16 %retval
+}
+
+define i16 @monotonic_i16_local_gpu_volatile() {
+; CHECK-LABEL: monotonic_i16_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot559[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot559;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i16, ptr addrspace(5) %slot syncscope("device") monotonic, align 2
+ ret i16 %retval
+}
+
+define i32 @monotonic_i32_local_sys() {
+; CHECK-LABEL: monotonic_i32_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot560[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot560;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i32, ptr addrspace(5) %slot syncscope("") monotonic, align 4
+ ret i32 %retval
+}
+
+define i32 @monotonic_i32_local_cta() {
+; CHECK-LABEL: monotonic_i32_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot561[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot561;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i32, ptr addrspace(5) %slot syncscope("block") monotonic, align 4
+ ret i32 %retval
+}
+
+define i32 @monotonic_i32_local_cluster() {
+; CHECK-LABEL: monotonic_i32_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot562[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot562;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i32, ptr addrspace(5) %slot syncscope("cluster") monotonic, align 4
+ ret i32 %retval
+}
+
+define i32 @monotonic_i32_local_gpu() {
+; CHECK-LABEL: monotonic_i32_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot563[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot563;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i32, ptr addrspace(5) %slot syncscope("device") monotonic, align 4
+ ret i32 %retval
+}
+
+define i32 @monotonic_i32_local_sys_volatile() {
+; CHECK-LABEL: monotonic_i32_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot564[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot564;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i32, ptr addrspace(5) %slot syncscope("") monotonic, align 4
+ ret i32 %retval
+}
+
+define i32 @monotonic_i32_local_cta_volatile() {
+; CHECK-LABEL: monotonic_i32_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot565[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot565;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i32, ptr addrspace(5) %slot syncscope("block") monotonic, align 4
+ ret i32 %retval
+}
+
+define i32 @monotonic_i32_local_cluster_volatile() {
+; CHECK-LABEL: monotonic_i32_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot566[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot566;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i32, ptr addrspace(5) %slot syncscope("cluster") monotonic, align 4
+ ret i32 %retval
+}
+
+define i32 @monotonic_i32_local_gpu_volatile() {
+; CHECK-LABEL: monotonic_i32_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot567[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot567;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i32, ptr addrspace(5) %slot syncscope("device") monotonic, align 4
+ ret i32 %retval
+}
+
+define i64 @monotonic_i64_local_sys() {
+; CHECK-LABEL: monotonic_i64_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot568[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot568;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i64, ptr addrspace(5) %slot syncscope("") monotonic, align 8
+ ret i64 %retval
+}
+
+define i64 @monotonic_i64_local_cta() {
+; CHECK-LABEL: monotonic_i64_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot569[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot569;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i64, ptr addrspace(5) %slot syncscope("block") monotonic, align 8
+ ret i64 %retval
+}
+
+define i64 @monotonic_i64_local_cluster() {
+; CHECK-LABEL: monotonic_i64_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot570[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot570;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i64, ptr addrspace(5) %slot syncscope("cluster") monotonic, align 8
+ ret i64 %retval
+}
+
+define i64 @monotonic_i64_local_gpu() {
+; CHECK-LABEL: monotonic_i64_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot571[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot571;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i64, ptr addrspace(5) %slot syncscope("device") monotonic, align 8
+ ret i64 %retval
+}
+
+define i64 @monotonic_i64_local_sys_volatile() {
+; CHECK-LABEL: monotonic_i64_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot572[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot572;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i64, ptr addrspace(5) %slot syncscope("") monotonic, align 8
+ ret i64 %retval
+}
+
+define i64 @monotonic_i64_local_cta_volatile() {
+; CHECK-LABEL: monotonic_i64_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot573[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot573;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i64, ptr addrspace(5) %slot syncscope("block") monotonic, align 8
+ ret i64 %retval
+}
+
+define i64 @monotonic_i64_local_cluster_volatile() {
+; CHECK-LABEL: monotonic_i64_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot574[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot574;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i64, ptr addrspace(5) %slot syncscope("cluster") monotonic, align 8
+ ret i64 %retval
+}
+
+define i64 @monotonic_i64_local_gpu_volatile() {
+; CHECK-LABEL: monotonic_i64_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot575[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot575;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i64, ptr addrspace(5) %slot syncscope("device") monotonic, align 8
+ ret i64 %retval
+}
+
+define half @monotonic_half_local_sys() {
+; CHECK-LABEL: monotonic_half_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot576[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot576;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic half, ptr addrspace(5) %slot syncscope("") monotonic, align 2
+ ret half %retval
+}
+
+define half @monotonic_half_local_cta() {
+; CHECK-LABEL: monotonic_half_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot577[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot577;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic half, ptr addrspace(5) %slot syncscope("block") monotonic, align 2
+ ret half %retval
+}
+
+define half @monotonic_half_local_cluster() {
+; CHECK-LABEL: monotonic_half_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot578[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot578;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic half, ptr addrspace(5) %slot syncscope("cluster") monotonic, align 2
+ ret half %retval
+}
+
+define half @monotonic_half_local_gpu() {
+; CHECK-LABEL: monotonic_half_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot579[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot579;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic half, ptr addrspace(5) %slot syncscope("device") monotonic, align 2
+ ret half %retval
+}
+
+define half @monotonic_half_local_sys_volatile() {
+; CHECK-LABEL: monotonic_half_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot580[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot580;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile half, ptr addrspace(5) %slot syncscope("") monotonic, align 2
+ ret half %retval
+}
+
+define half @monotonic_half_local_cta_volatile() {
+; CHECK-LABEL: monotonic_half_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot581[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot581;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile half, ptr addrspace(5) %slot syncscope("block") monotonic, align 2
+ ret half %retval
+}
+
+define half @monotonic_half_local_cluster_volatile() {
+; CHECK-LABEL: monotonic_half_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot582[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot582;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile half, ptr addrspace(5) %slot syncscope("cluster") monotonic, align 2
+ ret half %retval
+}
+
+define half @monotonic_half_local_gpu_volatile() {
+; CHECK-LABEL: monotonic_half_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot583[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot583;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile half, ptr addrspace(5) %slot syncscope("device") monotonic, align 2
+ ret half %retval
+}
+
+define bfloat @monotonic_bfloat_local_sys() {
+; CHECK-LABEL: monotonic_bfloat_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot584[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot584;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic bfloat, ptr addrspace(5) %slot syncscope("") monotonic, align 2
+ ret bfloat %retval
+}
+
+define bfloat @monotonic_bfloat_local_cta() {
+; CHECK-LABEL: monotonic_bfloat_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot585[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot585;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic bfloat, ptr addrspace(5) %slot syncscope("block") monotonic, align 2
+ ret bfloat %retval
+}
+
+define bfloat @monotonic_bfloat_local_cluster() {
+; CHECK-LABEL: monotonic_bfloat_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot586[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot586;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic bfloat, ptr addrspace(5) %slot syncscope("cluster") monotonic, align 2
+ ret bfloat %retval
+}
+
+define bfloat @monotonic_bfloat_local_gpu() {
+; CHECK-LABEL: monotonic_bfloat_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot587[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot587;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic bfloat, ptr addrspace(5) %slot syncscope("device") monotonic, align 2
+ ret bfloat %retval
+}
+
+define bfloat @monotonic_bfloat_local_sys_volatile() {
+; CHECK-LABEL: monotonic_bfloat_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot588[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot588;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile bfloat, ptr addrspace(5) %slot syncscope("") monotonic, align 2
+ ret bfloat %retval
+}
+
+define bfloat @monotonic_bfloat_local_cta_volatile() {
+; CHECK-LABEL: monotonic_bfloat_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot589[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot589;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile bfloat, ptr addrspace(5) %slot syncscope("block") monotonic, align 2
+ ret bfloat %retval
+}
+
+define bfloat @monotonic_bfloat_local_cluster_volatile() {
+; CHECK-LABEL: monotonic_bfloat_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot590[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot590;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile bfloat, ptr addrspace(5) %slot syncscope("cluster") monotonic, align 2
+ ret bfloat %retval
+}
+
+define bfloat @monotonic_bfloat_local_gpu_volatile() {
+; CHECK-LABEL: monotonic_bfloat_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot591[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot591;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile bfloat, ptr addrspace(5) %slot syncscope("device") monotonic, align 2
+ ret bfloat %retval
+}
+
+define float @monotonic_float_local_sys() {
+; CHECK-LABEL: monotonic_float_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot592[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot592;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic float, ptr addrspace(5) %slot syncscope("") monotonic, align 4
+ ret float %retval
+}
+
+define float @monotonic_float_local_cta() {
+; CHECK-LABEL: monotonic_float_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot593[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot593;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic float, ptr addrspace(5) %slot syncscope("block") monotonic, align 4
+ ret float %retval
+}
+
+define float @monotonic_float_local_cluster() {
+; CHECK-LABEL: monotonic_float_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot594[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot594;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic float, ptr addrspace(5) %slot syncscope("cluster") monotonic, align 4
+ ret float %retval
+}
+
+define float @monotonic_float_local_gpu() {
+; CHECK-LABEL: monotonic_float_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot595[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot595;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic float, ptr addrspace(5) %slot syncscope("device") monotonic, align 4
+ ret float %retval
+}
+
+define float @monotonic_float_local_sys_volatile() {
+; CHECK-LABEL: monotonic_float_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot596[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot596;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile float, ptr addrspace(5) %slot syncscope("") monotonic, align 4
+ ret float %retval
+}
+
+define float @monotonic_float_local_cta_volatile() {
+; CHECK-LABEL: monotonic_float_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot597[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot597;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile float, ptr addrspace(5) %slot syncscope("block") monotonic, align 4
+ ret float %retval
+}
+
+define float @monotonic_float_local_cluster_volatile() {
+; CHECK-LABEL: monotonic_float_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot598[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot598;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile float, ptr addrspace(5) %slot syncscope("cluster") monotonic, align 4
+ ret float %retval
+}
+
+define float @monotonic_float_local_gpu_volatile() {
+; CHECK-LABEL: monotonic_float_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot599[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot599;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile float, ptr addrspace(5) %slot syncscope("device") monotonic, align 4
+ ret float %retval
+}
+
+define double @monotonic_double_local_sys() {
+; CHECK-LABEL: monotonic_double_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot600[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot600;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic double, ptr addrspace(5) %slot syncscope("") monotonic, align 8
+ ret double %retval
+}
+
+define double @monotonic_double_local_cta() {
+; CHECK-LABEL: monotonic_double_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot601[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot601;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic double, ptr addrspace(5) %slot syncscope("block") monotonic, align 8
+ ret double %retval
+}
+
+define double @monotonic_double_local_cluster() {
+; CHECK-LABEL: monotonic_double_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot602[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot602;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic double, ptr addrspace(5) %slot syncscope("cluster") monotonic, align 8
+ ret double %retval
+}
+
+define double @monotonic_double_local_gpu() {
+; CHECK-LABEL: monotonic_double_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot603[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot603;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic double, ptr addrspace(5) %slot syncscope("device") monotonic, align 8
+ ret double %retval
+}
+
+define double @monotonic_double_local_sys_volatile() {
+; CHECK-LABEL: monotonic_double_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot604[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot604;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile double, ptr addrspace(5) %slot syncscope("") monotonic, align 8
+ ret double %retval
+}
+
+define double @monotonic_double_local_cta_volatile() {
+; CHECK-LABEL: monotonic_double_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot605[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot605;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile double, ptr addrspace(5) %slot syncscope("block") monotonic, align 8
+ ret double %retval
+}
+
+define double @monotonic_double_local_cluster_volatile() {
+; CHECK-LABEL: monotonic_double_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot606[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot606;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile double, ptr addrspace(5) %slot syncscope("cluster") monotonic, align 8
+ ret double %retval
+}
+
+define double @monotonic_double_local_gpu_volatile() {
+; CHECK-LABEL: monotonic_double_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot607[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot607;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile double, ptr addrspace(5) %slot syncscope("device") monotonic, align 8
+ ret double %retval
+}
+
+define i8 @acquire_i8_generic_sys(ptr %addr) {
+; CHECK-LABEL: acquire_i8_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i8_generic_sys_param_0];
+; CHECK-NEXT: ld.volatile.b8 %rs1, [%rd1];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr %addr syncscope("") acquire, align 1
+ ret i8 %retval
+}
+
+define i8 @acquire_i8_generic_cta(ptr %addr) {
+; CHECK-LABEL: acquire_i8_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i8_generic_cta_param_0];
+; CHECK-NEXT: ld.volatile.b8 %rs1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr %addr syncscope("block") acquire, align 1
+ ret i8 %retval
+}
+
+define i8 @acquire_i8_generic_cluster(ptr %addr) {
+; CHECK-LABEL: acquire_i8_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i8_generic_cluster_param_0];
+; CHECK-NEXT: ld.volatile.b8 %rs1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr %addr syncscope("cluster") acquire, align 1
+ ret i8 %retval
+}
+
+define i8 @acquire_i8_generic_gpu(ptr %addr) {
+; CHECK-LABEL: acquire_i8_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i8_generic_gpu_param_0];
+; CHECK-NEXT: ld.volatile.b8 %rs1, [%rd1];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr %addr syncscope("device") acquire, align 1
+ ret i8 %retval
+}
+
+define i8 @acquire_i8_generic_sys_volatile(ptr %addr) {
+; CHECK-LABEL: acquire_i8_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i8_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b8 %rs1, [%rd1];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr %addr syncscope("") acquire, align 1
+ ret i8 %retval
+}
+
+define i8 @acquire_i8_generic_cta_volatile(ptr %addr) {
+; CHECK-LABEL: acquire_i8_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i8_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b8 %rs1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr %addr syncscope("block") acquire, align 1
+ ret i8 %retval
+}
+
+define i8 @acquire_i8_generic_cluster_volatile(ptr %addr) {
+; CHECK-LABEL: acquire_i8_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i8_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b8 %rs1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr %addr syncscope("cluster") acquire, align 1
+ ret i8 %retval
+}
+
+define i8 @acquire_i8_generic_gpu_volatile(ptr %addr) {
+; CHECK-LABEL: acquire_i8_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i8_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b8 %rs1, [%rd1];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr %addr syncscope("device") acquire, align 1
+ ret i8 %retval
+}
+
+define i16 @acquire_i16_generic_sys(ptr %addr) {
+; CHECK-LABEL: acquire_i16_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i16_generic_sys_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr %addr syncscope("") acquire, align 2
+ ret i16 %retval
+}
+
+define i16 @acquire_i16_generic_cta(ptr %addr) {
+; CHECK-LABEL: acquire_i16_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i16_generic_cta_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr %addr syncscope("block") acquire, align 2
+ ret i16 %retval
+}
+
+define i16 @acquire_i16_generic_cluster(ptr %addr) {
+; CHECK-LABEL: acquire_i16_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i16_generic_cluster_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr %addr syncscope("cluster") acquire, align 2
+ ret i16 %retval
+}
+
+define i16 @acquire_i16_generic_gpu(ptr %addr) {
+; CHECK-LABEL: acquire_i16_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i16_generic_gpu_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr %addr syncscope("device") acquire, align 2
+ ret i16 %retval
+}
+
+define i16 @acquire_i16_generic_sys_volatile(ptr %addr) {
+; CHECK-LABEL: acquire_i16_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i16_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr %addr syncscope("") acquire, align 2
+ ret i16 %retval
+}
+
+define i16 @acquire_i16_generic_cta_volatile(ptr %addr) {
+; CHECK-LABEL: acquire_i16_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i16_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr %addr syncscope("block") acquire, align 2
+ ret i16 %retval
+}
+
+define i16 @acquire_i16_generic_cluster_volatile(ptr %addr) {
+; CHECK-LABEL: acquire_i16_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i16_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr %addr syncscope("cluster") acquire, align 2
+ ret i16 %retval
+}
+
+define i16 @acquire_i16_generic_gpu_volatile(ptr %addr) {
+; CHECK-LABEL: acquire_i16_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i16_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr %addr syncscope("device") acquire, align 2
+ ret i16 %retval
+}
+
+define i32 @acquire_i32_generic_sys(ptr %addr) {
+; CHECK-LABEL: acquire_i32_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i32_generic_sys_param_0];
+; CHECK-NEXT: ld.volatile.b32 %r1, [%rd1];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr %addr syncscope("") acquire, align 4
+ ret i32 %retval
+}
+
+define i32 @acquire_i32_generic_cta(ptr %addr) {
+; CHECK-LABEL: acquire_i32_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i32_generic_cta_param_0];
+; CHECK-NEXT: ld.volatile.b32 %r1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr %addr syncscope("block") acquire, align 4
+ ret i32 %retval
+}
+
+define i32 @acquire_i32_generic_cluster(ptr %addr) {
+; CHECK-LABEL: acquire_i32_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i32_generic_cluster_param_0];
+; CHECK-NEXT: ld.volatile.b32 %r1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr %addr syncscope("cluster") acquire, align 4
+ ret i32 %retval
+}
+
+define i32 @acquire_i32_generic_gpu(ptr %addr) {
+; CHECK-LABEL: acquire_i32_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i32_generic_gpu_param_0];
+; CHECK-NEXT: ld.volatile.b32 %r1, [%rd1];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr %addr syncscope("device") acquire, align 4
+ ret i32 %retval
+}
+
+define i32 @acquire_i32_generic_sys_volatile(ptr %addr) {
+; CHECK-LABEL: acquire_i32_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i32_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b32 %r1, [%rd1];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr %addr syncscope("") acquire, align 4
+ ret i32 %retval
+}
+
+define i32 @acquire_i32_generic_cta_volatile(ptr %addr) {
+; CHECK-LABEL: acquire_i32_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i32_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b32 %r1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr %addr syncscope("block") acquire, align 4
+ ret i32 %retval
+}
+
+define i32 @acquire_i32_generic_cluster_volatile(ptr %addr) {
+; CHECK-LABEL: acquire_i32_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i32_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b32 %r1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr %addr syncscope("cluster") acquire, align 4
+ ret i32 %retval
+}
+
+define i32 @acquire_i32_generic_gpu_volatile(ptr %addr) {
+; CHECK-LABEL: acquire_i32_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i32_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b32 %r1, [%rd1];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr %addr syncscope("device") acquire, align 4
+ ret i32 %retval
+}
+
+define i64 @acquire_i64_generic_sys(ptr %addr) {
+; CHECK-LABEL: acquire_i64_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i64_generic_sys_param_0];
+; CHECK-NEXT: ld.volatile.b64 %rd2, [%rd1];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr %addr syncscope("") acquire, align 8
+ ret i64 %retval
+}
+
+define i64 @acquire_i64_generic_cta(ptr %addr) {
+; CHECK-LABEL: acquire_i64_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i64_generic_cta_param_0];
+; CHECK-NEXT: ld.volatile.b64 %rd2, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr %addr syncscope("block") acquire, align 8
+ ret i64 %retval
+}
+
+define i64 @acquire_i64_generic_cluster(ptr %addr) {
+; CHECK-LABEL: acquire_i64_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i64_generic_cluster_param_0];
+; CHECK-NEXT: ld.volatile.b64 %rd2, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr %addr syncscope("cluster") acquire, align 8
+ ret i64 %retval
+}
+
+define i64 @acquire_i64_generic_gpu(ptr %addr) {
+; CHECK-LABEL: acquire_i64_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i64_generic_gpu_param_0];
+; CHECK-NEXT: ld.volatile.b64 %rd2, [%rd1];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr %addr syncscope("device") acquire, align 8
+ ret i64 %retval
+}
+
+define i64 @acquire_i64_generic_sys_volatile(ptr %addr) {
+; CHECK-LABEL: acquire_i64_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i64_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b64 %rd2, [%rd1];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr %addr syncscope("") acquire, align 8
+ ret i64 %retval
+}
+
+define i64 @acquire_i64_generic_cta_volatile(ptr %addr) {
+; CHECK-LABEL: acquire_i64_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i64_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b64 %rd2, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr %addr syncscope("block") acquire, align 8
+ ret i64 %retval
+}
+
+define i64 @acquire_i64_generic_cluster_volatile(ptr %addr) {
+; CHECK-LABEL: acquire_i64_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i64_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b64 %rd2, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr %addr syncscope("cluster") acquire, align 8
+ ret i64 %retval
+}
+
+define i64 @acquire_i64_generic_gpu_volatile(ptr %addr) {
+; CHECK-LABEL: acquire_i64_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i64_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b64 %rd2, [%rd1];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr %addr syncscope("device") acquire, align 8
+ ret i64 %retval
+}
+
+define half @acquire_half_generic_sys(ptr %addr) {
+; CHECK-LABEL: acquire_half_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_half_generic_sys_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr %addr syncscope("") acquire, align 2
+ ret half %retval
+}
+
+define half @acquire_half_generic_cta(ptr %addr) {
+; CHECK-LABEL: acquire_half_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_half_generic_cta_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr %addr syncscope("block") acquire, align 2
+ ret half %retval
+}
+
+define half @acquire_half_generic_cluster(ptr %addr) {
+; CHECK-LABEL: acquire_half_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_half_generic_cluster_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr %addr syncscope("cluster") acquire, align 2
+ ret half %retval
+}
+
+define half @acquire_half_generic_gpu(ptr %addr) {
+; CHECK-LABEL: acquire_half_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_half_generic_gpu_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr %addr syncscope("device") acquire, align 2
+ ret half %retval
+}
+
+define half @acquire_half_generic_sys_volatile(ptr %addr) {
+; CHECK-LABEL: acquire_half_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_half_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr %addr syncscope("") acquire, align 2
+ ret half %retval
+}
+
+define half @acquire_half_generic_cta_volatile(ptr %addr) {
+; CHECK-LABEL: acquire_half_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_half_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr %addr syncscope("block") acquire, align 2
+ ret half %retval
+}
+
+define half @acquire_half_generic_cluster_volatile(ptr %addr) {
+; CHECK-LABEL: acquire_half_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_half_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr %addr syncscope("cluster") acquire, align 2
+ ret half %retval
+}
+
+define half @acquire_half_generic_gpu_volatile(ptr %addr) {
+; CHECK-LABEL: acquire_half_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_half_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr %addr syncscope("device") acquire, align 2
+ ret half %retval
+}
+
+define bfloat @acquire_bfloat_generic_sys(ptr %addr) {
+; CHECK-LABEL: acquire_bfloat_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_bfloat_generic_sys_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr %addr syncscope("") acquire, align 2
+ ret bfloat %retval
+}
+
+define bfloat @acquire_bfloat_generic_cta(ptr %addr) {
+; CHECK-LABEL: acquire_bfloat_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_bfloat_generic_cta_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr %addr syncscope("block") acquire, align 2
+ ret bfloat %retval
+}
+
+define bfloat @acquire_bfloat_generic_cluster(ptr %addr) {
+; CHECK-LABEL: acquire_bfloat_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_bfloat_generic_cluster_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr %addr syncscope("cluster") acquire, align 2
+ ret bfloat %retval
+}
+
+define bfloat @acquire_bfloat_generic_gpu(ptr %addr) {
+; CHECK-LABEL: acquire_bfloat_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_bfloat_generic_gpu_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr %addr syncscope("device") acquire, align 2
+ ret bfloat %retval
+}
+
+define bfloat @acquire_bfloat_generic_sys_volatile(ptr %addr) {
+; CHECK-LABEL: acquire_bfloat_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_bfloat_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr %addr syncscope("") acquire, align 2
+ ret bfloat %retval
+}
+
+define bfloat @acquire_bfloat_generic_cta_volatile(ptr %addr) {
+; CHECK-LABEL: acquire_bfloat_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_bfloat_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr %addr syncscope("block") acquire, align 2
+ ret bfloat %retval
+}
+
+define bfloat @acquire_bfloat_generic_cluster_volatile(ptr %addr) {
+; CHECK-LABEL: acquire_bfloat_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_bfloat_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr %addr syncscope("cluster") acquire, align 2
+ ret bfloat %retval
+}
+
+define bfloat @acquire_bfloat_generic_gpu_volatile(ptr %addr) {
+; CHECK-LABEL: acquire_bfloat_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_bfloat_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr %addr syncscope("device") acquire, align 2
+ ret bfloat %retval
+}
+
+define float @acquire_float_generic_sys(ptr %addr) {
+; CHECK-LABEL: acquire_float_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_float_generic_sys_param_0];
+; CHECK-NEXT: ld.volatile.b32 %r1, [%rd1];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr %addr syncscope("") acquire, align 4
+ ret float %retval
+}
+
+define float @acquire_float_generic_cta(ptr %addr) {
+; CHECK-LABEL: acquire_float_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_float_generic_cta_param_0];
+; CHECK-NEXT: ld.volatile.b32 %r1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr %addr syncscope("block") acquire, align 4
+ ret float %retval
+}
+
+define float @acquire_float_generic_cluster(ptr %addr) {
+; CHECK-LABEL: acquire_float_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_float_generic_cluster_param_0];
+; CHECK-NEXT: ld.volatile.b32 %r1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr %addr syncscope("cluster") acquire, align 4
+ ret float %retval
+}
+
+define float @acquire_float_generic_gpu(ptr %addr) {
+; CHECK-LABEL: acquire_float_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_float_generic_gpu_param_0];
+; CHECK-NEXT: ld.volatile.b32 %r1, [%rd1];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr %addr syncscope("device") acquire, align 4
+ ret float %retval
+}
+
+define float @acquire_float_generic_sys_volatile(ptr %addr) {
+; CHECK-LABEL: acquire_float_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_float_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b32 %r1, [%rd1];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr %addr syncscope("") acquire, align 4
+ ret float %retval
+}
+
+define float @acquire_float_generic_cta_volatile(ptr %addr) {
+; CHECK-LABEL: acquire_float_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_float_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b32 %r1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr %addr syncscope("block") acquire, align 4
+ ret float %retval
+}
+
+define float @acquire_float_generic_cluster_volatile(ptr %addr) {
+; CHECK-LABEL: acquire_float_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_float_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b32 %r1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr %addr syncscope("cluster") acquire, align 4
+ ret float %retval
+}
+
+define float @acquire_float_generic_gpu_volatile(ptr %addr) {
+; CHECK-LABEL: acquire_float_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_float_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b32 %r1, [%rd1];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr %addr syncscope("device") acquire, align 4
+ ret float %retval
+}
+
+define double @acquire_double_generic_sys(ptr %addr) {
+; CHECK-LABEL: acquire_double_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_double_generic_sys_param_0];
+; CHECK-NEXT: ld.volatile.b64 %rd2, [%rd1];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr %addr syncscope("") acquire, align 8
+ ret double %retval
+}
+
+define double @acquire_double_generic_cta(ptr %addr) {
+; CHECK-LABEL: acquire_double_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_double_generic_cta_param_0];
+; CHECK-NEXT: ld.volatile.b64 %rd2, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr %addr syncscope("block") acquire, align 8
+ ret double %retval
+}
+
+define double @acquire_double_generic_cluster(ptr %addr) {
+; CHECK-LABEL: acquire_double_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_double_generic_cluster_param_0];
+; CHECK-NEXT: ld.volatile.b64 %rd2, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr %addr syncscope("cluster") acquire, align 8
+ ret double %retval
+}
+
+define double @acquire_double_generic_gpu(ptr %addr) {
+; CHECK-LABEL: acquire_double_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_double_generic_gpu_param_0];
+; CHECK-NEXT: ld.volatile.b64 %rd2, [%rd1];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr %addr syncscope("device") acquire, align 8
+ ret double %retval
+}
+
+define double @acquire_double_generic_sys_volatile(ptr %addr) {
+; CHECK-LABEL: acquire_double_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_double_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b64 %rd2, [%rd1];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr %addr syncscope("") acquire, align 8
+ ret double %retval
+}
+
+define double @acquire_double_generic_cta_volatile(ptr %addr) {
+; CHECK-LABEL: acquire_double_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_double_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b64 %rd2, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr %addr syncscope("block") acquire, align 8
+ ret double %retval
+}
+
+define double @acquire_double_generic_cluster_volatile(ptr %addr) {
+; CHECK-LABEL: acquire_double_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_double_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b64 %rd2, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr %addr syncscope("cluster") acquire, align 8
+ ret double %retval
+}
+
+define double @acquire_double_generic_gpu_volatile(ptr %addr) {
+; CHECK-LABEL: acquire_double_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_double_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b64 %rd2, [%rd1];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr %addr syncscope("device") acquire, align 8
+ ret double %retval
+}
+
+define i8 @acquire_i8_global_sys(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_i8_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i8_global_sys_param_0];
+; CHECK-NEXT: ld.volatile.global.b8 %rs1, [%rd1];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr addrspace(1) %addr syncscope("") acquire, align 1
+ ret i8 %retval
+}
+
+define i8 @acquire_i8_global_cta(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_i8_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i8_global_cta_param_0];
+; CHECK-NEXT: ld.volatile.global.b8 %rs1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr addrspace(1) %addr syncscope("block") acquire, align 1
+ ret i8 %retval
+}
+
+define i8 @acquire_i8_global_cluster(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_i8_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i8_global_cluster_param_0];
+; CHECK-NEXT: ld.volatile.global.b8 %rs1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr addrspace(1) %addr syncscope("cluster") acquire, align 1
+ ret i8 %retval
+}
+
+define i8 @acquire_i8_global_gpu(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_i8_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i8_global_gpu_param_0];
+; CHECK-NEXT: ld.volatile.global.b8 %rs1, [%rd1];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr addrspace(1) %addr syncscope("device") acquire, align 1
+ ret i8 %retval
+}
+
+define i8 @acquire_i8_global_sys_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_i8_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i8_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b8 %rs1, [%rd1];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr addrspace(1) %addr syncscope("") acquire, align 1
+ ret i8 %retval
+}
+
+define i8 @acquire_i8_global_cta_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_i8_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i8_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b8 %rs1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr addrspace(1) %addr syncscope("block") acquire, align 1
+ ret i8 %retval
+}
+
+define i8 @acquire_i8_global_cluster_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_i8_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i8_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b8 %rs1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr addrspace(1) %addr syncscope("cluster") acquire, align 1
+ ret i8 %retval
+}
+
+define i8 @acquire_i8_global_gpu_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_i8_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i8_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b8 %rs1, [%rd1];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr addrspace(1) %addr syncscope("device") acquire, align 1
+ ret i8 %retval
+}
+
+define i16 @acquire_i16_global_sys(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_i16_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i16_global_sys_param_0];
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr addrspace(1) %addr syncscope("") acquire, align 2
+ ret i16 %retval
+}
+
+define i16 @acquire_i16_global_cta(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_i16_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i16_global_cta_param_0];
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr addrspace(1) %addr syncscope("block") acquire, align 2
+ ret i16 %retval
+}
+
+define i16 @acquire_i16_global_cluster(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_i16_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i16_global_cluster_param_0];
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr addrspace(1) %addr syncscope("cluster") acquire, align 2
+ ret i16 %retval
+}
+
+define i16 @acquire_i16_global_gpu(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_i16_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i16_global_gpu_param_0];
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr addrspace(1) %addr syncscope("device") acquire, align 2
+ ret i16 %retval
+}
+
+define i16 @acquire_i16_global_sys_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_i16_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i16_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr addrspace(1) %addr syncscope("") acquire, align 2
+ ret i16 %retval
+}
+
+define i16 @acquire_i16_global_cta_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_i16_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i16_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr addrspace(1) %addr syncscope("block") acquire, align 2
+ ret i16 %retval
+}
+
+define i16 @acquire_i16_global_cluster_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_i16_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i16_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr addrspace(1) %addr syncscope("cluster") acquire, align 2
+ ret i16 %retval
+}
+
+define i16 @acquire_i16_global_gpu_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_i16_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i16_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr addrspace(1) %addr syncscope("device") acquire, align 2
+ ret i16 %retval
+}
+
+define i32 @acquire_i32_global_sys(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_i32_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i32_global_sys_param_0];
+; CHECK-NEXT: ld.volatile.global.b32 %r1, [%rd1];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr addrspace(1) %addr syncscope("") acquire, align 4
+ ret i32 %retval
+}
+
+define i32 @acquire_i32_global_cta(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_i32_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i32_global_cta_param_0];
+; CHECK-NEXT: ld.volatile.global.b32 %r1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr addrspace(1) %addr syncscope("block") acquire, align 4
+ ret i32 %retval
+}
+
+define i32 @acquire_i32_global_cluster(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_i32_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i32_global_cluster_param_0];
+; CHECK-NEXT: ld.volatile.global.b32 %r1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr addrspace(1) %addr syncscope("cluster") acquire, align 4
+ ret i32 %retval
+}
+
+define i32 @acquire_i32_global_gpu(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_i32_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i32_global_gpu_param_0];
+; CHECK-NEXT: ld.volatile.global.b32 %r1, [%rd1];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr addrspace(1) %addr syncscope("device") acquire, align 4
+ ret i32 %retval
+}
+
+define i32 @acquire_i32_global_sys_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_i32_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i32_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b32 %r1, [%rd1];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr addrspace(1) %addr syncscope("") acquire, align 4
+ ret i32 %retval
+}
+
+define i32 @acquire_i32_global_cta_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_i32_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i32_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b32 %r1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr addrspace(1) %addr syncscope("block") acquire, align 4
+ ret i32 %retval
+}
+
+define i32 @acquire_i32_global_cluster_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_i32_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i32_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b32 %r1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr addrspace(1) %addr syncscope("cluster") acquire, align 4
+ ret i32 %retval
+}
+
+define i32 @acquire_i32_global_gpu_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_i32_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i32_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b32 %r1, [%rd1];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr addrspace(1) %addr syncscope("device") acquire, align 4
+ ret i32 %retval
+}
+
+define i64 @acquire_i64_global_sys(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_i64_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i64_global_sys_param_0];
+; CHECK-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr addrspace(1) %addr syncscope("") acquire, align 8
+ ret i64 %retval
+}
+
+define i64 @acquire_i64_global_cta(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_i64_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i64_global_cta_param_0];
+; CHECK-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr addrspace(1) %addr syncscope("block") acquire, align 8
+ ret i64 %retval
+}
+
+define i64 @acquire_i64_global_cluster(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_i64_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i64_global_cluster_param_0];
+; CHECK-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr addrspace(1) %addr syncscope("cluster") acquire, align 8
+ ret i64 %retval
+}
+
+define i64 @acquire_i64_global_gpu(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_i64_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i64_global_gpu_param_0];
+; CHECK-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr addrspace(1) %addr syncscope("device") acquire, align 8
+ ret i64 %retval
+}
+
+define i64 @acquire_i64_global_sys_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_i64_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i64_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr addrspace(1) %addr syncscope("") acquire, align 8
+ ret i64 %retval
+}
+
+define i64 @acquire_i64_global_cta_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_i64_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i64_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr addrspace(1) %addr syncscope("block") acquire, align 8
+ ret i64 %retval
+}
+
+define i64 @acquire_i64_global_cluster_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_i64_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i64_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr addrspace(1) %addr syncscope("cluster") acquire, align 8
+ ret i64 %retval
+}
+
+define i64 @acquire_i64_global_gpu_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_i64_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i64_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr addrspace(1) %addr syncscope("device") acquire, align 8
+ ret i64 %retval
+}
+
+define half @acquire_half_global_sys(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_half_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_half_global_sys_param_0];
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr addrspace(1) %addr syncscope("") acquire, align 2
+ ret half %retval
+}
+
+define half @acquire_half_global_cta(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_half_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_half_global_cta_param_0];
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr addrspace(1) %addr syncscope("block") acquire, align 2
+ ret half %retval
+}
+
+define half @acquire_half_global_cluster(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_half_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_half_global_cluster_param_0];
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr addrspace(1) %addr syncscope("cluster") acquire, align 2
+ ret half %retval
+}
+
+define half @acquire_half_global_gpu(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_half_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_half_global_gpu_param_0];
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr addrspace(1) %addr syncscope("device") acquire, align 2
+ ret half %retval
+}
+
+define half @acquire_half_global_sys_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_half_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_half_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr addrspace(1) %addr syncscope("") acquire, align 2
+ ret half %retval
+}
+
+define half @acquire_half_global_cta_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_half_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_half_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr addrspace(1) %addr syncscope("block") acquire, align 2
+ ret half %retval
+}
+
+define half @acquire_half_global_cluster_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_half_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_half_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr addrspace(1) %addr syncscope("cluster") acquire, align 2
+ ret half %retval
+}
+
+define half @acquire_half_global_gpu_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_half_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_half_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr addrspace(1) %addr syncscope("device") acquire, align 2
+ ret half %retval
+}
+
+define bfloat @acquire_bfloat_global_sys(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_bfloat_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_bfloat_global_sys_param_0];
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr addrspace(1) %addr syncscope("") acquire, align 2
+ ret bfloat %retval
+}
+
+define bfloat @acquire_bfloat_global_cta(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_bfloat_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_bfloat_global_cta_param_0];
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr addrspace(1) %addr syncscope("block") acquire, align 2
+ ret bfloat %retval
+}
+
+define bfloat @acquire_bfloat_global_cluster(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_bfloat_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_bfloat_global_cluster_param_0];
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr addrspace(1) %addr syncscope("cluster") acquire, align 2
+ ret bfloat %retval
+}
+
+define bfloat @acquire_bfloat_global_gpu(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_bfloat_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_bfloat_global_gpu_param_0];
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr addrspace(1) %addr syncscope("device") acquire, align 2
+ ret bfloat %retval
+}
+
+define bfloat @acquire_bfloat_global_sys_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_bfloat_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_bfloat_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr addrspace(1) %addr syncscope("") acquire, align 2
+ ret bfloat %retval
+}
+
+define bfloat @acquire_bfloat_global_cta_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_bfloat_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_bfloat_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr addrspace(1) %addr syncscope("block") acquire, align 2
+ ret bfloat %retval
+}
+
+define bfloat @acquire_bfloat_global_cluster_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_bfloat_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_bfloat_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr addrspace(1) %addr syncscope("cluster") acquire, align 2
+ ret bfloat %retval
+}
+
+define bfloat @acquire_bfloat_global_gpu_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_bfloat_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_bfloat_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr addrspace(1) %addr syncscope("device") acquire, align 2
+ ret bfloat %retval
+}
+
+define float @acquire_float_global_sys(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_float_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_float_global_sys_param_0];
+; CHECK-NEXT: ld.volatile.global.b32 %r1, [%rd1];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr addrspace(1) %addr syncscope("") acquire, align 4
+ ret float %retval
+}
+
+define float @acquire_float_global_cta(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_float_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_float_global_cta_param_0];
+; CHECK-NEXT: ld.volatile.global.b32 %r1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr addrspace(1) %addr syncscope("block") acquire, align 4
+ ret float %retval
+}
+
+define float @acquire_float_global_cluster(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_float_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_float_global_cluster_param_0];
+; CHECK-NEXT: ld.volatile.global.b32 %r1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr addrspace(1) %addr syncscope("cluster") acquire, align 4
+ ret float %retval
+}
+
+define float @acquire_float_global_gpu(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_float_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_float_global_gpu_param_0];
+; CHECK-NEXT: ld.volatile.global.b32 %r1, [%rd1];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr addrspace(1) %addr syncscope("device") acquire, align 4
+ ret float %retval
+}
+
+define float @acquire_float_global_sys_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_float_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_float_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b32 %r1, [%rd1];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr addrspace(1) %addr syncscope("") acquire, align 4
+ ret float %retval
+}
+
+define float @acquire_float_global_cta_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_float_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_float_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b32 %r1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr addrspace(1) %addr syncscope("block") acquire, align 4
+ ret float %retval
+}
+
+define float @acquire_float_global_cluster_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_float_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_float_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b32 %r1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr addrspace(1) %addr syncscope("cluster") acquire, align 4
+ ret float %retval
+}
+
+define float @acquire_float_global_gpu_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_float_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_float_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b32 %r1, [%rd1];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr addrspace(1) %addr syncscope("device") acquire, align 4
+ ret float %retval
+}
+
+define double @acquire_double_global_sys(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_double_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_double_global_sys_param_0];
+; CHECK-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr addrspace(1) %addr syncscope("") acquire, align 8
+ ret double %retval
+}
+
+define double @acquire_double_global_cta(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_double_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_double_global_cta_param_0];
+; CHECK-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr addrspace(1) %addr syncscope("block") acquire, align 8
+ ret double %retval
+}
+
+define double @acquire_double_global_cluster(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_double_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_double_global_cluster_param_0];
+; CHECK-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr addrspace(1) %addr syncscope("cluster") acquire, align 8
+ ret double %retval
+}
+
+define double @acquire_double_global_gpu(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_double_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_double_global_gpu_param_0];
+; CHECK-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr addrspace(1) %addr syncscope("device") acquire, align 8
+ ret double %retval
+}
+
+define double @acquire_double_global_sys_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_double_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_double_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr addrspace(1) %addr syncscope("") acquire, align 8
+ ret double %retval
+}
+
+define double @acquire_double_global_cta_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_double_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_double_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr addrspace(1) %addr syncscope("block") acquire, align 8
+ ret double %retval
+}
+
+define double @acquire_double_global_cluster_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_double_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_double_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr addrspace(1) %addr syncscope("cluster") acquire, align 8
+ ret double %retval
+}
+
+define double @acquire_double_global_gpu_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_double_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_double_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr addrspace(1) %addr syncscope("device") acquire, align 8
+ ret double %retval
+}
+
+define i8 @acquire_i8_shared_sys(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_i8_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i8_shared_sys_param_0];
+; CHECK-NEXT: ld.volatile.shared.b8 %rs1, [%rd1];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr addrspace(3) %addr syncscope("") acquire, align 1
+ ret i8 %retval
+}
+
+define i8 @acquire_i8_shared_cta(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_i8_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i8_shared_cta_param_0];
+; CHECK-NEXT: ld.volatile.shared.b8 %rs1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr addrspace(3) %addr syncscope("block") acquire, align 1
+ ret i8 %retval
+}
+
+define i8 @acquire_i8_shared_cluster(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_i8_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i8_shared_cluster_param_0];
+; CHECK-NEXT: ld.volatile.shared.b8 %rs1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr addrspace(3) %addr syncscope("cluster") acquire, align 1
+ ret i8 %retval
+}
+
+define i8 @acquire_i8_shared_gpu(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_i8_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i8_shared_gpu_param_0];
+; CHECK-NEXT: ld.volatile.shared.b8 %rs1, [%rd1];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr addrspace(3) %addr syncscope("device") acquire, align 1
+ ret i8 %retval
+}
+
+define i8 @acquire_i8_shared_sys_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_i8_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i8_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b8 %rs1, [%rd1];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr addrspace(3) %addr syncscope("") acquire, align 1
+ ret i8 %retval
+}
+
+define i8 @acquire_i8_shared_cta_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_i8_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i8_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b8 %rs1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr addrspace(3) %addr syncscope("block") acquire, align 1
+ ret i8 %retval
+}
+
+define i8 @acquire_i8_shared_cluster_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_i8_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i8_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b8 %rs1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr addrspace(3) %addr syncscope("cluster") acquire, align 1
+ ret i8 %retval
+}
+
+define i8 @acquire_i8_shared_gpu_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_i8_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i8_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b8 %rs1, [%rd1];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr addrspace(3) %addr syncscope("device") acquire, align 1
+ ret i8 %retval
+}
+
+define i16 @acquire_i16_shared_sys(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_i16_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i16_shared_sys_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr addrspace(3) %addr syncscope("") acquire, align 2
+ ret i16 %retval
+}
+
+define i16 @acquire_i16_shared_cta(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_i16_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i16_shared_cta_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr addrspace(3) %addr syncscope("block") acquire, align 2
+ ret i16 %retval
+}
+
+define i16 @acquire_i16_shared_cluster(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_i16_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i16_shared_cluster_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr addrspace(3) %addr syncscope("cluster") acquire, align 2
+ ret i16 %retval
+}
+
+define i16 @acquire_i16_shared_gpu(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_i16_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i16_shared_gpu_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr addrspace(3) %addr syncscope("device") acquire, align 2
+ ret i16 %retval
+}
+
+define i16 @acquire_i16_shared_sys_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_i16_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i16_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr addrspace(3) %addr syncscope("") acquire, align 2
+ ret i16 %retval
+}
+
+define i16 @acquire_i16_shared_cta_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_i16_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i16_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr addrspace(3) %addr syncscope("block") acquire, align 2
+ ret i16 %retval
+}
+
+define i16 @acquire_i16_shared_cluster_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_i16_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i16_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr addrspace(3) %addr syncscope("cluster") acquire, align 2
+ ret i16 %retval
+}
+
+define i16 @acquire_i16_shared_gpu_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_i16_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i16_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr addrspace(3) %addr syncscope("device") acquire, align 2
+ ret i16 %retval
+}
+
+define i32 @acquire_i32_shared_sys(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_i32_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i32_shared_sys_param_0];
+; CHECK-NEXT: ld.volatile.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr addrspace(3) %addr syncscope("") acquire, align 4
+ ret i32 %retval
+}
+
+define i32 @acquire_i32_shared_cta(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_i32_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i32_shared_cta_param_0];
+; CHECK-NEXT: ld.volatile.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr addrspace(3) %addr syncscope("block") acquire, align 4
+ ret i32 %retval
+}
+
+define i32 @acquire_i32_shared_cluster(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_i32_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i32_shared_cluster_param_0];
+; CHECK-NEXT: ld.volatile.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr addrspace(3) %addr syncscope("cluster") acquire, align 4
+ ret i32 %retval
+}
+
+define i32 @acquire_i32_shared_gpu(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_i32_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i32_shared_gpu_param_0];
+; CHECK-NEXT: ld.volatile.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr addrspace(3) %addr syncscope("device") acquire, align 4
+ ret i32 %retval
+}
+
+define i32 @acquire_i32_shared_sys_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_i32_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i32_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr addrspace(3) %addr syncscope("") acquire, align 4
+ ret i32 %retval
+}
+
+define i32 @acquire_i32_shared_cta_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_i32_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i32_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr addrspace(3) %addr syncscope("block") acquire, align 4
+ ret i32 %retval
+}
+
+define i32 @acquire_i32_shared_cluster_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_i32_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i32_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr addrspace(3) %addr syncscope("cluster") acquire, align 4
+ ret i32 %retval
+}
+
+define i32 @acquire_i32_shared_gpu_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_i32_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i32_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr addrspace(3) %addr syncscope("device") acquire, align 4
+ ret i32 %retval
+}
+
+define i64 @acquire_i64_shared_sys(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_i64_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i64_shared_sys_param_0];
+; CHECK-NEXT: ld.volatile.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr addrspace(3) %addr syncscope("") acquire, align 8
+ ret i64 %retval
+}
+
+define i64 @acquire_i64_shared_cta(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_i64_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i64_shared_cta_param_0];
+; CHECK-NEXT: ld.volatile.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr addrspace(3) %addr syncscope("block") acquire, align 8
+ ret i64 %retval
+}
+
+define i64 @acquire_i64_shared_cluster(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_i64_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i64_shared_cluster_param_0];
+; CHECK-NEXT: ld.volatile.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr addrspace(3) %addr syncscope("cluster") acquire, align 8
+ ret i64 %retval
+}
+
+define i64 @acquire_i64_shared_gpu(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_i64_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i64_shared_gpu_param_0];
+; CHECK-NEXT: ld.volatile.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr addrspace(3) %addr syncscope("device") acquire, align 8
+ ret i64 %retval
+}
+
+define i64 @acquire_i64_shared_sys_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_i64_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i64_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr addrspace(3) %addr syncscope("") acquire, align 8
+ ret i64 %retval
+}
+
+define i64 @acquire_i64_shared_cta_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_i64_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i64_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr addrspace(3) %addr syncscope("block") acquire, align 8
+ ret i64 %retval
+}
+
+define i64 @acquire_i64_shared_cluster_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_i64_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i64_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr addrspace(3) %addr syncscope("cluster") acquire, align 8
+ ret i64 %retval
+}
+
+define i64 @acquire_i64_shared_gpu_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_i64_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i64_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr addrspace(3) %addr syncscope("device") acquire, align 8
+ ret i64 %retval
+}
+
+define half @acquire_half_shared_sys(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_half_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_half_shared_sys_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr addrspace(3) %addr syncscope("") acquire, align 2
+ ret half %retval
+}
+
+define half @acquire_half_shared_cta(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_half_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_half_shared_cta_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr addrspace(3) %addr syncscope("block") acquire, align 2
+ ret half %retval
+}
+
+define half @acquire_half_shared_cluster(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_half_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_half_shared_cluster_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr addrspace(3) %addr syncscope("cluster") acquire, align 2
+ ret half %retval
+}
+
+define half @acquire_half_shared_gpu(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_half_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_half_shared_gpu_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr addrspace(3) %addr syncscope("device") acquire, align 2
+ ret half %retval
+}
+
+define half @acquire_half_shared_sys_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_half_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_half_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr addrspace(3) %addr syncscope("") acquire, align 2
+ ret half %retval
+}
+
+define half @acquire_half_shared_cta_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_half_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_half_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr addrspace(3) %addr syncscope("block") acquire, align 2
+ ret half %retval
+}
+
+define half @acquire_half_shared_cluster_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_half_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_half_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr addrspace(3) %addr syncscope("cluster") acquire, align 2
+ ret half %retval
+}
+
+define half @acquire_half_shared_gpu_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_half_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_half_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr addrspace(3) %addr syncscope("device") acquire, align 2
+ ret half %retval
+}
+
+define bfloat @acquire_bfloat_shared_sys(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_bfloat_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_bfloat_shared_sys_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr addrspace(3) %addr syncscope("") acquire, align 2
+ ret bfloat %retval
+}
+
+define bfloat @acquire_bfloat_shared_cta(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_bfloat_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_bfloat_shared_cta_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr addrspace(3) %addr syncscope("block") acquire, align 2
+ ret bfloat %retval
+}
+
+define bfloat @acquire_bfloat_shared_cluster(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_bfloat_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_bfloat_shared_cluster_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr addrspace(3) %addr syncscope("cluster") acquire, align 2
+ ret bfloat %retval
+}
+
+define bfloat @acquire_bfloat_shared_gpu(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_bfloat_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_bfloat_shared_gpu_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr addrspace(3) %addr syncscope("device") acquire, align 2
+ ret bfloat %retval
+}
+
+define bfloat @acquire_bfloat_shared_sys_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_bfloat_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_bfloat_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr addrspace(3) %addr syncscope("") acquire, align 2
+ ret bfloat %retval
+}
+
+define bfloat @acquire_bfloat_shared_cta_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_bfloat_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_bfloat_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr addrspace(3) %addr syncscope("block") acquire, align 2
+ ret bfloat %retval
+}
+
+define bfloat @acquire_bfloat_shared_cluster_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_bfloat_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_bfloat_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr addrspace(3) %addr syncscope("cluster") acquire, align 2
+ ret bfloat %retval
+}
+
+define bfloat @acquire_bfloat_shared_gpu_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_bfloat_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_bfloat_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr addrspace(3) %addr syncscope("device") acquire, align 2
+ ret bfloat %retval
+}
+
+define float @acquire_float_shared_sys(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_float_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_float_shared_sys_param_0];
+; CHECK-NEXT: ld.volatile.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr addrspace(3) %addr syncscope("") acquire, align 4
+ ret float %retval
+}
+
+define float @acquire_float_shared_cta(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_float_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_float_shared_cta_param_0];
+; CHECK-NEXT: ld.volatile.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr addrspace(3) %addr syncscope("block") acquire, align 4
+ ret float %retval
+}
+
+define float @acquire_float_shared_cluster(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_float_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_float_shared_cluster_param_0];
+; CHECK-NEXT: ld.volatile.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr addrspace(3) %addr syncscope("cluster") acquire, align 4
+ ret float %retval
+}
+
+define float @acquire_float_shared_gpu(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_float_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_float_shared_gpu_param_0];
+; CHECK-NEXT: ld.volatile.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr addrspace(3) %addr syncscope("device") acquire, align 4
+ ret float %retval
+}
+
+define float @acquire_float_shared_sys_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_float_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_float_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr addrspace(3) %addr syncscope("") acquire, align 4
+ ret float %retval
+}
+
+define float @acquire_float_shared_cta_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_float_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_float_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr addrspace(3) %addr syncscope("block") acquire, align 4
+ ret float %retval
+}
+
+define float @acquire_float_shared_cluster_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_float_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_float_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr addrspace(3) %addr syncscope("cluster") acquire, align 4
+ ret float %retval
+}
+
+define float @acquire_float_shared_gpu_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_float_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_float_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr addrspace(3) %addr syncscope("device") acquire, align 4
+ ret float %retval
+}
+
+define double @acquire_double_shared_sys(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_double_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_double_shared_sys_param_0];
+; CHECK-NEXT: ld.volatile.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr addrspace(3) %addr syncscope("") acquire, align 8
+ ret double %retval
+}
+
+define double @acquire_double_shared_cta(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_double_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_double_shared_cta_param_0];
+; CHECK-NEXT: ld.volatile.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr addrspace(3) %addr syncscope("block") acquire, align 8
+ ret double %retval
+}
+
+define double @acquire_double_shared_cluster(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_double_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_double_shared_cluster_param_0];
+; CHECK-NEXT: ld.volatile.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr addrspace(3) %addr syncscope("cluster") acquire, align 8
+ ret double %retval
+}
+
+define double @acquire_double_shared_gpu(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_double_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_double_shared_gpu_param_0];
+; CHECK-NEXT: ld.volatile.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr addrspace(3) %addr syncscope("device") acquire, align 8
+ ret double %retval
+}
+
+define double @acquire_double_shared_sys_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_double_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_double_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr addrspace(3) %addr syncscope("") acquire, align 8
+ ret double %retval
+}
+
+define double @acquire_double_shared_cta_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_double_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_double_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr addrspace(3) %addr syncscope("block") acquire, align 8
+ ret double %retval
+}
+
+define double @acquire_double_shared_cluster_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_double_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_double_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr addrspace(3) %addr syncscope("cluster") acquire, align 8
+ ret double %retval
+}
+
+define double @acquire_double_shared_gpu_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_double_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_double_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr addrspace(3) %addr syncscope("device") acquire, align 8
+ ret double %retval
+}
+
+define i8 @acquire_i8_local_sys() {
+; CHECK-LABEL: acquire_i8_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot800[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot800;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b8 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i8, ptr addrspace(5) %slot syncscope("") acquire, align 1
+ ret i8 %retval
+}
+
+define i8 @acquire_i8_local_cta() {
+; CHECK-LABEL: acquire_i8_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot801[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot801;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b8 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i8, ptr addrspace(5) %slot syncscope("block") acquire, align 1
+ ret i8 %retval
+}
+
+define i8 @acquire_i8_local_cluster() {
+; CHECK-LABEL: acquire_i8_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot802[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot802;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b8 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i8, ptr addrspace(5) %slot syncscope("cluster") acquire, align 1
+ ret i8 %retval
+}
+
+define i8 @acquire_i8_local_gpu() {
+; CHECK-LABEL: acquire_i8_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot803[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot803;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b8 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i8, ptr addrspace(5) %slot syncscope("device") acquire, align 1
+ ret i8 %retval
+}
+
+define i8 @acquire_i8_local_sys_volatile() {
+; CHECK-LABEL: acquire_i8_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot804[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot804;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b8 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i8, ptr addrspace(5) %slot syncscope("") acquire, align 1
+ ret i8 %retval
+}
+
+define i8 @acquire_i8_local_cta_volatile() {
+; CHECK-LABEL: acquire_i8_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot805[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot805;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b8 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i8, ptr addrspace(5) %slot syncscope("block") acquire, align 1
+ ret i8 %retval
+}
+
+define i8 @acquire_i8_local_cluster_volatile() {
+; CHECK-LABEL: acquire_i8_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot806[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot806;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b8 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i8, ptr addrspace(5) %slot syncscope("cluster") acquire, align 1
+ ret i8 %retval
+}
+
+define i8 @acquire_i8_local_gpu_volatile() {
+; CHECK-LABEL: acquire_i8_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot807[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot807;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b8 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i8, ptr addrspace(5) %slot syncscope("device") acquire, align 1
+ ret i8 %retval
+}
+
+define i16 @acquire_i16_local_sys() {
+; CHECK-LABEL: acquire_i16_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot808[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot808;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i16, ptr addrspace(5) %slot syncscope("") acquire, align 2
+ ret i16 %retval
+}
+
+define i16 @acquire_i16_local_cta() {
+; CHECK-LABEL: acquire_i16_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot809[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot809;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i16, ptr addrspace(5) %slot syncscope("block") acquire, align 2
+ ret i16 %retval
+}
+
+define i16 @acquire_i16_local_cluster() {
+; CHECK-LABEL: acquire_i16_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot810[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot810;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i16, ptr addrspace(5) %slot syncscope("cluster") acquire, align 2
+ ret i16 %retval
+}
+
+define i16 @acquire_i16_local_gpu() {
+; CHECK-LABEL: acquire_i16_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot811[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot811;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i16, ptr addrspace(5) %slot syncscope("device") acquire, align 2
+ ret i16 %retval
+}
+
+define i16 @acquire_i16_local_sys_volatile() {
+; CHECK-LABEL: acquire_i16_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot812[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot812;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i16, ptr addrspace(5) %slot syncscope("") acquire, align 2
+ ret i16 %retval
+}
+
+define i16 @acquire_i16_local_cta_volatile() {
+; CHECK-LABEL: acquire_i16_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot813[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot813;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i16, ptr addrspace(5) %slot syncscope("block") acquire, align 2
+ ret i16 %retval
+}
+
+define i16 @acquire_i16_local_cluster_volatile() {
+; CHECK-LABEL: acquire_i16_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot814[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot814;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i16, ptr addrspace(5) %slot syncscope("cluster") acquire, align 2
+ ret i16 %retval
+}
+
+define i16 @acquire_i16_local_gpu_volatile() {
+; CHECK-LABEL: acquire_i16_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot815[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot815;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i16, ptr addrspace(5) %slot syncscope("device") acquire, align 2
+ ret i16 %retval
+}
+
+define i32 @acquire_i32_local_sys() {
+; CHECK-LABEL: acquire_i32_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot816[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot816;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i32, ptr addrspace(5) %slot syncscope("") acquire, align 4
+ ret i32 %retval
+}
+
+define i32 @acquire_i32_local_cta() {
+; CHECK-LABEL: acquire_i32_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot817[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot817;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i32, ptr addrspace(5) %slot syncscope("block") acquire, align 4
+ ret i32 %retval
+}
+
+define i32 @acquire_i32_local_cluster() {
+; CHECK-LABEL: acquire_i32_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot818[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot818;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i32, ptr addrspace(5) %slot syncscope("cluster") acquire, align 4
+ ret i32 %retval
+}
+
+define i32 @acquire_i32_local_gpu() {
+; CHECK-LABEL: acquire_i32_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot819[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot819;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i32, ptr addrspace(5) %slot syncscope("device") acquire, align 4
+ ret i32 %retval
+}
+
+define i32 @acquire_i32_local_sys_volatile() {
+; CHECK-LABEL: acquire_i32_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot820[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot820;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i32, ptr addrspace(5) %slot syncscope("") acquire, align 4
+ ret i32 %retval
+}
+
+define i32 @acquire_i32_local_cta_volatile() {
+; CHECK-LABEL: acquire_i32_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot821[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot821;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i32, ptr addrspace(5) %slot syncscope("block") acquire, align 4
+ ret i32 %retval
+}
+
+define i32 @acquire_i32_local_cluster_volatile() {
+; CHECK-LABEL: acquire_i32_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot822[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot822;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i32, ptr addrspace(5) %slot syncscope("cluster") acquire, align 4
+ ret i32 %retval
+}
+
+define i32 @acquire_i32_local_gpu_volatile() {
+; CHECK-LABEL: acquire_i32_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot823[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot823;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i32, ptr addrspace(5) %slot syncscope("device") acquire, align 4
+ ret i32 %retval
+}
+
+define i64 @acquire_i64_local_sys() {
+; CHECK-LABEL: acquire_i64_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot824[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot824;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i64, ptr addrspace(5) %slot syncscope("") acquire, align 8
+ ret i64 %retval
+}
+
+define i64 @acquire_i64_local_cta() {
+; CHECK-LABEL: acquire_i64_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot825[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot825;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i64, ptr addrspace(5) %slot syncscope("block") acquire, align 8
+ ret i64 %retval
+}
+
+define i64 @acquire_i64_local_cluster() {
+; CHECK-LABEL: acquire_i64_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot826[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot826;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i64, ptr addrspace(5) %slot syncscope("cluster") acquire, align 8
+ ret i64 %retval
+}
+
+define i64 @acquire_i64_local_gpu() {
+; CHECK-LABEL: acquire_i64_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot827[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot827;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i64, ptr addrspace(5) %slot syncscope("device") acquire, align 8
+ ret i64 %retval
+}
+
+define i64 @acquire_i64_local_sys_volatile() {
+; CHECK-LABEL: acquire_i64_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot828[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot828;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i64, ptr addrspace(5) %slot syncscope("") acquire, align 8
+ ret i64 %retval
+}
+
+define i64 @acquire_i64_local_cta_volatile() {
+; CHECK-LABEL: acquire_i64_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot829[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot829;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i64, ptr addrspace(5) %slot syncscope("block") acquire, align 8
+ ret i64 %retval
+}
+
+define i64 @acquire_i64_local_cluster_volatile() {
+; CHECK-LABEL: acquire_i64_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot830[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot830;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i64, ptr addrspace(5) %slot syncscope("cluster") acquire, align 8
+ ret i64 %retval
+}
+
+define i64 @acquire_i64_local_gpu_volatile() {
+; CHECK-LABEL: acquire_i64_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot831[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot831;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i64, ptr addrspace(5) %slot syncscope("device") acquire, align 8
+ ret i64 %retval
+}
+
+define half @acquire_half_local_sys() {
+; CHECK-LABEL: acquire_half_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot832[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot832;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic half, ptr addrspace(5) %slot syncscope("") acquire, align 2
+ ret half %retval
+}
+
+define half @acquire_half_local_cta() {
+; CHECK-LABEL: acquire_half_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot833[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot833;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic half, ptr addrspace(5) %slot syncscope("block") acquire, align 2
+ ret half %retval
+}
+
+define half @acquire_half_local_cluster() {
+; CHECK-LABEL: acquire_half_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot834[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot834;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic half, ptr addrspace(5) %slot syncscope("cluster") acquire, align 2
+ ret half %retval
+}
+
+define half @acquire_half_local_gpu() {
+; CHECK-LABEL: acquire_half_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot835[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot835;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic half, ptr addrspace(5) %slot syncscope("device") acquire, align 2
+ ret half %retval
+}
+
+define half @acquire_half_local_sys_volatile() {
+; CHECK-LABEL: acquire_half_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot836[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot836;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile half, ptr addrspace(5) %slot syncscope("") acquire, align 2
+ ret half %retval
+}
+
+define half @acquire_half_local_cta_volatile() {
+; CHECK-LABEL: acquire_half_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot837[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot837;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile half, ptr addrspace(5) %slot syncscope("block") acquire, align 2
+ ret half %retval
+}
+
+define half @acquire_half_local_cluster_volatile() {
+; CHECK-LABEL: acquire_half_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot838[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot838;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile half, ptr addrspace(5) %slot syncscope("cluster") acquire, align 2
+ ret half %retval
+}
+
+define half @acquire_half_local_gpu_volatile() {
+; CHECK-LABEL: acquire_half_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot839[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot839;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile half, ptr addrspace(5) %slot syncscope("device") acquire, align 2
+ ret half %retval
+}
+
+define bfloat @acquire_bfloat_local_sys() {
+; CHECK-LABEL: acquire_bfloat_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot840[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot840;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic bfloat, ptr addrspace(5) %slot syncscope("") acquire, align 2
+ ret bfloat %retval
+}
+
+define bfloat @acquire_bfloat_local_cta() {
+; CHECK-LABEL: acquire_bfloat_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot841[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot841;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic bfloat, ptr addrspace(5) %slot syncscope("block") acquire, align 2
+ ret bfloat %retval
+}
+
+define bfloat @acquire_bfloat_local_cluster() {
+; CHECK-LABEL: acquire_bfloat_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot842[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot842;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic bfloat, ptr addrspace(5) %slot syncscope("cluster") acquire, align 2
+ ret bfloat %retval
+}
+
+define bfloat @acquire_bfloat_local_gpu() {
+; CHECK-LABEL: acquire_bfloat_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot843[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot843;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic bfloat, ptr addrspace(5) %slot syncscope("device") acquire, align 2
+ ret bfloat %retval
+}
+
+define bfloat @acquire_bfloat_local_sys_volatile() {
+; CHECK-LABEL: acquire_bfloat_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot844[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot844;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile bfloat, ptr addrspace(5) %slot syncscope("") acquire, align 2
+ ret bfloat %retval
+}
+
+define bfloat @acquire_bfloat_local_cta_volatile() {
+; CHECK-LABEL: acquire_bfloat_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot845[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot845;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile bfloat, ptr addrspace(5) %slot syncscope("block") acquire, align 2
+ ret bfloat %retval
+}
+
+define bfloat @acquire_bfloat_local_cluster_volatile() {
+; CHECK-LABEL: acquire_bfloat_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot846[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot846;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile bfloat, ptr addrspace(5) %slot syncscope("cluster") acquire, align 2
+ ret bfloat %retval
+}
+
+define bfloat @acquire_bfloat_local_gpu_volatile() {
+; CHECK-LABEL: acquire_bfloat_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot847[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot847;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile bfloat, ptr addrspace(5) %slot syncscope("device") acquire, align 2
+ ret bfloat %retval
+}
+
+define float @acquire_float_local_sys() {
+; CHECK-LABEL: acquire_float_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot848[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot848;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic float, ptr addrspace(5) %slot syncscope("") acquire, align 4
+ ret float %retval
+}
+
+define float @acquire_float_local_cta() {
+; CHECK-LABEL: acquire_float_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot849[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot849;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic float, ptr addrspace(5) %slot syncscope("block") acquire, align 4
+ ret float %retval
+}
+
+define float @acquire_float_local_cluster() {
+; CHECK-LABEL: acquire_float_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot850[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot850;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic float, ptr addrspace(5) %slot syncscope("cluster") acquire, align 4
+ ret float %retval
+}
+
+define float @acquire_float_local_gpu() {
+; CHECK-LABEL: acquire_float_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot851[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot851;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic float, ptr addrspace(5) %slot syncscope("device") acquire, align 4
+ ret float %retval
+}
+
+define float @acquire_float_local_sys_volatile() {
+; CHECK-LABEL: acquire_float_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot852[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot852;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile float, ptr addrspace(5) %slot syncscope("") acquire, align 4
+ ret float %retval
+}
+
+define float @acquire_float_local_cta_volatile() {
+; CHECK-LABEL: acquire_float_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot853[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot853;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile float, ptr addrspace(5) %slot syncscope("block") acquire, align 4
+ ret float %retval
+}
+
+define float @acquire_float_local_cluster_volatile() {
+; CHECK-LABEL: acquire_float_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot854[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot854;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile float, ptr addrspace(5) %slot syncscope("cluster") acquire, align 4
+ ret float %retval
+}
+
+define float @acquire_float_local_gpu_volatile() {
+; CHECK-LABEL: acquire_float_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot855[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot855;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile float, ptr addrspace(5) %slot syncscope("device") acquire, align 4
+ ret float %retval
+}
+
+define double @acquire_double_local_sys() {
+; CHECK-LABEL: acquire_double_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot856[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot856;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic double, ptr addrspace(5) %slot syncscope("") acquire, align 8
+ ret double %retval
+}
+
+define double @acquire_double_local_cta() {
+; CHECK-LABEL: acquire_double_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot857[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot857;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic double, ptr addrspace(5) %slot syncscope("block") acquire, align 8
+ ret double %retval
+}
+
+define double @acquire_double_local_cluster() {
+; CHECK-LABEL: acquire_double_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot858[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot858;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic double, ptr addrspace(5) %slot syncscope("cluster") acquire, align 8
+ ret double %retval
+}
+
+define double @acquire_double_local_gpu() {
+; CHECK-LABEL: acquire_double_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot859[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot859;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic double, ptr addrspace(5) %slot syncscope("device") acquire, align 8
+ ret double %retval
+}
+
+define double @acquire_double_local_sys_volatile() {
+; CHECK-LABEL: acquire_double_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot860[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot860;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile double, ptr addrspace(5) %slot syncscope("") acquire, align 8
+ ret double %retval
+}
+
+define double @acquire_double_local_cta_volatile() {
+; CHECK-LABEL: acquire_double_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot861[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot861;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile double, ptr addrspace(5) %slot syncscope("block") acquire, align 8
+ ret double %retval
+}
+
+define double @acquire_double_local_cluster_volatile() {
+; CHECK-LABEL: acquire_double_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot862[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot862;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile double, ptr addrspace(5) %slot syncscope("cluster") acquire, align 8
+ ret double %retval
+}
+
+define double @acquire_double_local_gpu_volatile() {
+; CHECK-LABEL: acquire_double_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot863[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot863;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile double, ptr addrspace(5) %slot syncscope("device") acquire, align 8
+ ret double %retval
+}
+
+define i8 @seq_cst_i8_generic_sys(ptr %addr) {
+; CHECK-LABEL: seq_cst_i8_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_generic_sys_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.volatile.b8 %rs1, [%rd1];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr %addr syncscope("") seq_cst, align 1
+ ret i8 %retval
+}
+
+define i8 @seq_cst_i8_generic_cta(ptr %addr) {
+; CHECK-LABEL: seq_cst_i8_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_generic_cta_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.volatile.b8 %rs1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr %addr syncscope("block") seq_cst, align 1
+ ret i8 %retval
+}
+
+define i8 @seq_cst_i8_generic_cluster(ptr %addr) {
+; CHECK-LABEL: seq_cst_i8_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_generic_cluster_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.volatile.b8 %rs1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr %addr syncscope("cluster") seq_cst, align 1
+ ret i8 %retval
+}
+
+define i8 @seq_cst_i8_generic_gpu(ptr %addr) {
+; CHECK-LABEL: seq_cst_i8_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_generic_gpu_param_0];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: ld.volatile.b8 %rs1, [%rd1];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr %addr syncscope("device") seq_cst, align 1
+ ret i8 %retval
+}
+
+define i8 @seq_cst_i8_generic_sys_volatile(ptr %addr) {
+; CHECK-LABEL: seq_cst_i8_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_generic_sys_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.volatile.b8 %rs1, [%rd1];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr %addr syncscope("") seq_cst, align 1
+ ret i8 %retval
+}
+
+define i8 @seq_cst_i8_generic_cta_volatile(ptr %addr) {
+; CHECK-LABEL: seq_cst_i8_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_generic_cta_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.volatile.b8 %rs1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr %addr syncscope("block") seq_cst, align 1
+ ret i8 %retval
+}
+
+define i8 @seq_cst_i8_generic_cluster_volatile(ptr %addr) {
+; CHECK-LABEL: seq_cst_i8_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_generic_cluster_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.volatile.b8 %rs1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr %addr syncscope("cluster") seq_cst, align 1
+ ret i8 %retval
+}
+
+define i8 @seq_cst_i8_generic_gpu_volatile(ptr %addr) {
+; CHECK-LABEL: seq_cst_i8_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_generic_gpu_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.volatile.b8 %rs1, [%rd1];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr %addr syncscope("device") seq_cst, align 1
+ ret i8 %retval
+}
+
+define i16 @seq_cst_i16_generic_sys(ptr %addr) {
+; CHECK-LABEL: seq_cst_i16_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_generic_sys_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr %addr syncscope("") seq_cst, align 2
+ ret i16 %retval
+}
+
+define i16 @seq_cst_i16_generic_cta(ptr %addr) {
+; CHECK-LABEL: seq_cst_i16_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_generic_cta_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr %addr syncscope("block") seq_cst, align 2
+ ret i16 %retval
+}
+
+define i16 @seq_cst_i16_generic_cluster(ptr %addr) {
+; CHECK-LABEL: seq_cst_i16_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_generic_cluster_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr %addr syncscope("cluster") seq_cst, align 2
+ ret i16 %retval
+}
+
+define i16 @seq_cst_i16_generic_gpu(ptr %addr) {
+; CHECK-LABEL: seq_cst_i16_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_generic_gpu_param_0];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr %addr syncscope("device") seq_cst, align 2
+ ret i16 %retval
+}
+
+define i16 @seq_cst_i16_generic_sys_volatile(ptr %addr) {
+; CHECK-LABEL: seq_cst_i16_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_generic_sys_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr %addr syncscope("") seq_cst, align 2
+ ret i16 %retval
+}
+
+define i16 @seq_cst_i16_generic_cta_volatile(ptr %addr) {
+; CHECK-LABEL: seq_cst_i16_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_generic_cta_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr %addr syncscope("block") seq_cst, align 2
+ ret i16 %retval
+}
+
+define i16 @seq_cst_i16_generic_cluster_volatile(ptr %addr) {
+; CHECK-LABEL: seq_cst_i16_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_generic_cluster_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr %addr syncscope("cluster") seq_cst, align 2
+ ret i16 %retval
+}
+
+define i16 @seq_cst_i16_generic_gpu_volatile(ptr %addr) {
+; CHECK-LABEL: seq_cst_i16_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_generic_gpu_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr %addr syncscope("device") seq_cst, align 2
+ ret i16 %retval
+}
+
+define i32 @seq_cst_i32_generic_sys(ptr %addr) {
+; CHECK-LABEL: seq_cst_i32_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_generic_sys_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.volatile.b32 %r1, [%rd1];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr %addr syncscope("") seq_cst, align 4
+ ret i32 %retval
+}
+
+define i32 @seq_cst_i32_generic_cta(ptr %addr) {
+; CHECK-LABEL: seq_cst_i32_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_generic_cta_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.volatile.b32 %r1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr %addr syncscope("block") seq_cst, align 4
+ ret i32 %retval
+}
+
+define i32 @seq_cst_i32_generic_cluster(ptr %addr) {
+; CHECK-LABEL: seq_cst_i32_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_generic_cluster_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.volatile.b32 %r1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr %addr syncscope("cluster") seq_cst, align 4
+ ret i32 %retval
+}
+
+define i32 @seq_cst_i32_generic_gpu(ptr %addr) {
+; CHECK-LABEL: seq_cst_i32_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_generic_gpu_param_0];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: ld.volatile.b32 %r1, [%rd1];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr %addr syncscope("device") seq_cst, align 4
+ ret i32 %retval
+}
+
+define i32 @seq_cst_i32_generic_sys_volatile(ptr %addr) {
+; CHECK-LABEL: seq_cst_i32_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_generic_sys_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.volatile.b32 %r1, [%rd1];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr %addr syncscope("") seq_cst, align 4
+ ret i32 %retval
+}
+
+define i32 @seq_cst_i32_generic_cta_volatile(ptr %addr) {
+; CHECK-LABEL: seq_cst_i32_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_generic_cta_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.volatile.b32 %r1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr %addr syncscope("block") seq_cst, align 4
+ ret i32 %retval
+}
+
+define i32 @seq_cst_i32_generic_cluster_volatile(ptr %addr) {
+; CHECK-LABEL: seq_cst_i32_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_generic_cluster_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.volatile.b32 %r1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr %addr syncscope("cluster") seq_cst, align 4
+ ret i32 %retval
+}
+
+define i32 @seq_cst_i32_generic_gpu_volatile(ptr %addr) {
+; CHECK-LABEL: seq_cst_i32_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_generic_gpu_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.volatile.b32 %r1, [%rd1];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr %addr syncscope("device") seq_cst, align 4
+ ret i32 %retval
+}
+
+define i64 @seq_cst_i64_generic_sys(ptr %addr) {
+; CHECK-LABEL: seq_cst_i64_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_generic_sys_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.volatile.b64 %rd2, [%rd1];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr %addr syncscope("") seq_cst, align 8
+ ret i64 %retval
+}
+
+define i64 @seq_cst_i64_generic_cta(ptr %addr) {
+; CHECK-LABEL: seq_cst_i64_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_generic_cta_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.volatile.b64 %rd2, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr %addr syncscope("block") seq_cst, align 8
+ ret i64 %retval
+}
+
+define i64 @seq_cst_i64_generic_cluster(ptr %addr) {
+; CHECK-LABEL: seq_cst_i64_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_generic_cluster_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.volatile.b64 %rd2, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr %addr syncscope("cluster") seq_cst, align 8
+ ret i64 %retval
+}
+
+define i64 @seq_cst_i64_generic_gpu(ptr %addr) {
+; CHECK-LABEL: seq_cst_i64_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_generic_gpu_param_0];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: ld.volatile.b64 %rd2, [%rd1];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr %addr syncscope("device") seq_cst, align 8
+ ret i64 %retval
+}
+
+define i64 @seq_cst_i64_generic_sys_volatile(ptr %addr) {
+; CHECK-LABEL: seq_cst_i64_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_generic_sys_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.volatile.b64 %rd2, [%rd1];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr %addr syncscope("") seq_cst, align 8
+ ret i64 %retval
+}
+
+define i64 @seq_cst_i64_generic_cta_volatile(ptr %addr) {
+; CHECK-LABEL: seq_cst_i64_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_generic_cta_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.volatile.b64 %rd2, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr %addr syncscope("block") seq_cst, align 8
+ ret i64 %retval
+}
+
+define i64 @seq_cst_i64_generic_cluster_volatile(ptr %addr) {
+; CHECK-LABEL: seq_cst_i64_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_generic_cluster_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.volatile.b64 %rd2, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr %addr syncscope("cluster") seq_cst, align 8
+ ret i64 %retval
+}
+
+define i64 @seq_cst_i64_generic_gpu_volatile(ptr %addr) {
+; CHECK-LABEL: seq_cst_i64_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_generic_gpu_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.volatile.b64 %rd2, [%rd1];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr %addr syncscope("device") seq_cst, align 8
+ ret i64 %retval
+}
+
+define half @seq_cst_half_generic_sys(ptr %addr) {
+; CHECK-LABEL: seq_cst_half_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_generic_sys_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr %addr syncscope("") seq_cst, align 2
+ ret half %retval
+}
+
+define half @seq_cst_half_generic_cta(ptr %addr) {
+; CHECK-LABEL: seq_cst_half_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_generic_cta_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr %addr syncscope("block") seq_cst, align 2
+ ret half %retval
+}
+
+define half @seq_cst_half_generic_cluster(ptr %addr) {
+; CHECK-LABEL: seq_cst_half_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_generic_cluster_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr %addr syncscope("cluster") seq_cst, align 2
+ ret half %retval
+}
+
+define half @seq_cst_half_generic_gpu(ptr %addr) {
+; CHECK-LABEL: seq_cst_half_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_generic_gpu_param_0];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr %addr syncscope("device") seq_cst, align 2
+ ret half %retval
+}
+
+define half @seq_cst_half_generic_sys_volatile(ptr %addr) {
+; CHECK-LABEL: seq_cst_half_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_generic_sys_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr %addr syncscope("") seq_cst, align 2
+ ret half %retval
+}
+
+define half @seq_cst_half_generic_cta_volatile(ptr %addr) {
+; CHECK-LABEL: seq_cst_half_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_generic_cta_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr %addr syncscope("block") seq_cst, align 2
+ ret half %retval
+}
+
+define half @seq_cst_half_generic_cluster_volatile(ptr %addr) {
+; CHECK-LABEL: seq_cst_half_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_generic_cluster_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr %addr syncscope("cluster") seq_cst, align 2
+ ret half %retval
+}
+
+define half @seq_cst_half_generic_gpu_volatile(ptr %addr) {
+; CHECK-LABEL: seq_cst_half_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_generic_gpu_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr %addr syncscope("device") seq_cst, align 2
+ ret half %retval
+}
+
+define bfloat @seq_cst_bfloat_generic_sys(ptr %addr) {
+; CHECK-LABEL: seq_cst_bfloat_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_generic_sys_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr %addr syncscope("") seq_cst, align 2
+ ret bfloat %retval
+}
+
+define bfloat @seq_cst_bfloat_generic_cta(ptr %addr) {
+; CHECK-LABEL: seq_cst_bfloat_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_generic_cta_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr %addr syncscope("block") seq_cst, align 2
+ ret bfloat %retval
+}
+
+define bfloat @seq_cst_bfloat_generic_cluster(ptr %addr) {
+; CHECK-LABEL: seq_cst_bfloat_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_generic_cluster_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr %addr syncscope("cluster") seq_cst, align 2
+ ret bfloat %retval
+}
+
+define bfloat @seq_cst_bfloat_generic_gpu(ptr %addr) {
+; CHECK-LABEL: seq_cst_bfloat_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_generic_gpu_param_0];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr %addr syncscope("device") seq_cst, align 2
+ ret bfloat %retval
+}
+
+define bfloat @seq_cst_bfloat_generic_sys_volatile(ptr %addr) {
+; CHECK-LABEL: seq_cst_bfloat_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_generic_sys_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr %addr syncscope("") seq_cst, align 2
+ ret bfloat %retval
+}
+
+define bfloat @seq_cst_bfloat_generic_cta_volatile(ptr %addr) {
+; CHECK-LABEL: seq_cst_bfloat_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_generic_cta_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr %addr syncscope("block") seq_cst, align 2
+ ret bfloat %retval
+}
+
+define bfloat @seq_cst_bfloat_generic_cluster_volatile(ptr %addr) {
+; CHECK-LABEL: seq_cst_bfloat_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_generic_cluster_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr %addr syncscope("cluster") seq_cst, align 2
+ ret bfloat %retval
+}
+
+define bfloat @seq_cst_bfloat_generic_gpu_volatile(ptr %addr) {
+; CHECK-LABEL: seq_cst_bfloat_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_generic_gpu_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr %addr syncscope("device") seq_cst, align 2
+ ret bfloat %retval
+}
+
+define float @seq_cst_float_generic_sys(ptr %addr) {
+; CHECK-LABEL: seq_cst_float_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_generic_sys_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.volatile.b32 %r1, [%rd1];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr %addr syncscope("") seq_cst, align 4
+ ret float %retval
+}
+
+define float @seq_cst_float_generic_cta(ptr %addr) {
+; CHECK-LABEL: seq_cst_float_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_generic_cta_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.volatile.b32 %r1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr %addr syncscope("block") seq_cst, align 4
+ ret float %retval
+}
+
+define float @seq_cst_float_generic_cluster(ptr %addr) {
+; CHECK-LABEL: seq_cst_float_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_generic_cluster_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.volatile.b32 %r1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr %addr syncscope("cluster") seq_cst, align 4
+ ret float %retval
+}
+
+define float @seq_cst_float_generic_gpu(ptr %addr) {
+; CHECK-LABEL: seq_cst_float_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_generic_gpu_param_0];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: ld.volatile.b32 %r1, [%rd1];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr %addr syncscope("device") seq_cst, align 4
+ ret float %retval
+}
+
+define float @seq_cst_float_generic_sys_volatile(ptr %addr) {
+; CHECK-LABEL: seq_cst_float_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_generic_sys_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.volatile.b32 %r1, [%rd1];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr %addr syncscope("") seq_cst, align 4
+ ret float %retval
+}
+
+define float @seq_cst_float_generic_cta_volatile(ptr %addr) {
+; CHECK-LABEL: seq_cst_float_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_generic_cta_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.volatile.b32 %r1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr %addr syncscope("block") seq_cst, align 4
+ ret float %retval
+}
+
+define float @seq_cst_float_generic_cluster_volatile(ptr %addr) {
+; CHECK-LABEL: seq_cst_float_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_generic_cluster_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.volatile.b32 %r1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr %addr syncscope("cluster") seq_cst, align 4
+ ret float %retval
+}
+
+define float @seq_cst_float_generic_gpu_volatile(ptr %addr) {
+; CHECK-LABEL: seq_cst_float_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_generic_gpu_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.volatile.b32 %r1, [%rd1];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr %addr syncscope("device") seq_cst, align 4
+ ret float %retval
+}
+
+define double @seq_cst_double_generic_sys(ptr %addr) {
+; CHECK-LABEL: seq_cst_double_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_generic_sys_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.volatile.b64 %rd2, [%rd1];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr %addr syncscope("") seq_cst, align 8
+ ret double %retval
+}
+
+define double @seq_cst_double_generic_cta(ptr %addr) {
+; CHECK-LABEL: seq_cst_double_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_generic_cta_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.volatile.b64 %rd2, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr %addr syncscope("block") seq_cst, align 8
+ ret double %retval
+}
+
+define double @seq_cst_double_generic_cluster(ptr %addr) {
+; CHECK-LABEL: seq_cst_double_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_generic_cluster_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.volatile.b64 %rd2, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr %addr syncscope("cluster") seq_cst, align 8
+ ret double %retval
+}
+
+define double @seq_cst_double_generic_gpu(ptr %addr) {
+; CHECK-LABEL: seq_cst_double_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_generic_gpu_param_0];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: ld.volatile.b64 %rd2, [%rd1];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr %addr syncscope("device") seq_cst, align 8
+ ret double %retval
+}
+
+define double @seq_cst_double_generic_sys_volatile(ptr %addr) {
+; CHECK-LABEL: seq_cst_double_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_generic_sys_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.volatile.b64 %rd2, [%rd1];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr %addr syncscope("") seq_cst, align 8
+ ret double %retval
+}
+
+define double @seq_cst_double_generic_cta_volatile(ptr %addr) {
+; CHECK-LABEL: seq_cst_double_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_generic_cta_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.volatile.b64 %rd2, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr %addr syncscope("block") seq_cst, align 8
+ ret double %retval
+}
+
+define double @seq_cst_double_generic_cluster_volatile(ptr %addr) {
+; CHECK-LABEL: seq_cst_double_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_generic_cluster_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.volatile.b64 %rd2, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr %addr syncscope("cluster") seq_cst, align 8
+ ret double %retval
+}
+
+define double @seq_cst_double_generic_gpu_volatile(ptr %addr) {
+; CHECK-LABEL: seq_cst_double_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_generic_gpu_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.volatile.b64 %rd2, [%rd1];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr %addr syncscope("device") seq_cst, align 8
+ ret double %retval
+}
+
+define i8 @seq_cst_i8_global_sys(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i8_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_global_sys_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.volatile.global.b8 %rs1, [%rd1];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr addrspace(1) %addr syncscope("") seq_cst, align 1
+ ret i8 %retval
+}
+
+define i8 @seq_cst_i8_global_cta(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i8_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_global_cta_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.volatile.global.b8 %rs1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr addrspace(1) %addr syncscope("block") seq_cst, align 1
+ ret i8 %retval
+}
+
+define i8 @seq_cst_i8_global_cluster(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i8_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_global_cluster_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.volatile.global.b8 %rs1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr addrspace(1) %addr syncscope("cluster") seq_cst, align 1
+ ret i8 %retval
+}
+
+define i8 @seq_cst_i8_global_gpu(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i8_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_global_gpu_param_0];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: ld.volatile.global.b8 %rs1, [%rd1];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr addrspace(1) %addr syncscope("device") seq_cst, align 1
+ ret i8 %retval
+}
+
+define i8 @seq_cst_i8_global_sys_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i8_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_global_sys_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.volatile.global.b8 %rs1, [%rd1];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr addrspace(1) %addr syncscope("") seq_cst, align 1
+ ret i8 %retval
+}
+
+define i8 @seq_cst_i8_global_cta_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i8_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_global_cta_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.volatile.global.b8 %rs1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr addrspace(1) %addr syncscope("block") seq_cst, align 1
+ ret i8 %retval
+}
+
+define i8 @seq_cst_i8_global_cluster_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i8_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_global_cluster_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.volatile.global.b8 %rs1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr addrspace(1) %addr syncscope("cluster") seq_cst, align 1
+ ret i8 %retval
+}
+
+define i8 @seq_cst_i8_global_gpu_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i8_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_global_gpu_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.volatile.global.b8 %rs1, [%rd1];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr addrspace(1) %addr syncscope("device") seq_cst, align 1
+ ret i8 %retval
+}
+
+define i16 @seq_cst_i16_global_sys(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i16_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_global_sys_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr addrspace(1) %addr syncscope("") seq_cst, align 2
+ ret i16 %retval
+}
+
+define i16 @seq_cst_i16_global_cta(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i16_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_global_cta_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr addrspace(1) %addr syncscope("block") seq_cst, align 2
+ ret i16 %retval
+}
+
+define i16 @seq_cst_i16_global_cluster(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i16_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_global_cluster_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr addrspace(1) %addr syncscope("cluster") seq_cst, align 2
+ ret i16 %retval
+}
+
+define i16 @seq_cst_i16_global_gpu(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i16_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_global_gpu_param_0];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr addrspace(1) %addr syncscope("device") seq_cst, align 2
+ ret i16 %retval
+}
+
+define i16 @seq_cst_i16_global_sys_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i16_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_global_sys_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr addrspace(1) %addr syncscope("") seq_cst, align 2
+ ret i16 %retval
+}
+
+define i16 @seq_cst_i16_global_cta_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i16_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_global_cta_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr addrspace(1) %addr syncscope("block") seq_cst, align 2
+ ret i16 %retval
+}
+
+define i16 @seq_cst_i16_global_cluster_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i16_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_global_cluster_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr addrspace(1) %addr syncscope("cluster") seq_cst, align 2
+ ret i16 %retval
+}
+
+define i16 @seq_cst_i16_global_gpu_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i16_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_global_gpu_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr addrspace(1) %addr syncscope("device") seq_cst, align 2
+ ret i16 %retval
+}
+
+define i32 @seq_cst_i32_global_sys(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i32_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_global_sys_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.volatile.global.b32 %r1, [%rd1];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr addrspace(1) %addr syncscope("") seq_cst, align 4
+ ret i32 %retval
+}
+
+define i32 @seq_cst_i32_global_cta(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i32_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_global_cta_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.volatile.global.b32 %r1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr addrspace(1) %addr syncscope("block") seq_cst, align 4
+ ret i32 %retval
+}
+
+define i32 @seq_cst_i32_global_cluster(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i32_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_global_cluster_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.volatile.global.b32 %r1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr addrspace(1) %addr syncscope("cluster") seq_cst, align 4
+ ret i32 %retval
+}
+
+define i32 @seq_cst_i32_global_gpu(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i32_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_global_gpu_param_0];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: ld.volatile.global.b32 %r1, [%rd1];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr addrspace(1) %addr syncscope("device") seq_cst, align 4
+ ret i32 %retval
+}
+
+define i32 @seq_cst_i32_global_sys_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i32_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_global_sys_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.volatile.global.b32 %r1, [%rd1];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr addrspace(1) %addr syncscope("") seq_cst, align 4
+ ret i32 %retval
+}
+
+define i32 @seq_cst_i32_global_cta_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i32_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_global_cta_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.volatile.global.b32 %r1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr addrspace(1) %addr syncscope("block") seq_cst, align 4
+ ret i32 %retval
+}
+
+define i32 @seq_cst_i32_global_cluster_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i32_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_global_cluster_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.volatile.global.b32 %r1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr addrspace(1) %addr syncscope("cluster") seq_cst, align 4
+ ret i32 %retval
+}
+
+define i32 @seq_cst_i32_global_gpu_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i32_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_global_gpu_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.volatile.global.b32 %r1, [%rd1];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr addrspace(1) %addr syncscope("device") seq_cst, align 4
+ ret i32 %retval
+}
+
+define i64 @seq_cst_i64_global_sys(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i64_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_global_sys_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr addrspace(1) %addr syncscope("") seq_cst, align 8
+ ret i64 %retval
+}
+
+define i64 @seq_cst_i64_global_cta(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i64_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_global_cta_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr addrspace(1) %addr syncscope("block") seq_cst, align 8
+ ret i64 %retval
+}
+
+define i64 @seq_cst_i64_global_cluster(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i64_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_global_cluster_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr addrspace(1) %addr syncscope("cluster") seq_cst, align 8
+ ret i64 %retval
+}
+
+define i64 @seq_cst_i64_global_gpu(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i64_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_global_gpu_param_0];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr addrspace(1) %addr syncscope("device") seq_cst, align 8
+ ret i64 %retval
+}
+
+define i64 @seq_cst_i64_global_sys_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i64_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_global_sys_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr addrspace(1) %addr syncscope("") seq_cst, align 8
+ ret i64 %retval
+}
+
+define i64 @seq_cst_i64_global_cta_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i64_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_global_cta_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr addrspace(1) %addr syncscope("block") seq_cst, align 8
+ ret i64 %retval
+}
+
+define i64 @seq_cst_i64_global_cluster_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i64_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_global_cluster_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr addrspace(1) %addr syncscope("cluster") seq_cst, align 8
+ ret i64 %retval
+}
+
+define i64 @seq_cst_i64_global_gpu_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i64_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_global_gpu_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr addrspace(1) %addr syncscope("device") seq_cst, align 8
+ ret i64 %retval
+}
+
+define half @seq_cst_half_global_sys(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_half_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_global_sys_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr addrspace(1) %addr syncscope("") seq_cst, align 2
+ ret half %retval
+}
+
+define half @seq_cst_half_global_cta(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_half_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_global_cta_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr addrspace(1) %addr syncscope("block") seq_cst, align 2
+ ret half %retval
+}
+
+define half @seq_cst_half_global_cluster(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_half_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_global_cluster_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr addrspace(1) %addr syncscope("cluster") seq_cst, align 2
+ ret half %retval
+}
+
+define half @seq_cst_half_global_gpu(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_half_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_global_gpu_param_0];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr addrspace(1) %addr syncscope("device") seq_cst, align 2
+ ret half %retval
+}
+
+define half @seq_cst_half_global_sys_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_half_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_global_sys_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr addrspace(1) %addr syncscope("") seq_cst, align 2
+ ret half %retval
+}
+
+define half @seq_cst_half_global_cta_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_half_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_global_cta_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr addrspace(1) %addr syncscope("block") seq_cst, align 2
+ ret half %retval
+}
+
+define half @seq_cst_half_global_cluster_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_half_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_global_cluster_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr addrspace(1) %addr syncscope("cluster") seq_cst, align 2
+ ret half %retval
+}
+
+define half @seq_cst_half_global_gpu_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_half_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_global_gpu_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr addrspace(1) %addr syncscope("device") seq_cst, align 2
+ ret half %retval
+}
+
+define bfloat @seq_cst_bfloat_global_sys(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_bfloat_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_global_sys_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr addrspace(1) %addr syncscope("") seq_cst, align 2
+ ret bfloat %retval
+}
+
+define bfloat @seq_cst_bfloat_global_cta(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_bfloat_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_global_cta_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr addrspace(1) %addr syncscope("block") seq_cst, align 2
+ ret bfloat %retval
+}
+
+define bfloat @seq_cst_bfloat_global_cluster(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_bfloat_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_global_cluster_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr addrspace(1) %addr syncscope("cluster") seq_cst, align 2
+ ret bfloat %retval
+}
+
+define bfloat @seq_cst_bfloat_global_gpu(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_bfloat_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_global_gpu_param_0];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr addrspace(1) %addr syncscope("device") seq_cst, align 2
+ ret bfloat %retval
+}
+
+define bfloat @seq_cst_bfloat_global_sys_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_bfloat_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_global_sys_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr addrspace(1) %addr syncscope("") seq_cst, align 2
+ ret bfloat %retval
+}
+
+define bfloat @seq_cst_bfloat_global_cta_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_bfloat_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_global_cta_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr addrspace(1) %addr syncscope("block") seq_cst, align 2
+ ret bfloat %retval
+}
+
+define bfloat @seq_cst_bfloat_global_cluster_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_bfloat_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_global_cluster_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr addrspace(1) %addr syncscope("cluster") seq_cst, align 2
+ ret bfloat %retval
+}
+
+define bfloat @seq_cst_bfloat_global_gpu_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_bfloat_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_global_gpu_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr addrspace(1) %addr syncscope("device") seq_cst, align 2
+ ret bfloat %retval
+}
+
+define float @seq_cst_float_global_sys(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_float_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_global_sys_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.volatile.global.b32 %r1, [%rd1];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr addrspace(1) %addr syncscope("") seq_cst, align 4
+ ret float %retval
+}
+
+define float @seq_cst_float_global_cta(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_float_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_global_cta_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.volatile.global.b32 %r1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr addrspace(1) %addr syncscope("block") seq_cst, align 4
+ ret float %retval
+}
+
+define float @seq_cst_float_global_cluster(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_float_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_global_cluster_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.volatile.global.b32 %r1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr addrspace(1) %addr syncscope("cluster") seq_cst, align 4
+ ret float %retval
+}
+
+define float @seq_cst_float_global_gpu(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_float_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_global_gpu_param_0];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: ld.volatile.global.b32 %r1, [%rd1];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr addrspace(1) %addr syncscope("device") seq_cst, align 4
+ ret float %retval
+}
+
+define float @seq_cst_float_global_sys_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_float_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_global_sys_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.volatile.global.b32 %r1, [%rd1];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr addrspace(1) %addr syncscope("") seq_cst, align 4
+ ret float %retval
+}
+
+define float @seq_cst_float_global_cta_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_float_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_global_cta_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.volatile.global.b32 %r1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr addrspace(1) %addr syncscope("block") seq_cst, align 4
+ ret float %retval
+}
+
+define float @seq_cst_float_global_cluster_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_float_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_global_cluster_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.volatile.global.b32 %r1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr addrspace(1) %addr syncscope("cluster") seq_cst, align 4
+ ret float %retval
+}
+
+define float @seq_cst_float_global_gpu_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_float_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_global_gpu_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.volatile.global.b32 %r1, [%rd1];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr addrspace(1) %addr syncscope("device") seq_cst, align 4
+ ret float %retval
+}
+
+define double @seq_cst_double_global_sys(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_double_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_global_sys_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr addrspace(1) %addr syncscope("") seq_cst, align 8
+ ret double %retval
+}
+
+define double @seq_cst_double_global_cta(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_double_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_global_cta_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr addrspace(1) %addr syncscope("block") seq_cst, align 8
+ ret double %retval
+}
+
+define double @seq_cst_double_global_cluster(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_double_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_global_cluster_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr addrspace(1) %addr syncscope("cluster") seq_cst, align 8
+ ret double %retval
+}
+
+define double @seq_cst_double_global_gpu(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_double_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_global_gpu_param_0];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr addrspace(1) %addr syncscope("device") seq_cst, align 8
+ ret double %retval
+}
+
+define double @seq_cst_double_global_sys_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_double_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_global_sys_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr addrspace(1) %addr syncscope("") seq_cst, align 8
+ ret double %retval
+}
+
+define double @seq_cst_double_global_cta_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_double_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_global_cta_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr addrspace(1) %addr syncscope("block") seq_cst, align 8
+ ret double %retval
+}
+
+define double @seq_cst_double_global_cluster_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_double_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_global_cluster_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr addrspace(1) %addr syncscope("cluster") seq_cst, align 8
+ ret double %retval
+}
+
+define double @seq_cst_double_global_gpu_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_double_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_global_gpu_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr addrspace(1) %addr syncscope("device") seq_cst, align 8
+ ret double %retval
+}
+
+define i8 @seq_cst_i8_shared_sys(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i8_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_shared_sys_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.volatile.shared.b8 %rs1, [%rd1];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr addrspace(3) %addr syncscope("") seq_cst, align 1
+ ret i8 %retval
+}
+
+define i8 @seq_cst_i8_shared_cta(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i8_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_shared_cta_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.volatile.shared.b8 %rs1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr addrspace(3) %addr syncscope("block") seq_cst, align 1
+ ret i8 %retval
+}
+
+define i8 @seq_cst_i8_shared_cluster(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i8_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_shared_cluster_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.volatile.shared.b8 %rs1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr addrspace(3) %addr syncscope("cluster") seq_cst, align 1
+ ret i8 %retval
+}
+
+define i8 @seq_cst_i8_shared_gpu(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i8_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_shared_gpu_param_0];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: ld.volatile.shared.b8 %rs1, [%rd1];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr addrspace(3) %addr syncscope("device") seq_cst, align 1
+ ret i8 %retval
+}
+
+define i8 @seq_cst_i8_shared_sys_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i8_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_shared_sys_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.volatile.shared.b8 %rs1, [%rd1];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr addrspace(3) %addr syncscope("") seq_cst, align 1
+ ret i8 %retval
+}
+
+define i8 @seq_cst_i8_shared_cta_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i8_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_shared_cta_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.volatile.shared.b8 %rs1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr addrspace(3) %addr syncscope("block") seq_cst, align 1
+ ret i8 %retval
+}
+
+define i8 @seq_cst_i8_shared_cluster_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i8_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_shared_cluster_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.volatile.shared.b8 %rs1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr addrspace(3) %addr syncscope("cluster") seq_cst, align 1
+ ret i8 %retval
+}
+
+define i8 @seq_cst_i8_shared_gpu_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i8_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_shared_gpu_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.volatile.shared.b8 %rs1, [%rd1];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr addrspace(3) %addr syncscope("device") seq_cst, align 1
+ ret i8 %retval
+}
+
+define i16 @seq_cst_i16_shared_sys(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i16_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_shared_sys_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr addrspace(3) %addr syncscope("") seq_cst, align 2
+ ret i16 %retval
+}
+
+define i16 @seq_cst_i16_shared_cta(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i16_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_shared_cta_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr addrspace(3) %addr syncscope("block") seq_cst, align 2
+ ret i16 %retval
+}
+
+define i16 @seq_cst_i16_shared_cluster(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i16_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_shared_cluster_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr addrspace(3) %addr syncscope("cluster") seq_cst, align 2
+ ret i16 %retval
+}
+
+define i16 @seq_cst_i16_shared_gpu(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i16_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_shared_gpu_param_0];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr addrspace(3) %addr syncscope("device") seq_cst, align 2
+ ret i16 %retval
+}
+
+define i16 @seq_cst_i16_shared_sys_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i16_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_shared_sys_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr addrspace(3) %addr syncscope("") seq_cst, align 2
+ ret i16 %retval
+}
+
+define i16 @seq_cst_i16_shared_cta_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i16_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_shared_cta_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr addrspace(3) %addr syncscope("block") seq_cst, align 2
+ ret i16 %retval
+}
+
+define i16 @seq_cst_i16_shared_cluster_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i16_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_shared_cluster_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr addrspace(3) %addr syncscope("cluster") seq_cst, align 2
+ ret i16 %retval
+}
+
+define i16 @seq_cst_i16_shared_gpu_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i16_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_shared_gpu_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr addrspace(3) %addr syncscope("device") seq_cst, align 2
+ ret i16 %retval
+}
+
+define i32 @seq_cst_i32_shared_sys(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i32_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_shared_sys_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.volatile.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr addrspace(3) %addr syncscope("") seq_cst, align 4
+ ret i32 %retval
+}
+
+define i32 @seq_cst_i32_shared_cta(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i32_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_shared_cta_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.volatile.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr addrspace(3) %addr syncscope("block") seq_cst, align 4
+ ret i32 %retval
+}
+
+define i32 @seq_cst_i32_shared_cluster(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i32_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_shared_cluster_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.volatile.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr addrspace(3) %addr syncscope("cluster") seq_cst, align 4
+ ret i32 %retval
+}
+
+define i32 @seq_cst_i32_shared_gpu(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i32_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_shared_gpu_param_0];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: ld.volatile.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr addrspace(3) %addr syncscope("device") seq_cst, align 4
+ ret i32 %retval
+}
+
+define i32 @seq_cst_i32_shared_sys_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i32_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_shared_sys_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.volatile.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr addrspace(3) %addr syncscope("") seq_cst, align 4
+ ret i32 %retval
+}
+
+define i32 @seq_cst_i32_shared_cta_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i32_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_shared_cta_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.volatile.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr addrspace(3) %addr syncscope("block") seq_cst, align 4
+ ret i32 %retval
+}
+
+define i32 @seq_cst_i32_shared_cluster_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i32_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_shared_cluster_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.volatile.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr addrspace(3) %addr syncscope("cluster") seq_cst, align 4
+ ret i32 %retval
+}
+
+define i32 @seq_cst_i32_shared_gpu_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i32_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_shared_gpu_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.volatile.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr addrspace(3) %addr syncscope("device") seq_cst, align 4
+ ret i32 %retval
+}
+
+define i64 @seq_cst_i64_shared_sys(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i64_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_shared_sys_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.volatile.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr addrspace(3) %addr syncscope("") seq_cst, align 8
+ ret i64 %retval
+}
+
+define i64 @seq_cst_i64_shared_cta(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i64_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_shared_cta_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.volatile.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr addrspace(3) %addr syncscope("block") seq_cst, align 8
+ ret i64 %retval
+}
+
+define i64 @seq_cst_i64_shared_cluster(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i64_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_shared_cluster_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.volatile.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr addrspace(3) %addr syncscope("cluster") seq_cst, align 8
+ ret i64 %retval
+}
+
+define i64 @seq_cst_i64_shared_gpu(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i64_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_shared_gpu_param_0];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: ld.volatile.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr addrspace(3) %addr syncscope("device") seq_cst, align 8
+ ret i64 %retval
+}
+
+define i64 @seq_cst_i64_shared_sys_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i64_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_shared_sys_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.volatile.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr addrspace(3) %addr syncscope("") seq_cst, align 8
+ ret i64 %retval
+}
+
+define i64 @seq_cst_i64_shared_cta_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i64_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_shared_cta_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.volatile.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr addrspace(3) %addr syncscope("block") seq_cst, align 8
+ ret i64 %retval
+}
+
+define i64 @seq_cst_i64_shared_cluster_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i64_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_shared_cluster_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.volatile.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr addrspace(3) %addr syncscope("cluster") seq_cst, align 8
+ ret i64 %retval
+}
+
+define i64 @seq_cst_i64_shared_gpu_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i64_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_shared_gpu_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.volatile.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr addrspace(3) %addr syncscope("device") seq_cst, align 8
+ ret i64 %retval
+}
+
+define half @seq_cst_half_shared_sys(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_half_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_shared_sys_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr addrspace(3) %addr syncscope("") seq_cst, align 2
+ ret half %retval
+}
+
+define half @seq_cst_half_shared_cta(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_half_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_shared_cta_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr addrspace(3) %addr syncscope("block") seq_cst, align 2
+ ret half %retval
+}
+
+define half @seq_cst_half_shared_cluster(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_half_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_shared_cluster_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr addrspace(3) %addr syncscope("cluster") seq_cst, align 2
+ ret half %retval
+}
+
+define half @seq_cst_half_shared_gpu(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_half_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_shared_gpu_param_0];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr addrspace(3) %addr syncscope("device") seq_cst, align 2
+ ret half %retval
+}
+
+define half @seq_cst_half_shared_sys_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_half_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_shared_sys_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr addrspace(3) %addr syncscope("") seq_cst, align 2
+ ret half %retval
+}
+
+define half @seq_cst_half_shared_cta_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_half_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_shared_cta_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr addrspace(3) %addr syncscope("block") seq_cst, align 2
+ ret half %retval
+}
+
+define half @seq_cst_half_shared_cluster_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_half_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_shared_cluster_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr addrspace(3) %addr syncscope("cluster") seq_cst, align 2
+ ret half %retval
+}
+
+define half @seq_cst_half_shared_gpu_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_half_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_shared_gpu_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr addrspace(3) %addr syncscope("device") seq_cst, align 2
+ ret half %retval
+}
+
+define bfloat @seq_cst_bfloat_shared_sys(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_bfloat_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_shared_sys_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr addrspace(3) %addr syncscope("") seq_cst, align 2
+ ret bfloat %retval
+}
+
+define bfloat @seq_cst_bfloat_shared_cta(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_bfloat_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_shared_cta_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr addrspace(3) %addr syncscope("block") seq_cst, align 2
+ ret bfloat %retval
+}
+
+define bfloat @seq_cst_bfloat_shared_cluster(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_bfloat_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_shared_cluster_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr addrspace(3) %addr syncscope("cluster") seq_cst, align 2
+ ret bfloat %retval
+}
+
+define bfloat @seq_cst_bfloat_shared_gpu(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_bfloat_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_shared_gpu_param_0];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr addrspace(3) %addr syncscope("device") seq_cst, align 2
+ ret bfloat %retval
+}
+
+define bfloat @seq_cst_bfloat_shared_sys_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_bfloat_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_shared_sys_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr addrspace(3) %addr syncscope("") seq_cst, align 2
+ ret bfloat %retval
+}
+
+define bfloat @seq_cst_bfloat_shared_cta_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_bfloat_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_shared_cta_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr addrspace(3) %addr syncscope("block") seq_cst, align 2
+ ret bfloat %retval
+}
+
+define bfloat @seq_cst_bfloat_shared_cluster_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_bfloat_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_shared_cluster_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr addrspace(3) %addr syncscope("cluster") seq_cst, align 2
+ ret bfloat %retval
+}
+
+define bfloat @seq_cst_bfloat_shared_gpu_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_bfloat_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_shared_gpu_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr addrspace(3) %addr syncscope("device") seq_cst, align 2
+ ret bfloat %retval
+}
+
+define float @seq_cst_float_shared_sys(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_float_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_shared_sys_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.volatile.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr addrspace(3) %addr syncscope("") seq_cst, align 4
+ ret float %retval
+}
+
+define float @seq_cst_float_shared_cta(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_float_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_shared_cta_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.volatile.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr addrspace(3) %addr syncscope("block") seq_cst, align 4
+ ret float %retval
+}
+
+define float @seq_cst_float_shared_cluster(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_float_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_shared_cluster_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.volatile.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr addrspace(3) %addr syncscope("cluster") seq_cst, align 4
+ ret float %retval
+}
+
+define float @seq_cst_float_shared_gpu(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_float_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_shared_gpu_param_0];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: ld.volatile.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr addrspace(3) %addr syncscope("device") seq_cst, align 4
+ ret float %retval
+}
+
+define float @seq_cst_float_shared_sys_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_float_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_shared_sys_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.volatile.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr addrspace(3) %addr syncscope("") seq_cst, align 4
+ ret float %retval
+}
+
+define float @seq_cst_float_shared_cta_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_float_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_shared_cta_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.volatile.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr addrspace(3) %addr syncscope("block") seq_cst, align 4
+ ret float %retval
+}
+
+define float @seq_cst_float_shared_cluster_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_float_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_shared_cluster_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.volatile.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr addrspace(3) %addr syncscope("cluster") seq_cst, align 4
+ ret float %retval
+}
+
+define float @seq_cst_float_shared_gpu_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_float_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_shared_gpu_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.volatile.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr addrspace(3) %addr syncscope("device") seq_cst, align 4
+ ret float %retval
+}
+
+define double @seq_cst_double_shared_sys(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_double_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_shared_sys_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.volatile.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr addrspace(3) %addr syncscope("") seq_cst, align 8
+ ret double %retval
+}
+
+define double @seq_cst_double_shared_cta(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_double_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_shared_cta_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.volatile.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr addrspace(3) %addr syncscope("block") seq_cst, align 8
+ ret double %retval
+}
+
+define double @seq_cst_double_shared_cluster(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_double_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_shared_cluster_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.volatile.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr addrspace(3) %addr syncscope("cluster") seq_cst, align 8
+ ret double %retval
+}
+
+define double @seq_cst_double_shared_gpu(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_double_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_shared_gpu_param_0];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: ld.volatile.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr addrspace(3) %addr syncscope("device") seq_cst, align 8
+ ret double %retval
+}
+
+define double @seq_cst_double_shared_sys_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_double_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_shared_sys_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.volatile.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr addrspace(3) %addr syncscope("") seq_cst, align 8
+ ret double %retval
+}
+
+define double @seq_cst_double_shared_cta_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_double_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_shared_cta_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.volatile.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr addrspace(3) %addr syncscope("block") seq_cst, align 8
+ ret double %retval
+}
+
+define double @seq_cst_double_shared_cluster_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_double_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_shared_cluster_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.volatile.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr addrspace(3) %addr syncscope("cluster") seq_cst, align 8
+ ret double %retval
+}
+
+define double @seq_cst_double_shared_gpu_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_double_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_shared_gpu_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.volatile.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr addrspace(3) %addr syncscope("device") seq_cst, align 8
+ ret double %retval
+}
+
+define i8 @seq_cst_i8_local_sys() {
+; CHECK-LABEL: seq_cst_i8_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot1056[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1056;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b8 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i8, ptr addrspace(5) %slot syncscope("") seq_cst, align 1
+ ret i8 %retval
+}
+
+define i8 @seq_cst_i8_local_cta() {
+; CHECK-LABEL: seq_cst_i8_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot1057[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1057;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b8 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i8, ptr addrspace(5) %slot syncscope("block") seq_cst, align 1
+ ret i8 %retval
+}
+
+define i8 @seq_cst_i8_local_cluster() {
+; CHECK-LABEL: seq_cst_i8_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot1058[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1058;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b8 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i8, ptr addrspace(5) %slot syncscope("cluster") seq_cst, align 1
+ ret i8 %retval
+}
+
+define i8 @seq_cst_i8_local_gpu() {
+; CHECK-LABEL: seq_cst_i8_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot1059[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1059;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b8 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i8, ptr addrspace(5) %slot syncscope("device") seq_cst, align 1
+ ret i8 %retval
+}
+
+define i8 @seq_cst_i8_local_sys_volatile() {
+; CHECK-LABEL: seq_cst_i8_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot1060[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1060;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b8 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i8, ptr addrspace(5) %slot syncscope("") seq_cst, align 1
+ ret i8 %retval
+}
+
+define i8 @seq_cst_i8_local_cta_volatile() {
+; CHECK-LABEL: seq_cst_i8_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot1061[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1061;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b8 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i8, ptr addrspace(5) %slot syncscope("block") seq_cst, align 1
+ ret i8 %retval
+}
+
+define i8 @seq_cst_i8_local_cluster_volatile() {
+; CHECK-LABEL: seq_cst_i8_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot1062[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1062;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b8 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i8, ptr addrspace(5) %slot syncscope("cluster") seq_cst, align 1
+ ret i8 %retval
+}
+
+define i8 @seq_cst_i8_local_gpu_volatile() {
+; CHECK-LABEL: seq_cst_i8_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot1063[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1063;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b8 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i8, ptr addrspace(5) %slot syncscope("device") seq_cst, align 1
+ ret i8 %retval
+}
+
+define i16 @seq_cst_i16_local_sys() {
+; CHECK-LABEL: seq_cst_i16_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1064[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1064;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i16, ptr addrspace(5) %slot syncscope("") seq_cst, align 2
+ ret i16 %retval
+}
+
+define i16 @seq_cst_i16_local_cta() {
+; CHECK-LABEL: seq_cst_i16_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1065[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1065;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i16, ptr addrspace(5) %slot syncscope("block") seq_cst, align 2
+ ret i16 %retval
+}
+
+define i16 @seq_cst_i16_local_cluster() {
+; CHECK-LABEL: seq_cst_i16_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1066[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1066;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i16, ptr addrspace(5) %slot syncscope("cluster") seq_cst, align 2
+ ret i16 %retval
+}
+
+define i16 @seq_cst_i16_local_gpu() {
+; CHECK-LABEL: seq_cst_i16_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1067[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1067;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i16, ptr addrspace(5) %slot syncscope("device") seq_cst, align 2
+ ret i16 %retval
+}
+
+define i16 @seq_cst_i16_local_sys_volatile() {
+; CHECK-LABEL: seq_cst_i16_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1068[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1068;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i16, ptr addrspace(5) %slot syncscope("") seq_cst, align 2
+ ret i16 %retval
+}
+
+define i16 @seq_cst_i16_local_cta_volatile() {
+; CHECK-LABEL: seq_cst_i16_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1069[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1069;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i16, ptr addrspace(5) %slot syncscope("block") seq_cst, align 2
+ ret i16 %retval
+}
+
+define i16 @seq_cst_i16_local_cluster_volatile() {
+; CHECK-LABEL: seq_cst_i16_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1070[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1070;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i16, ptr addrspace(5) %slot syncscope("cluster") seq_cst, align 2
+ ret i16 %retval
+}
+
+define i16 @seq_cst_i16_local_gpu_volatile() {
+; CHECK-LABEL: seq_cst_i16_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1071[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1071;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i16, ptr addrspace(5) %slot syncscope("device") seq_cst, align 2
+ ret i16 %retval
+}
+
+define i32 @seq_cst_i32_local_sys() {
+; CHECK-LABEL: seq_cst_i32_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot1072[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1072;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i32, ptr addrspace(5) %slot syncscope("") seq_cst, align 4
+ ret i32 %retval
+}
+
+define i32 @seq_cst_i32_local_cta() {
+; CHECK-LABEL: seq_cst_i32_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot1073[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1073;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i32, ptr addrspace(5) %slot syncscope("block") seq_cst, align 4
+ ret i32 %retval
+}
+
+define i32 @seq_cst_i32_local_cluster() {
+; CHECK-LABEL: seq_cst_i32_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot1074[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1074;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i32, ptr addrspace(5) %slot syncscope("cluster") seq_cst, align 4
+ ret i32 %retval
+}
+
+define i32 @seq_cst_i32_local_gpu() {
+; CHECK-LABEL: seq_cst_i32_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot1075[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1075;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i32, ptr addrspace(5) %slot syncscope("device") seq_cst, align 4
+ ret i32 %retval
+}
+
+define i32 @seq_cst_i32_local_sys_volatile() {
+; CHECK-LABEL: seq_cst_i32_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot1076[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1076;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i32, ptr addrspace(5) %slot syncscope("") seq_cst, align 4
+ ret i32 %retval
+}
+
+define i32 @seq_cst_i32_local_cta_volatile() {
+; CHECK-LABEL: seq_cst_i32_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot1077[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1077;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i32, ptr addrspace(5) %slot syncscope("block") seq_cst, align 4
+ ret i32 %retval
+}
+
+define i32 @seq_cst_i32_local_cluster_volatile() {
+; CHECK-LABEL: seq_cst_i32_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot1078[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1078;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i32, ptr addrspace(5) %slot syncscope("cluster") seq_cst, align 4
+ ret i32 %retval
+}
+
+define i32 @seq_cst_i32_local_gpu_volatile() {
+; CHECK-LABEL: seq_cst_i32_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot1079[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1079;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i32, ptr addrspace(5) %slot syncscope("device") seq_cst, align 4
+ ret i32 %retval
+}
+
+define i64 @seq_cst_i64_local_sys() {
+; CHECK-LABEL: seq_cst_i64_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot1080[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1080;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i64, ptr addrspace(5) %slot syncscope("") seq_cst, align 8
+ ret i64 %retval
+}
+
+define i64 @seq_cst_i64_local_cta() {
+; CHECK-LABEL: seq_cst_i64_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot1081[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1081;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i64, ptr addrspace(5) %slot syncscope("block") seq_cst, align 8
+ ret i64 %retval
+}
+
+define i64 @seq_cst_i64_local_cluster() {
+; CHECK-LABEL: seq_cst_i64_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot1082[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1082;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i64, ptr addrspace(5) %slot syncscope("cluster") seq_cst, align 8
+ ret i64 %retval
+}
+
+define i64 @seq_cst_i64_local_gpu() {
+; CHECK-LABEL: seq_cst_i64_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot1083[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1083;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i64, ptr addrspace(5) %slot syncscope("device") seq_cst, align 8
+ ret i64 %retval
+}
+
+define i64 @seq_cst_i64_local_sys_volatile() {
+; CHECK-LABEL: seq_cst_i64_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot1084[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1084;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i64, ptr addrspace(5) %slot syncscope("") seq_cst, align 8
+ ret i64 %retval
+}
+
+define i64 @seq_cst_i64_local_cta_volatile() {
+; CHECK-LABEL: seq_cst_i64_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot1085[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1085;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i64, ptr addrspace(5) %slot syncscope("block") seq_cst, align 8
+ ret i64 %retval
+}
+
+define i64 @seq_cst_i64_local_cluster_volatile() {
+; CHECK-LABEL: seq_cst_i64_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot1086[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1086;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i64, ptr addrspace(5) %slot syncscope("cluster") seq_cst, align 8
+ ret i64 %retval
+}
+
+define i64 @seq_cst_i64_local_gpu_volatile() {
+; CHECK-LABEL: seq_cst_i64_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot1087[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1087;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i64, ptr addrspace(5) %slot syncscope("device") seq_cst, align 8
+ ret i64 %retval
+}
+
+define half @seq_cst_half_local_sys() {
+; CHECK-LABEL: seq_cst_half_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1088[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1088;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic half, ptr addrspace(5) %slot syncscope("") seq_cst, align 2
+ ret half %retval
+}
+
+define half @seq_cst_half_local_cta() {
+; CHECK-LABEL: seq_cst_half_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1089[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1089;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic half, ptr addrspace(5) %slot syncscope("block") seq_cst, align 2
+ ret half %retval
+}
+
+define half @seq_cst_half_local_cluster() {
+; CHECK-LABEL: seq_cst_half_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1090[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1090;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic half, ptr addrspace(5) %slot syncscope("cluster") seq_cst, align 2
+ ret half %retval
+}
+
+define half @seq_cst_half_local_gpu() {
+; CHECK-LABEL: seq_cst_half_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1091[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1091;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic half, ptr addrspace(5) %slot syncscope("device") seq_cst, align 2
+ ret half %retval
+}
+
+define half @seq_cst_half_local_sys_volatile() {
+; CHECK-LABEL: seq_cst_half_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1092[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1092;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile half, ptr addrspace(5) %slot syncscope("") seq_cst, align 2
+ ret half %retval
+}
+
+define half @seq_cst_half_local_cta_volatile() {
+; CHECK-LABEL: seq_cst_half_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1093[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1093;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile half, ptr addrspace(5) %slot syncscope("block") seq_cst, align 2
+ ret half %retval
+}
+
+define half @seq_cst_half_local_cluster_volatile() {
+; CHECK-LABEL: seq_cst_half_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1094[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1094;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile half, ptr addrspace(5) %slot syncscope("cluster") seq_cst, align 2
+ ret half %retval
+}
+
+define half @seq_cst_half_local_gpu_volatile() {
+; CHECK-LABEL: seq_cst_half_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1095[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1095;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile half, ptr addrspace(5) %slot syncscope("device") seq_cst, align 2
+ ret half %retval
+}
+
+define bfloat @seq_cst_bfloat_local_sys() {
+; CHECK-LABEL: seq_cst_bfloat_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1096[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1096;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic bfloat, ptr addrspace(5) %slot syncscope("") seq_cst, align 2
+ ret bfloat %retval
+}
+
+define bfloat @seq_cst_bfloat_local_cta() {
+; CHECK-LABEL: seq_cst_bfloat_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1097[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1097;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic bfloat, ptr addrspace(5) %slot syncscope("block") seq_cst, align 2
+ ret bfloat %retval
+}
+
+define bfloat @seq_cst_bfloat_local_cluster() {
+; CHECK-LABEL: seq_cst_bfloat_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1098[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1098;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic bfloat, ptr addrspace(5) %slot syncscope("cluster") seq_cst, align 2
+ ret bfloat %retval
+}
+
+define bfloat @seq_cst_bfloat_local_gpu() {
+; CHECK-LABEL: seq_cst_bfloat_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1099[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1099;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic bfloat, ptr addrspace(5) %slot syncscope("device") seq_cst, align 2
+ ret bfloat %retval
+}
+
+define bfloat @seq_cst_bfloat_local_sys_volatile() {
+; CHECK-LABEL: seq_cst_bfloat_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1100[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1100;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile bfloat, ptr addrspace(5) %slot syncscope("") seq_cst, align 2
+ ret bfloat %retval
+}
+
+define bfloat @seq_cst_bfloat_local_cta_volatile() {
+; CHECK-LABEL: seq_cst_bfloat_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1101[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1101;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile bfloat, ptr addrspace(5) %slot syncscope("block") seq_cst, align 2
+ ret bfloat %retval
+}
+
+define bfloat @seq_cst_bfloat_local_cluster_volatile() {
+; CHECK-LABEL: seq_cst_bfloat_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1102[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1102;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile bfloat, ptr addrspace(5) %slot syncscope("cluster") seq_cst, align 2
+ ret bfloat %retval
+}
+
+define bfloat @seq_cst_bfloat_local_gpu_volatile() {
+; CHECK-LABEL: seq_cst_bfloat_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1103[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1103;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile bfloat, ptr addrspace(5) %slot syncscope("device") seq_cst, align 2
+ ret bfloat %retval
+}
+
+define float @seq_cst_float_local_sys() {
+; CHECK-LABEL: seq_cst_float_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot1104[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1104;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic float, ptr addrspace(5) %slot syncscope("") seq_cst, align 4
+ ret float %retval
+}
+
+define float @seq_cst_float_local_cta() {
+; CHECK-LABEL: seq_cst_float_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot1105[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1105;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic float, ptr addrspace(5) %slot syncscope("block") seq_cst, align 4
+ ret float %retval
+}
+
+define float @seq_cst_float_local_cluster() {
+; CHECK-LABEL: seq_cst_float_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot1106[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1106;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic float, ptr addrspace(5) %slot syncscope("cluster") seq_cst, align 4
+ ret float %retval
+}
+
+define float @seq_cst_float_local_gpu() {
+; CHECK-LABEL: seq_cst_float_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot1107[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1107;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic float, ptr addrspace(5) %slot syncscope("device") seq_cst, align 4
+ ret float %retval
+}
+
+define float @seq_cst_float_local_sys_volatile() {
+; CHECK-LABEL: seq_cst_float_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot1108[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1108;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile float, ptr addrspace(5) %slot syncscope("") seq_cst, align 4
+ ret float %retval
+}
+
+define float @seq_cst_float_local_cta_volatile() {
+; CHECK-LABEL: seq_cst_float_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot1109[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1109;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile float, ptr addrspace(5) %slot syncscope("block") seq_cst, align 4
+ ret float %retval
+}
+
+define float @seq_cst_float_local_cluster_volatile() {
+; CHECK-LABEL: seq_cst_float_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot1110[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1110;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile float, ptr addrspace(5) %slot syncscope("cluster") seq_cst, align 4
+ ret float %retval
+}
+
+define float @seq_cst_float_local_gpu_volatile() {
+; CHECK-LABEL: seq_cst_float_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot1111[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1111;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile float, ptr addrspace(5) %slot syncscope("device") seq_cst, align 4
+ ret float %retval
+}
+
+define double @seq_cst_double_local_sys() {
+; CHECK-LABEL: seq_cst_double_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot1112[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1112;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic double, ptr addrspace(5) %slot syncscope("") seq_cst, align 8
+ ret double %retval
+}
+
+define double @seq_cst_double_local_cta() {
+; CHECK-LABEL: seq_cst_double_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot1113[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1113;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic double, ptr addrspace(5) %slot syncscope("block") seq_cst, align 8
+ ret double %retval
+}
+
+define double @seq_cst_double_local_cluster() {
+; CHECK-LABEL: seq_cst_double_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot1114[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1114;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic double, ptr addrspace(5) %slot syncscope("cluster") seq_cst, align 8
+ ret double %retval
+}
+
+define double @seq_cst_double_local_gpu() {
+; CHECK-LABEL: seq_cst_double_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot1115[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1115;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic double, ptr addrspace(5) %slot syncscope("device") seq_cst, align 8
+ ret double %retval
+}
+
+define double @seq_cst_double_local_sys_volatile() {
+; CHECK-LABEL: seq_cst_double_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot1116[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1116;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile double, ptr addrspace(5) %slot syncscope("") seq_cst, align 8
+ ret double %retval
+}
+
+define double @seq_cst_double_local_cta_volatile() {
+; CHECK-LABEL: seq_cst_double_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot1117[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1117;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile double, ptr addrspace(5) %slot syncscope("block") seq_cst, align 8
+ ret double %retval
+}
+
+define double @seq_cst_double_local_cluster_volatile() {
+; CHECK-LABEL: seq_cst_double_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot1118[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1118;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile double, ptr addrspace(5) %slot syncscope("cluster") seq_cst, align 8
+ ret double %retval
+}
+
+define double @seq_cst_double_local_gpu_volatile() {
+; CHECK-LABEL: seq_cst_double_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot1119[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1119;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile double, ptr addrspace(5) %slot syncscope("device") seq_cst, align 8
+ ret double %retval
+}
+
diff --git a/llvm/test/CodeGen/NVPTX/load-sm70.ll b/llvm/test/CodeGen/NVPTX/load-sm70.ll
new file mode 100644
index 0000000000000..eaea18f1955ca
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/load-sm70.ll
@@ -0,0 +1,19956 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_70 -mattr=+ptx63 | FileCheck %s
+; RUN: %if ptxas-sm_70 && ptxas-isa-6.3 %{ llc < %s -march=nvptx64 -mcpu=sm_70 -mattr=+ptx63 | %ptxas-verify -arch=sm_70 %}
+
+define i8 @notatomic_i8_generic(ptr %addr) {
+; CHECK-LABEL: notatomic_i8_generic(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i8_generic_param_0];
+; CHECK-NEXT: ld.b8 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load i8, ptr %addr
+ ret i8 %retval
+}
+
+define i8 @notatomic_i8_generic_volatile(ptr %addr) {
+; CHECK-LABEL: notatomic_i8_generic_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i8_generic_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b8 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load volatile i8, ptr %addr
+ ret i8 %retval
+}
+
+define i16 @notatomic_i16_generic(ptr %addr) {
+; CHECK-LABEL: notatomic_i16_generic(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i16_generic_param_0];
+; CHECK-NEXT: ld.b16 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load i16, ptr %addr
+ ret i16 %retval
+}
+
+define i16 @notatomic_i16_generic_volatile(ptr %addr) {
+; CHECK-LABEL: notatomic_i16_generic_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i16_generic_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b16 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load volatile i16, ptr %addr
+ ret i16 %retval
+}
+
+define i32 @notatomic_i32_generic(ptr %addr) {
+; CHECK-LABEL: notatomic_i32_generic(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i32_generic_param_0];
+; CHECK-NEXT: ld.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load i32, ptr %addr
+ ret i32 %retval
+}
+
+define i32 @notatomic_i32_generic_volatile(ptr %addr) {
+; CHECK-LABEL: notatomic_i32_generic_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i32_generic_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load volatile i32, ptr %addr
+ ret i32 %retval
+}
+
+define i64 @notatomic_i64_generic(ptr %addr) {
+; CHECK-LABEL: notatomic_i64_generic(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i64_generic_param_0];
+; CHECK-NEXT: ld.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load i64, ptr %addr
+ ret i64 %retval
+}
+
+define i64 @notatomic_i64_generic_volatile(ptr %addr) {
+; CHECK-LABEL: notatomic_i64_generic_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i64_generic_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load volatile i64, ptr %addr
+ ret i64 %retval
+}
+
+define half @notatomic_half_generic(ptr %addr) {
+; CHECK-LABEL: notatomic_half_generic(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_half_generic_param_0];
+; CHECK-NEXT: ld.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load half, ptr %addr
+ ret half %retval
+}
+
+define half @notatomic_half_generic_volatile(ptr %addr) {
+; CHECK-LABEL: notatomic_half_generic_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_half_generic_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load volatile half, ptr %addr
+ ret half %retval
+}
+
+define bfloat @notatomic_bfloat_generic(ptr %addr) {
+; CHECK-LABEL: notatomic_bfloat_generic(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_bfloat_generic_param_0];
+; CHECK-NEXT: ld.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load bfloat, ptr %addr
+ ret bfloat %retval
+}
+
+define bfloat @notatomic_bfloat_generic_volatile(ptr %addr) {
+; CHECK-LABEL: notatomic_bfloat_generic_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_bfloat_generic_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load volatile bfloat, ptr %addr
+ ret bfloat %retval
+}
+
+define float @notatomic_float_generic(ptr %addr) {
+; CHECK-LABEL: notatomic_float_generic(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_float_generic_param_0];
+; CHECK-NEXT: ld.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load float, ptr %addr
+ ret float %retval
+}
+
+define float @notatomic_float_generic_volatile(ptr %addr) {
+; CHECK-LABEL: notatomic_float_generic_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_float_generic_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load volatile float, ptr %addr
+ ret float %retval
+}
+
+define double @notatomic_double_generic(ptr %addr) {
+; CHECK-LABEL: notatomic_double_generic(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_double_generic_param_0];
+; CHECK-NEXT: ld.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load double, ptr %addr
+ ret double %retval
+}
+
+define double @notatomic_double_generic_volatile(ptr %addr) {
+; CHECK-LABEL: notatomic_double_generic_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_double_generic_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load volatile double, ptr %addr
+ ret double %retval
+}
+
+define i8 @notatomic_i8_global(ptr addrspace(1) %addr) {
+; CHECK-LABEL: notatomic_i8_global(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i8_global_param_0];
+; CHECK-NEXT: ld.global.b8 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load i8, ptr addrspace(1) %addr
+ ret i8 %retval
+}
+
+define i8 @notatomic_i8_global_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: notatomic_i8_global_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i8_global_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b8 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load volatile i8, ptr addrspace(1) %addr
+ ret i8 %retval
+}
+
+define i16 @notatomic_i16_global(ptr addrspace(1) %addr) {
+; CHECK-LABEL: notatomic_i16_global(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i16_global_param_0];
+; CHECK-NEXT: ld.global.b16 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load i16, ptr addrspace(1) %addr
+ ret i16 %retval
+}
+
+define i16 @notatomic_i16_global_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: notatomic_i16_global_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i16_global_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b16 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load volatile i16, ptr addrspace(1) %addr
+ ret i16 %retval
+}
+
+define i32 @notatomic_i32_global(ptr addrspace(1) %addr) {
+; CHECK-LABEL: notatomic_i32_global(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i32_global_param_0];
+; CHECK-NEXT: ld.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load i32, ptr addrspace(1) %addr
+ ret i32 %retval
+}
+
+define i32 @notatomic_i32_global_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: notatomic_i32_global_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i32_global_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load volatile i32, ptr addrspace(1) %addr
+ ret i32 %retval
+}
+
+define i64 @notatomic_i64_global(ptr addrspace(1) %addr) {
+; CHECK-LABEL: notatomic_i64_global(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i64_global_param_0];
+; CHECK-NEXT: ld.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load i64, ptr addrspace(1) %addr
+ ret i64 %retval
+}
+
+define i64 @notatomic_i64_global_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: notatomic_i64_global_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i64_global_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load volatile i64, ptr addrspace(1) %addr
+ ret i64 %retval
+}
+
+define half @notatomic_half_global(ptr addrspace(1) %addr) {
+; CHECK-LABEL: notatomic_half_global(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_half_global_param_0];
+; CHECK-NEXT: ld.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load half, ptr addrspace(1) %addr
+ ret half %retval
+}
+
+define half @notatomic_half_global_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: notatomic_half_global_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_half_global_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load volatile half, ptr addrspace(1) %addr
+ ret half %retval
+}
+
+define bfloat @notatomic_bfloat_global(ptr addrspace(1) %addr) {
+; CHECK-LABEL: notatomic_bfloat_global(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_bfloat_global_param_0];
+; CHECK-NEXT: ld.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load bfloat, ptr addrspace(1) %addr
+ ret bfloat %retval
+}
+
+define bfloat @notatomic_bfloat_global_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: notatomic_bfloat_global_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_bfloat_global_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load volatile bfloat, ptr addrspace(1) %addr
+ ret bfloat %retval
+}
+
+define float @notatomic_float_global(ptr addrspace(1) %addr) {
+; CHECK-LABEL: notatomic_float_global(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_float_global_param_0];
+; CHECK-NEXT: ld.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load float, ptr addrspace(1) %addr
+ ret float %retval
+}
+
+define float @notatomic_float_global_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: notatomic_float_global_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_float_global_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load volatile float, ptr addrspace(1) %addr
+ ret float %retval
+}
+
+define double @notatomic_double_global(ptr addrspace(1) %addr) {
+; CHECK-LABEL: notatomic_double_global(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_double_global_param_0];
+; CHECK-NEXT: ld.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load double, ptr addrspace(1) %addr
+ ret double %retval
+}
+
+define double @notatomic_double_global_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: notatomic_double_global_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_double_global_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load volatile double, ptr addrspace(1) %addr
+ ret double %retval
+}
+
+define i8 @notatomic_i8_shared(ptr addrspace(3) %addr) {
+; CHECK-LABEL: notatomic_i8_shared(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i8_shared_param_0];
+; CHECK-NEXT: ld.shared.b8 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load i8, ptr addrspace(3) %addr
+ ret i8 %retval
+}
+
+define i8 @notatomic_i8_shared_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: notatomic_i8_shared_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i8_shared_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b8 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load volatile i8, ptr addrspace(3) %addr
+ ret i8 %retval
+}
+
+define i16 @notatomic_i16_shared(ptr addrspace(3) %addr) {
+; CHECK-LABEL: notatomic_i16_shared(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i16_shared_param_0];
+; CHECK-NEXT: ld.shared.b16 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load i16, ptr addrspace(3) %addr
+ ret i16 %retval
+}
+
+define i16 @notatomic_i16_shared_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: notatomic_i16_shared_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i16_shared_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load volatile i16, ptr addrspace(3) %addr
+ ret i16 %retval
+}
+
+define i32 @notatomic_i32_shared(ptr addrspace(3) %addr) {
+; CHECK-LABEL: notatomic_i32_shared(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i32_shared_param_0];
+; CHECK-NEXT: ld.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load i32, ptr addrspace(3) %addr
+ ret i32 %retval
+}
+
+define i32 @notatomic_i32_shared_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: notatomic_i32_shared_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i32_shared_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load volatile i32, ptr addrspace(3) %addr
+ ret i32 %retval
+}
+
+define i64 @notatomic_i64_shared(ptr addrspace(3) %addr) {
+; CHECK-LABEL: notatomic_i64_shared(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i64_shared_param_0];
+; CHECK-NEXT: ld.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load i64, ptr addrspace(3) %addr
+ ret i64 %retval
+}
+
+define i64 @notatomic_i64_shared_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: notatomic_i64_shared_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i64_shared_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load volatile i64, ptr addrspace(3) %addr
+ ret i64 %retval
+}
+
+define half @notatomic_half_shared(ptr addrspace(3) %addr) {
+; CHECK-LABEL: notatomic_half_shared(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_half_shared_param_0];
+; CHECK-NEXT: ld.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load half, ptr addrspace(3) %addr
+ ret half %retval
+}
+
+define half @notatomic_half_shared_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: notatomic_half_shared_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_half_shared_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load volatile half, ptr addrspace(3) %addr
+ ret half %retval
+}
+
+define bfloat @notatomic_bfloat_shared(ptr addrspace(3) %addr) {
+; CHECK-LABEL: notatomic_bfloat_shared(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_bfloat_shared_param_0];
+; CHECK-NEXT: ld.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load bfloat, ptr addrspace(3) %addr
+ ret bfloat %retval
+}
+
+define bfloat @notatomic_bfloat_shared_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: notatomic_bfloat_shared_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_bfloat_shared_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load volatile bfloat, ptr addrspace(3) %addr
+ ret bfloat %retval
+}
+
+define float @notatomic_float_shared(ptr addrspace(3) %addr) {
+; CHECK-LABEL: notatomic_float_shared(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_float_shared_param_0];
+; CHECK-NEXT: ld.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load float, ptr addrspace(3) %addr
+ ret float %retval
+}
+
+define float @notatomic_float_shared_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: notatomic_float_shared_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_float_shared_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load volatile float, ptr addrspace(3) %addr
+ ret float %retval
+}
+
+define double @notatomic_double_shared(ptr addrspace(3) %addr) {
+; CHECK-LABEL: notatomic_double_shared(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_double_shared_param_0];
+; CHECK-NEXT: ld.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load double, ptr addrspace(3) %addr
+ ret double %retval
+}
+
+define double @notatomic_double_shared_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: notatomic_double_shared_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_double_shared_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load volatile double, ptr addrspace(3) %addr
+ ret double %retval
+}
+
+define i8 @notatomic_i8_const(ptr addrspace(4) %addr) {
+; CHECK-LABEL: notatomic_i8_const(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i8_const_param_0];
+; CHECK-NEXT: ld.const.b8 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load i8, ptr addrspace(4) %addr
+ ret i8 %retval
+}
+
+define i8 @notatomic_i8_const_volatile(ptr addrspace(4) %addr) {
+; CHECK-LABEL: notatomic_i8_const_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i8_const_volatile_param_0];
+; CHECK-NEXT: ld.const.b8 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load volatile i8, ptr addrspace(4) %addr
+ ret i8 %retval
+}
+
+define i16 @notatomic_i16_const(ptr addrspace(4) %addr) {
+; CHECK-LABEL: notatomic_i16_const(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i16_const_param_0];
+; CHECK-NEXT: ld.const.b16 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load i16, ptr addrspace(4) %addr
+ ret i16 %retval
+}
+
+define i16 @notatomic_i16_const_volatile(ptr addrspace(4) %addr) {
+; CHECK-LABEL: notatomic_i16_const_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i16_const_volatile_param_0];
+; CHECK-NEXT: ld.const.b16 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load volatile i16, ptr addrspace(4) %addr
+ ret i16 %retval
+}
+
+define i32 @notatomic_i32_const(ptr addrspace(4) %addr) {
+; CHECK-LABEL: notatomic_i32_const(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i32_const_param_0];
+; CHECK-NEXT: ld.const.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load i32, ptr addrspace(4) %addr
+ ret i32 %retval
+}
+
+define i32 @notatomic_i32_const_volatile(ptr addrspace(4) %addr) {
+; CHECK-LABEL: notatomic_i32_const_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i32_const_volatile_param_0];
+; CHECK-NEXT: ld.const.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load volatile i32, ptr addrspace(4) %addr
+ ret i32 %retval
+}
+
+define i64 @notatomic_i64_const(ptr addrspace(4) %addr) {
+; CHECK-LABEL: notatomic_i64_const(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i64_const_param_0];
+; CHECK-NEXT: ld.const.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load i64, ptr addrspace(4) %addr
+ ret i64 %retval
+}
+
+define i64 @notatomic_i64_const_volatile(ptr addrspace(4) %addr) {
+; CHECK-LABEL: notatomic_i64_const_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i64_const_volatile_param_0];
+; CHECK-NEXT: ld.const.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load volatile i64, ptr addrspace(4) %addr
+ ret i64 %retval
+}
+
+define half @notatomic_half_const(ptr addrspace(4) %addr) {
+; CHECK-LABEL: notatomic_half_const(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_half_const_param_0];
+; CHECK-NEXT: ld.const.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load half, ptr addrspace(4) %addr
+ ret half %retval
+}
+
+define half @notatomic_half_const_volatile(ptr addrspace(4) %addr) {
+; CHECK-LABEL: notatomic_half_const_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_half_const_volatile_param_0];
+; CHECK-NEXT: ld.const.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load volatile half, ptr addrspace(4) %addr
+ ret half %retval
+}
+
+define bfloat @notatomic_bfloat_const(ptr addrspace(4) %addr) {
+; CHECK-LABEL: notatomic_bfloat_const(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_bfloat_const_param_0];
+; CHECK-NEXT: ld.const.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load bfloat, ptr addrspace(4) %addr
+ ret bfloat %retval
+}
+
+define bfloat @notatomic_bfloat_const_volatile(ptr addrspace(4) %addr) {
+; CHECK-LABEL: notatomic_bfloat_const_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_bfloat_const_volatile_param_0];
+; CHECK-NEXT: ld.const.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load volatile bfloat, ptr addrspace(4) %addr
+ ret bfloat %retval
+}
+
+define float @notatomic_float_const(ptr addrspace(4) %addr) {
+; CHECK-LABEL: notatomic_float_const(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_float_const_param_0];
+; CHECK-NEXT: ld.const.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load float, ptr addrspace(4) %addr
+ ret float %retval
+}
+
+define float @notatomic_float_const_volatile(ptr addrspace(4) %addr) {
+; CHECK-LABEL: notatomic_float_const_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_float_const_volatile_param_0];
+; CHECK-NEXT: ld.const.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load volatile float, ptr addrspace(4) %addr
+ ret float %retval
+}
+
+define double @notatomic_double_const(ptr addrspace(4) %addr) {
+; CHECK-LABEL: notatomic_double_const(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_double_const_param_0];
+; CHECK-NEXT: ld.const.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load double, ptr addrspace(4) %addr
+ ret double %retval
+}
+
+define double @notatomic_double_const_volatile(ptr addrspace(4) %addr) {
+; CHECK-LABEL: notatomic_double_const_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_double_const_volatile_param_0];
+; CHECK-NEXT: ld.const.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load volatile double, ptr addrspace(4) %addr
+ ret double %retval
+}
+
+define i8 @notatomic_i8_local() {
+; CHECK-LABEL: notatomic_i8_local(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot64[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot64;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b8 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load i8, ptr addrspace(5) %slot
+ ret i8 %retval
+}
+
+define i8 @notatomic_i8_local_volatile() {
+; CHECK-LABEL: notatomic_i8_local_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot65[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot65;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b8 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load volatile i8, ptr addrspace(5) %slot
+ ret i8 %retval
+}
+
+define i16 @notatomic_i16_local() {
+; CHECK-LABEL: notatomic_i16_local(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot66[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot66;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load i16, ptr addrspace(5) %slot
+ ret i16 %retval
+}
+
+define i16 @notatomic_i16_local_volatile() {
+; CHECK-LABEL: notatomic_i16_local_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot67[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot67;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load volatile i16, ptr addrspace(5) %slot
+ ret i16 %retval
+}
+
+define i32 @notatomic_i32_local() {
+; CHECK-LABEL: notatomic_i32_local(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot68[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot68;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load i32, ptr addrspace(5) %slot
+ ret i32 %retval
+}
+
+define i32 @notatomic_i32_local_volatile() {
+; CHECK-LABEL: notatomic_i32_local_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot69[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot69;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load volatile i32, ptr addrspace(5) %slot
+ ret i32 %retval
+}
+
+define i64 @notatomic_i64_local() {
+; CHECK-LABEL: notatomic_i64_local(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot70[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot70;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load i64, ptr addrspace(5) %slot
+ ret i64 %retval
+}
+
+define i64 @notatomic_i64_local_volatile() {
+; CHECK-LABEL: notatomic_i64_local_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot71[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot71;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load volatile i64, ptr addrspace(5) %slot
+ ret i64 %retval
+}
+
+define half @notatomic_half_local() {
+; CHECK-LABEL: notatomic_half_local(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot72[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot72;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load half, ptr addrspace(5) %slot
+ ret half %retval
+}
+
+define half @notatomic_half_local_volatile() {
+; CHECK-LABEL: notatomic_half_local_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot73[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot73;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load volatile half, ptr addrspace(5) %slot
+ ret half %retval
+}
+
+define bfloat @notatomic_bfloat_local() {
+; CHECK-LABEL: notatomic_bfloat_local(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot74[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot74;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load bfloat, ptr addrspace(5) %slot
+ ret bfloat %retval
+}
+
+define bfloat @notatomic_bfloat_local_volatile() {
+; CHECK-LABEL: notatomic_bfloat_local_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot75[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot75;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load volatile bfloat, ptr addrspace(5) %slot
+ ret bfloat %retval
+}
+
+define float @notatomic_float_local() {
+; CHECK-LABEL: notatomic_float_local(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot76[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot76;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load float, ptr addrspace(5) %slot
+ ret float %retval
+}
+
+define float @notatomic_float_local_volatile() {
+; CHECK-LABEL: notatomic_float_local_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot77[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot77;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load volatile float, ptr addrspace(5) %slot
+ ret float %retval
+}
+
+define double @notatomic_double_local() {
+; CHECK-LABEL: notatomic_double_local(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot78[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot78;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca double, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load double, ptr addrspace(5) %slot
+ ret double %retval
+}
+
+define double @notatomic_double_local_volatile() {
+; CHECK-LABEL: notatomic_double_local_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot79[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot79;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca double, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load volatile double, ptr addrspace(5) %slot
+ ret double %retval
+}
+
+define i8 @notatomic_i8_param(ptr addrspace(101) %addr) {
+; CHECK-LABEL: notatomic_i8_param(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i8_param_param_0];
+; CHECK-NEXT: ld.param.b8 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load i8, ptr addrspace(101) %addr
+ ret i8 %retval
+}
+
+define i8 @notatomic_i8_param_volatile(ptr addrspace(101) %addr) {
+; CHECK-LABEL: notatomic_i8_param_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i8_param_volatile_param_0];
+; CHECK-NEXT: ld.param.b8 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load volatile i8, ptr addrspace(101) %addr
+ ret i8 %retval
+}
+
+define i16 @notatomic_i16_param(ptr addrspace(101) %addr) {
+; CHECK-LABEL: notatomic_i16_param(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i16_param_param_0];
+; CHECK-NEXT: ld.param.b16 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load i16, ptr addrspace(101) %addr
+ ret i16 %retval
+}
+
+define i16 @notatomic_i16_param_volatile(ptr addrspace(101) %addr) {
+; CHECK-LABEL: notatomic_i16_param_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i16_param_volatile_param_0];
+; CHECK-NEXT: ld.param.b16 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load volatile i16, ptr addrspace(101) %addr
+ ret i16 %retval
+}
+
+define i32 @notatomic_i32_param(ptr addrspace(101) %addr) {
+; CHECK-LABEL: notatomic_i32_param(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i32_param_param_0];
+; CHECK-NEXT: ld.param.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load i32, ptr addrspace(101) %addr
+ ret i32 %retval
+}
+
+define i32 @notatomic_i32_param_volatile(ptr addrspace(101) %addr) {
+; CHECK-LABEL: notatomic_i32_param_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i32_param_volatile_param_0];
+; CHECK-NEXT: ld.param.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load volatile i32, ptr addrspace(101) %addr
+ ret i32 %retval
+}
+
+define i64 @notatomic_i64_param(ptr addrspace(101) %addr) {
+; CHECK-LABEL: notatomic_i64_param(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i64_param_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load i64, ptr addrspace(101) %addr
+ ret i64 %retval
+}
+
+define i64 @notatomic_i64_param_volatile(ptr addrspace(101) %addr) {
+; CHECK-LABEL: notatomic_i64_param_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i64_param_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load volatile i64, ptr addrspace(101) %addr
+ ret i64 %retval
+}
+
+define half @notatomic_half_param(ptr addrspace(101) %addr) {
+; CHECK-LABEL: notatomic_half_param(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_half_param_param_0];
+; CHECK-NEXT: ld.param.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load half, ptr addrspace(101) %addr
+ ret half %retval
+}
+
+define half @notatomic_half_param_volatile(ptr addrspace(101) %addr) {
+; CHECK-LABEL: notatomic_half_param_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_half_param_volatile_param_0];
+; CHECK-NEXT: ld.param.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load volatile half, ptr addrspace(101) %addr
+ ret half %retval
+}
+
+define bfloat @notatomic_bfloat_param(ptr addrspace(101) %addr) {
+; CHECK-LABEL: notatomic_bfloat_param(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_bfloat_param_param_0];
+; CHECK-NEXT: ld.param.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load bfloat, ptr addrspace(101) %addr
+ ret bfloat %retval
+}
+
+define bfloat @notatomic_bfloat_param_volatile(ptr addrspace(101) %addr) {
+; CHECK-LABEL: notatomic_bfloat_param_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_bfloat_param_volatile_param_0];
+; CHECK-NEXT: ld.param.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load volatile bfloat, ptr addrspace(101) %addr
+ ret bfloat %retval
+}
+
+define float @notatomic_float_param(ptr addrspace(101) %addr) {
+; CHECK-LABEL: notatomic_float_param(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_float_param_param_0];
+; CHECK-NEXT: ld.param.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load float, ptr addrspace(101) %addr
+ ret float %retval
+}
+
+define float @notatomic_float_param_volatile(ptr addrspace(101) %addr) {
+; CHECK-LABEL: notatomic_float_param_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_float_param_volatile_param_0];
+; CHECK-NEXT: ld.param.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load volatile float, ptr addrspace(101) %addr
+ ret float %retval
+}
+
+define double @notatomic_double_param(ptr addrspace(101) %addr) {
+; CHECK-LABEL: notatomic_double_param(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_double_param_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load double, ptr addrspace(101) %addr
+ ret double %retval
+}
+
+define double @notatomic_double_param_volatile(ptr addrspace(101) %addr) {
+; CHECK-LABEL: notatomic_double_param_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_double_param_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load volatile double, ptr addrspace(101) %addr
+ ret double %retval
+}
+
+define i8 @unordered_i8_generic_sys(ptr %addr) {
+; CHECK-LABEL: unordered_i8_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_generic_sys_param_0];
+; CHECK-NEXT: ld.relaxed.sys.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr %addr syncscope("") unordered, align 1
+ ret i8 %retval
+}
+
+define i8 @unordered_i8_generic_cta(ptr %addr) {
+; CHECK-LABEL: unordered_i8_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_generic_cta_param_0];
+; CHECK-NEXT: ld.relaxed.cta.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr %addr syncscope("block") unordered, align 1
+ ret i8 %retval
+}
+
+define i8 @unordered_i8_generic_cluster(ptr %addr) {
+; CHECK-LABEL: unordered_i8_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_generic_cluster_param_0];
+; CHECK-NEXT: ld.relaxed.cta.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr %addr syncscope("cluster") unordered, align 1
+ ret i8 %retval
+}
+
+define i8 @unordered_i8_generic_gpu(ptr %addr) {
+; CHECK-LABEL: unordered_i8_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_generic_gpu_param_0];
+; CHECK-NEXT: ld.relaxed.gpu.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr %addr syncscope("device") unordered, align 1
+ ret i8 %retval
+}
+
+define i8 @unordered_i8_generic_sys_volatile(ptr %addr) {
+; CHECK-LABEL: unordered_i8_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr %addr syncscope("") unordered, align 1
+ ret i8 %retval
+}
+
+define i8 @unordered_i8_generic_cta_volatile(ptr %addr) {
+; CHECK-LABEL: unordered_i8_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr %addr syncscope("block") unordered, align 1
+ ret i8 %retval
+}
+
+define i8 @unordered_i8_generic_cluster_volatile(ptr %addr) {
+; CHECK-LABEL: unordered_i8_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr %addr syncscope("cluster") unordered, align 1
+ ret i8 %retval
+}
+
+define i8 @unordered_i8_generic_gpu_volatile(ptr %addr) {
+; CHECK-LABEL: unordered_i8_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr %addr syncscope("device") unordered, align 1
+ ret i8 %retval
+}
+
+define i16 @unordered_i16_generic_sys(ptr %addr) {
+; CHECK-LABEL: unordered_i16_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_generic_sys_param_0];
+; CHECK-NEXT: ld.relaxed.sys.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr %addr syncscope("") unordered, align 2
+ ret i16 %retval
+}
+
+define i16 @unordered_i16_generic_cta(ptr %addr) {
+; CHECK-LABEL: unordered_i16_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_generic_cta_param_0];
+; CHECK-NEXT: ld.relaxed.cta.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr %addr syncscope("block") unordered, align 2
+ ret i16 %retval
+}
+
+define i16 @unordered_i16_generic_cluster(ptr %addr) {
+; CHECK-LABEL: unordered_i16_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_generic_cluster_param_0];
+; CHECK-NEXT: ld.relaxed.cta.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr %addr syncscope("cluster") unordered, align 2
+ ret i16 %retval
+}
+
+define i16 @unordered_i16_generic_gpu(ptr %addr) {
+; CHECK-LABEL: unordered_i16_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_generic_gpu_param_0];
+; CHECK-NEXT: ld.relaxed.gpu.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr %addr syncscope("device") unordered, align 2
+ ret i16 %retval
+}
+
+define i16 @unordered_i16_generic_sys_volatile(ptr %addr) {
+; CHECK-LABEL: unordered_i16_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr %addr syncscope("") unordered, align 2
+ ret i16 %retval
+}
+
+define i16 @unordered_i16_generic_cta_volatile(ptr %addr) {
+; CHECK-LABEL: unordered_i16_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr %addr syncscope("block") unordered, align 2
+ ret i16 %retval
+}
+
+define i16 @unordered_i16_generic_cluster_volatile(ptr %addr) {
+; CHECK-LABEL: unordered_i16_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr %addr syncscope("cluster") unordered, align 2
+ ret i16 %retval
+}
+
+define i16 @unordered_i16_generic_gpu_volatile(ptr %addr) {
+; CHECK-LABEL: unordered_i16_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr %addr syncscope("device") unordered, align 2
+ ret i16 %retval
+}
+
+define i32 @unordered_i32_generic_sys(ptr %addr) {
+; CHECK-LABEL: unordered_i32_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_generic_sys_param_0];
+; CHECK-NEXT: ld.relaxed.sys.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr %addr syncscope("") unordered, align 4
+ ret i32 %retval
+}
+
+define i32 @unordered_i32_generic_cta(ptr %addr) {
+; CHECK-LABEL: unordered_i32_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_generic_cta_param_0];
+; CHECK-NEXT: ld.relaxed.cta.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr %addr syncscope("block") unordered, align 4
+ ret i32 %retval
+}
+
+define i32 @unordered_i32_generic_cluster(ptr %addr) {
+; CHECK-LABEL: unordered_i32_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_generic_cluster_param_0];
+; CHECK-NEXT: ld.relaxed.cta.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr %addr syncscope("cluster") unordered, align 4
+ ret i32 %retval
+}
+
+define i32 @unordered_i32_generic_gpu(ptr %addr) {
+; CHECK-LABEL: unordered_i32_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_generic_gpu_param_0];
+; CHECK-NEXT: ld.relaxed.gpu.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr %addr syncscope("device") unordered, align 4
+ ret i32 %retval
+}
+
+define i32 @unordered_i32_generic_sys_volatile(ptr %addr) {
+; CHECK-LABEL: unordered_i32_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr %addr syncscope("") unordered, align 4
+ ret i32 %retval
+}
+
+define i32 @unordered_i32_generic_cta_volatile(ptr %addr) {
+; CHECK-LABEL: unordered_i32_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr %addr syncscope("block") unordered, align 4
+ ret i32 %retval
+}
+
+define i32 @unordered_i32_generic_cluster_volatile(ptr %addr) {
+; CHECK-LABEL: unordered_i32_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr %addr syncscope("cluster") unordered, align 4
+ ret i32 %retval
+}
+
+define i32 @unordered_i32_generic_gpu_volatile(ptr %addr) {
+; CHECK-LABEL: unordered_i32_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr %addr syncscope("device") unordered, align 4
+ ret i32 %retval
+}
+
+define i64 @unordered_i64_generic_sys(ptr %addr) {
+; CHECK-LABEL: unordered_i64_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_generic_sys_param_0];
+; CHECK-NEXT: ld.relaxed.sys.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr %addr syncscope("") unordered, align 8
+ ret i64 %retval
+}
+
+define i64 @unordered_i64_generic_cta(ptr %addr) {
+; CHECK-LABEL: unordered_i64_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_generic_cta_param_0];
+; CHECK-NEXT: ld.relaxed.cta.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr %addr syncscope("block") unordered, align 8
+ ret i64 %retval
+}
+
+define i64 @unordered_i64_generic_cluster(ptr %addr) {
+; CHECK-LABEL: unordered_i64_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_generic_cluster_param_0];
+; CHECK-NEXT: ld.relaxed.cta.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr %addr syncscope("cluster") unordered, align 8
+ ret i64 %retval
+}
+
+define i64 @unordered_i64_generic_gpu(ptr %addr) {
+; CHECK-LABEL: unordered_i64_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_generic_gpu_param_0];
+; CHECK-NEXT: ld.relaxed.gpu.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr %addr syncscope("device") unordered, align 8
+ ret i64 %retval
+}
+
+define i64 @unordered_i64_generic_sys_volatile(ptr %addr) {
+; CHECK-LABEL: unordered_i64_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr %addr syncscope("") unordered, align 8
+ ret i64 %retval
+}
+
+define i64 @unordered_i64_generic_cta_volatile(ptr %addr) {
+; CHECK-LABEL: unordered_i64_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr %addr syncscope("block") unordered, align 8
+ ret i64 %retval
+}
+
+define i64 @unordered_i64_generic_cluster_volatile(ptr %addr) {
+; CHECK-LABEL: unordered_i64_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr %addr syncscope("cluster") unordered, align 8
+ ret i64 %retval
+}
+
+define i64 @unordered_i64_generic_gpu_volatile(ptr %addr) {
+; CHECK-LABEL: unordered_i64_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr %addr syncscope("device") unordered, align 8
+ ret i64 %retval
+}
+
+define half @unordered_half_generic_sys(ptr %addr) {
+; CHECK-LABEL: unordered_half_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_generic_sys_param_0];
+; CHECK-NEXT: ld.relaxed.sys.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr %addr syncscope("") unordered, align 2
+ ret half %retval
+}
+
+define half @unordered_half_generic_cta(ptr %addr) {
+; CHECK-LABEL: unordered_half_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_generic_cta_param_0];
+; CHECK-NEXT: ld.relaxed.cta.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr %addr syncscope("block") unordered, align 2
+ ret half %retval
+}
+
+define half @unordered_half_generic_cluster(ptr %addr) {
+; CHECK-LABEL: unordered_half_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_generic_cluster_param_0];
+; CHECK-NEXT: ld.relaxed.cta.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr %addr syncscope("cluster") unordered, align 2
+ ret half %retval
+}
+
+define half @unordered_half_generic_gpu(ptr %addr) {
+; CHECK-LABEL: unordered_half_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_generic_gpu_param_0];
+; CHECK-NEXT: ld.relaxed.gpu.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr %addr syncscope("device") unordered, align 2
+ ret half %retval
+}
+
+define half @unordered_half_generic_sys_volatile(ptr %addr) {
+; CHECK-LABEL: unordered_half_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr %addr syncscope("") unordered, align 2
+ ret half %retval
+}
+
+define half @unordered_half_generic_cta_volatile(ptr %addr) {
+; CHECK-LABEL: unordered_half_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr %addr syncscope("block") unordered, align 2
+ ret half %retval
+}
+
+define half @unordered_half_generic_cluster_volatile(ptr %addr) {
+; CHECK-LABEL: unordered_half_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr %addr syncscope("cluster") unordered, align 2
+ ret half %retval
+}
+
+define half @unordered_half_generic_gpu_volatile(ptr %addr) {
+; CHECK-LABEL: unordered_half_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr %addr syncscope("device") unordered, align 2
+ ret half %retval
+}
+
+define bfloat @unordered_bfloat_generic_sys(ptr %addr) {
+; CHECK-LABEL: unordered_bfloat_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_generic_sys_param_0];
+; CHECK-NEXT: ld.relaxed.sys.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr %addr syncscope("") unordered, align 2
+ ret bfloat %retval
+}
+
+define bfloat @unordered_bfloat_generic_cta(ptr %addr) {
+; CHECK-LABEL: unordered_bfloat_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_generic_cta_param_0];
+; CHECK-NEXT: ld.relaxed.cta.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr %addr syncscope("block") unordered, align 2
+ ret bfloat %retval
+}
+
+define bfloat @unordered_bfloat_generic_cluster(ptr %addr) {
+; CHECK-LABEL: unordered_bfloat_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_generic_cluster_param_0];
+; CHECK-NEXT: ld.relaxed.cta.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr %addr syncscope("cluster") unordered, align 2
+ ret bfloat %retval
+}
+
+define bfloat @unordered_bfloat_generic_gpu(ptr %addr) {
+; CHECK-LABEL: unordered_bfloat_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_generic_gpu_param_0];
+; CHECK-NEXT: ld.relaxed.gpu.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr %addr syncscope("device") unordered, align 2
+ ret bfloat %retval
+}
+
+define bfloat @unordered_bfloat_generic_sys_volatile(ptr %addr) {
+; CHECK-LABEL: unordered_bfloat_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr %addr syncscope("") unordered, align 2
+ ret bfloat %retval
+}
+
+define bfloat @unordered_bfloat_generic_cta_volatile(ptr %addr) {
+; CHECK-LABEL: unordered_bfloat_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr %addr syncscope("block") unordered, align 2
+ ret bfloat %retval
+}
+
+define bfloat @unordered_bfloat_generic_cluster_volatile(ptr %addr) {
+; CHECK-LABEL: unordered_bfloat_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr %addr syncscope("cluster") unordered, align 2
+ ret bfloat %retval
+}
+
+define bfloat @unordered_bfloat_generic_gpu_volatile(ptr %addr) {
+; CHECK-LABEL: unordered_bfloat_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr %addr syncscope("device") unordered, align 2
+ ret bfloat %retval
+}
+
+define float @unordered_float_generic_sys(ptr %addr) {
+; CHECK-LABEL: unordered_float_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_generic_sys_param_0];
+; CHECK-NEXT: ld.relaxed.sys.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr %addr syncscope("") unordered, align 4
+ ret float %retval
+}
+
+define float @unordered_float_generic_cta(ptr %addr) {
+; CHECK-LABEL: unordered_float_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_generic_cta_param_0];
+; CHECK-NEXT: ld.relaxed.cta.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr %addr syncscope("block") unordered, align 4
+ ret float %retval
+}
+
+define float @unordered_float_generic_cluster(ptr %addr) {
+; CHECK-LABEL: unordered_float_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_generic_cluster_param_0];
+; CHECK-NEXT: ld.relaxed.cta.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr %addr syncscope("cluster") unordered, align 4
+ ret float %retval
+}
+
+define float @unordered_float_generic_gpu(ptr %addr) {
+; CHECK-LABEL: unordered_float_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_generic_gpu_param_0];
+; CHECK-NEXT: ld.relaxed.gpu.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr %addr syncscope("device") unordered, align 4
+ ret float %retval
+}
+
+define float @unordered_float_generic_sys_volatile(ptr %addr) {
+; CHECK-LABEL: unordered_float_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr %addr syncscope("") unordered, align 4
+ ret float %retval
+}
+
+define float @unordered_float_generic_cta_volatile(ptr %addr) {
+; CHECK-LABEL: unordered_float_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr %addr syncscope("block") unordered, align 4
+ ret float %retval
+}
+
+define float @unordered_float_generic_cluster_volatile(ptr %addr) {
+; CHECK-LABEL: unordered_float_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr %addr syncscope("cluster") unordered, align 4
+ ret float %retval
+}
+
+define float @unordered_float_generic_gpu_volatile(ptr %addr) {
+; CHECK-LABEL: unordered_float_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr %addr syncscope("device") unordered, align 4
+ ret float %retval
+}
+
+define double @unordered_double_generic_sys(ptr %addr) {
+; CHECK-LABEL: unordered_double_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_generic_sys_param_0];
+; CHECK-NEXT: ld.relaxed.sys.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr %addr syncscope("") unordered, align 8
+ ret double %retval
+}
+
+define double @unordered_double_generic_cta(ptr %addr) {
+; CHECK-LABEL: unordered_double_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_generic_cta_param_0];
+; CHECK-NEXT: ld.relaxed.cta.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr %addr syncscope("block") unordered, align 8
+ ret double %retval
+}
+
+define double @unordered_double_generic_cluster(ptr %addr) {
+; CHECK-LABEL: unordered_double_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_generic_cluster_param_0];
+; CHECK-NEXT: ld.relaxed.cta.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr %addr syncscope("cluster") unordered, align 8
+ ret double %retval
+}
+
+define double @unordered_double_generic_gpu(ptr %addr) {
+; CHECK-LABEL: unordered_double_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_generic_gpu_param_0];
+; CHECK-NEXT: ld.relaxed.gpu.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr %addr syncscope("device") unordered, align 8
+ ret double %retval
+}
+
+define double @unordered_double_generic_sys_volatile(ptr %addr) {
+; CHECK-LABEL: unordered_double_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr %addr syncscope("") unordered, align 8
+ ret double %retval
+}
+
+define double @unordered_double_generic_cta_volatile(ptr %addr) {
+; CHECK-LABEL: unordered_double_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr %addr syncscope("block") unordered, align 8
+ ret double %retval
+}
+
+define double @unordered_double_generic_cluster_volatile(ptr %addr) {
+; CHECK-LABEL: unordered_double_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr %addr syncscope("cluster") unordered, align 8
+ ret double %retval
+}
+
+define double @unordered_double_generic_gpu_volatile(ptr %addr) {
+; CHECK-LABEL: unordered_double_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr %addr syncscope("device") unordered, align 8
+ ret double %retval
+}
+
+define i8 @unordered_i8_global_sys(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i8_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_global_sys_param_0];
+; CHECK-NEXT: ld.relaxed.sys.global.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr addrspace(1) %addr syncscope("") unordered, align 1
+ ret i8 %retval
+}
+
+define i8 @unordered_i8_global_cta(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i8_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_global_cta_param_0];
+; CHECK-NEXT: ld.relaxed.cta.global.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr addrspace(1) %addr syncscope("block") unordered, align 1
+ ret i8 %retval
+}
+
+define i8 @unordered_i8_global_cluster(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i8_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_global_cluster_param_0];
+; CHECK-NEXT: ld.relaxed.cta.global.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr addrspace(1) %addr syncscope("cluster") unordered, align 1
+ ret i8 %retval
+}
+
+define i8 @unordered_i8_global_gpu(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i8_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_global_gpu_param_0];
+; CHECK-NEXT: ld.relaxed.gpu.global.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr addrspace(1) %addr syncscope("device") unordered, align 1
+ ret i8 %retval
+}
+
+define i8 @unordered_i8_global_sys_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i8_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr addrspace(1) %addr syncscope("") unordered, align 1
+ ret i8 %retval
+}
+
+define i8 @unordered_i8_global_cta_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i8_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr addrspace(1) %addr syncscope("block") unordered, align 1
+ ret i8 %retval
+}
+
+define i8 @unordered_i8_global_cluster_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i8_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr addrspace(1) %addr syncscope("cluster") unordered, align 1
+ ret i8 %retval
+}
+
+define i8 @unordered_i8_global_gpu_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i8_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr addrspace(1) %addr syncscope("device") unordered, align 1
+ ret i8 %retval
+}
+
+define i16 @unordered_i16_global_sys(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i16_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_global_sys_param_0];
+; CHECK-NEXT: ld.relaxed.sys.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr addrspace(1) %addr syncscope("") unordered, align 2
+ ret i16 %retval
+}
+
+define i16 @unordered_i16_global_cta(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i16_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_global_cta_param_0];
+; CHECK-NEXT: ld.relaxed.cta.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr addrspace(1) %addr syncscope("block") unordered, align 2
+ ret i16 %retval
+}
+
+define i16 @unordered_i16_global_cluster(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i16_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_global_cluster_param_0];
+; CHECK-NEXT: ld.relaxed.cta.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr addrspace(1) %addr syncscope("cluster") unordered, align 2
+ ret i16 %retval
+}
+
+define i16 @unordered_i16_global_gpu(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i16_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_global_gpu_param_0];
+; CHECK-NEXT: ld.relaxed.gpu.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr addrspace(1) %addr syncscope("device") unordered, align 2
+ ret i16 %retval
+}
+
+define i16 @unordered_i16_global_sys_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i16_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr addrspace(1) %addr syncscope("") unordered, align 2
+ ret i16 %retval
+}
+
+define i16 @unordered_i16_global_cta_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i16_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr addrspace(1) %addr syncscope("block") unordered, align 2
+ ret i16 %retval
+}
+
+define i16 @unordered_i16_global_cluster_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i16_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr addrspace(1) %addr syncscope("cluster") unordered, align 2
+ ret i16 %retval
+}
+
+define i16 @unordered_i16_global_gpu_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i16_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr addrspace(1) %addr syncscope("device") unordered, align 2
+ ret i16 %retval
+}
+
+define i32 @unordered_i32_global_sys(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i32_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_global_sys_param_0];
+; CHECK-NEXT: ld.relaxed.sys.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr addrspace(1) %addr syncscope("") unordered, align 4
+ ret i32 %retval
+}
+
+define i32 @unordered_i32_global_cta(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i32_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_global_cta_param_0];
+; CHECK-NEXT: ld.relaxed.cta.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr addrspace(1) %addr syncscope("block") unordered, align 4
+ ret i32 %retval
+}
+
+define i32 @unordered_i32_global_cluster(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i32_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_global_cluster_param_0];
+; CHECK-NEXT: ld.relaxed.cta.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr addrspace(1) %addr syncscope("cluster") unordered, align 4
+ ret i32 %retval
+}
+
+define i32 @unordered_i32_global_gpu(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i32_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_global_gpu_param_0];
+; CHECK-NEXT: ld.relaxed.gpu.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr addrspace(1) %addr syncscope("device") unordered, align 4
+ ret i32 %retval
+}
+
+define i32 @unordered_i32_global_sys_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i32_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr addrspace(1) %addr syncscope("") unordered, align 4
+ ret i32 %retval
+}
+
+define i32 @unordered_i32_global_cta_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i32_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr addrspace(1) %addr syncscope("block") unordered, align 4
+ ret i32 %retval
+}
+
+define i32 @unordered_i32_global_cluster_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i32_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr addrspace(1) %addr syncscope("cluster") unordered, align 4
+ ret i32 %retval
+}
+
+define i32 @unordered_i32_global_gpu_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i32_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr addrspace(1) %addr syncscope("device") unordered, align 4
+ ret i32 %retval
+}
+
+define i64 @unordered_i64_global_sys(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i64_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_global_sys_param_0];
+; CHECK-NEXT: ld.relaxed.sys.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr addrspace(1) %addr syncscope("") unordered, align 8
+ ret i64 %retval
+}
+
+define i64 @unordered_i64_global_cta(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i64_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_global_cta_param_0];
+; CHECK-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr addrspace(1) %addr syncscope("block") unordered, align 8
+ ret i64 %retval
+}
+
+define i64 @unordered_i64_global_cluster(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i64_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_global_cluster_param_0];
+; CHECK-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr addrspace(1) %addr syncscope("cluster") unordered, align 8
+ ret i64 %retval
+}
+
+define i64 @unordered_i64_global_gpu(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i64_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_global_gpu_param_0];
+; CHECK-NEXT: ld.relaxed.gpu.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr addrspace(1) %addr syncscope("device") unordered, align 8
+ ret i64 %retval
+}
+
+define i64 @unordered_i64_global_sys_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i64_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr addrspace(1) %addr syncscope("") unordered, align 8
+ ret i64 %retval
+}
+
+define i64 @unordered_i64_global_cta_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i64_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr addrspace(1) %addr syncscope("block") unordered, align 8
+ ret i64 %retval
+}
+
+define i64 @unordered_i64_global_cluster_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i64_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr addrspace(1) %addr syncscope("cluster") unordered, align 8
+ ret i64 %retval
+}
+
+define i64 @unordered_i64_global_gpu_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i64_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr addrspace(1) %addr syncscope("device") unordered, align 8
+ ret i64 %retval
+}
+
+define half @unordered_half_global_sys(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_half_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_global_sys_param_0];
+; CHECK-NEXT: ld.relaxed.sys.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr addrspace(1) %addr syncscope("") unordered, align 2
+ ret half %retval
+}
+
+define half @unordered_half_global_cta(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_half_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_global_cta_param_0];
+; CHECK-NEXT: ld.relaxed.cta.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr addrspace(1) %addr syncscope("block") unordered, align 2
+ ret half %retval
+}
+
+define half @unordered_half_global_cluster(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_half_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_global_cluster_param_0];
+; CHECK-NEXT: ld.relaxed.cta.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr addrspace(1) %addr syncscope("cluster") unordered, align 2
+ ret half %retval
+}
+
+define half @unordered_half_global_gpu(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_half_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_global_gpu_param_0];
+; CHECK-NEXT: ld.relaxed.gpu.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr addrspace(1) %addr syncscope("device") unordered, align 2
+ ret half %retval
+}
+
+define half @unordered_half_global_sys_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_half_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr addrspace(1) %addr syncscope("") unordered, align 2
+ ret half %retval
+}
+
+define half @unordered_half_global_cta_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_half_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr addrspace(1) %addr syncscope("block") unordered, align 2
+ ret half %retval
+}
+
+define half @unordered_half_global_cluster_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_half_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr addrspace(1) %addr syncscope("cluster") unordered, align 2
+ ret half %retval
+}
+
+define half @unordered_half_global_gpu_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_half_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr addrspace(1) %addr syncscope("device") unordered, align 2
+ ret half %retval
+}
+
+define bfloat @unordered_bfloat_global_sys(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_bfloat_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_global_sys_param_0];
+; CHECK-NEXT: ld.relaxed.sys.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr addrspace(1) %addr syncscope("") unordered, align 2
+ ret bfloat %retval
+}
+
+define bfloat @unordered_bfloat_global_cta(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_bfloat_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_global_cta_param_0];
+; CHECK-NEXT: ld.relaxed.cta.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr addrspace(1) %addr syncscope("block") unordered, align 2
+ ret bfloat %retval
+}
+
+define bfloat @unordered_bfloat_global_cluster(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_bfloat_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_global_cluster_param_0];
+; CHECK-NEXT: ld.relaxed.cta.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr addrspace(1) %addr syncscope("cluster") unordered, align 2
+ ret bfloat %retval
+}
+
+define bfloat @unordered_bfloat_global_gpu(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_bfloat_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_global_gpu_param_0];
+; CHECK-NEXT: ld.relaxed.gpu.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr addrspace(1) %addr syncscope("device") unordered, align 2
+ ret bfloat %retval
+}
+
+define bfloat @unordered_bfloat_global_sys_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_bfloat_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr addrspace(1) %addr syncscope("") unordered, align 2
+ ret bfloat %retval
+}
+
+define bfloat @unordered_bfloat_global_cta_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_bfloat_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr addrspace(1) %addr syncscope("block") unordered, align 2
+ ret bfloat %retval
+}
+
+define bfloat @unordered_bfloat_global_cluster_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_bfloat_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr addrspace(1) %addr syncscope("cluster") unordered, align 2
+ ret bfloat %retval
+}
+
+define bfloat @unordered_bfloat_global_gpu_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_bfloat_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr addrspace(1) %addr syncscope("device") unordered, align 2
+ ret bfloat %retval
+}
+
+define float @unordered_float_global_sys(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_float_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_global_sys_param_0];
+; CHECK-NEXT: ld.relaxed.sys.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr addrspace(1) %addr syncscope("") unordered, align 4
+ ret float %retval
+}
+
+define float @unordered_float_global_cta(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_float_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_global_cta_param_0];
+; CHECK-NEXT: ld.relaxed.cta.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr addrspace(1) %addr syncscope("block") unordered, align 4
+ ret float %retval
+}
+
+define float @unordered_float_global_cluster(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_float_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_global_cluster_param_0];
+; CHECK-NEXT: ld.relaxed.cta.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr addrspace(1) %addr syncscope("cluster") unordered, align 4
+ ret float %retval
+}
+
+define float @unordered_float_global_gpu(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_float_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_global_gpu_param_0];
+; CHECK-NEXT: ld.relaxed.gpu.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr addrspace(1) %addr syncscope("device") unordered, align 4
+ ret float %retval
+}
+
+define float @unordered_float_global_sys_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_float_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr addrspace(1) %addr syncscope("") unordered, align 4
+ ret float %retval
+}
+
+define float @unordered_float_global_cta_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_float_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr addrspace(1) %addr syncscope("block") unordered, align 4
+ ret float %retval
+}
+
+define float @unordered_float_global_cluster_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_float_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr addrspace(1) %addr syncscope("cluster") unordered, align 4
+ ret float %retval
+}
+
+define float @unordered_float_global_gpu_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_float_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr addrspace(1) %addr syncscope("device") unordered, align 4
+ ret float %retval
+}
+
+define double @unordered_double_global_sys(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_double_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_global_sys_param_0];
+; CHECK-NEXT: ld.relaxed.sys.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr addrspace(1) %addr syncscope("") unordered, align 8
+ ret double %retval
+}
+
+define double @unordered_double_global_cta(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_double_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_global_cta_param_0];
+; CHECK-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr addrspace(1) %addr syncscope("block") unordered, align 8
+ ret double %retval
+}
+
+define double @unordered_double_global_cluster(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_double_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_global_cluster_param_0];
+; CHECK-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr addrspace(1) %addr syncscope("cluster") unordered, align 8
+ ret double %retval
+}
+
+define double @unordered_double_global_gpu(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_double_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_global_gpu_param_0];
+; CHECK-NEXT: ld.relaxed.gpu.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr addrspace(1) %addr syncscope("device") unordered, align 8
+ ret double %retval
+}
+
+define double @unordered_double_global_sys_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_double_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr addrspace(1) %addr syncscope("") unordered, align 8
+ ret double %retval
+}
+
+define double @unordered_double_global_cta_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_double_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr addrspace(1) %addr syncscope("block") unordered, align 8
+ ret double %retval
+}
+
+define double @unordered_double_global_cluster_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_double_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr addrspace(1) %addr syncscope("cluster") unordered, align 8
+ ret double %retval
+}
+
+define double @unordered_double_global_gpu_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_double_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr addrspace(1) %addr syncscope("device") unordered, align 8
+ ret double %retval
+}
+
+define i8 @unordered_i8_shared_sys(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i8_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_shared_sys_param_0];
+; CHECK-NEXT: ld.relaxed.sys.shared.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr addrspace(3) %addr syncscope("") unordered, align 1
+ ret i8 %retval
+}
+
+define i8 @unordered_i8_shared_cta(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i8_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_shared_cta_param_0];
+; CHECK-NEXT: ld.relaxed.cta.shared.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr addrspace(3) %addr syncscope("block") unordered, align 1
+ ret i8 %retval
+}
+
+define i8 @unordered_i8_shared_cluster(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i8_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_shared_cluster_param_0];
+; CHECK-NEXT: ld.relaxed.cta.shared.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr addrspace(3) %addr syncscope("cluster") unordered, align 1
+ ret i8 %retval
+}
+
+define i8 @unordered_i8_shared_gpu(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i8_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_shared_gpu_param_0];
+; CHECK-NEXT: ld.relaxed.gpu.shared.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr addrspace(3) %addr syncscope("device") unordered, align 1
+ ret i8 %retval
+}
+
+define i8 @unordered_i8_shared_sys_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i8_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr addrspace(3) %addr syncscope("") unordered, align 1
+ ret i8 %retval
+}
+
+define i8 @unordered_i8_shared_cta_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i8_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr addrspace(3) %addr syncscope("block") unordered, align 1
+ ret i8 %retval
+}
+
+define i8 @unordered_i8_shared_cluster_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i8_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr addrspace(3) %addr syncscope("cluster") unordered, align 1
+ ret i8 %retval
+}
+
+define i8 @unordered_i8_shared_gpu_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i8_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr addrspace(3) %addr syncscope("device") unordered, align 1
+ ret i8 %retval
+}
+
+define i16 @unordered_i16_shared_sys(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i16_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_shared_sys_param_0];
+; CHECK-NEXT: ld.relaxed.sys.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr addrspace(3) %addr syncscope("") unordered, align 2
+ ret i16 %retval
+}
+
+define i16 @unordered_i16_shared_cta(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i16_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_shared_cta_param_0];
+; CHECK-NEXT: ld.relaxed.cta.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr addrspace(3) %addr syncscope("block") unordered, align 2
+ ret i16 %retval
+}
+
+define i16 @unordered_i16_shared_cluster(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i16_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_shared_cluster_param_0];
+; CHECK-NEXT: ld.relaxed.cta.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr addrspace(3) %addr syncscope("cluster") unordered, align 2
+ ret i16 %retval
+}
+
+define i16 @unordered_i16_shared_gpu(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i16_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_shared_gpu_param_0];
+; CHECK-NEXT: ld.relaxed.gpu.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr addrspace(3) %addr syncscope("device") unordered, align 2
+ ret i16 %retval
+}
+
+define i16 @unordered_i16_shared_sys_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i16_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr addrspace(3) %addr syncscope("") unordered, align 2
+ ret i16 %retval
+}
+
+define i16 @unordered_i16_shared_cta_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i16_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr addrspace(3) %addr syncscope("block") unordered, align 2
+ ret i16 %retval
+}
+
+define i16 @unordered_i16_shared_cluster_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i16_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr addrspace(3) %addr syncscope("cluster") unordered, align 2
+ ret i16 %retval
+}
+
+define i16 @unordered_i16_shared_gpu_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i16_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr addrspace(3) %addr syncscope("device") unordered, align 2
+ ret i16 %retval
+}
+
+define i32 @unordered_i32_shared_sys(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i32_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_shared_sys_param_0];
+; CHECK-NEXT: ld.relaxed.sys.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr addrspace(3) %addr syncscope("") unordered, align 4
+ ret i32 %retval
+}
+
+define i32 @unordered_i32_shared_cta(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i32_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_shared_cta_param_0];
+; CHECK-NEXT: ld.relaxed.cta.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr addrspace(3) %addr syncscope("block") unordered, align 4
+ ret i32 %retval
+}
+
+define i32 @unordered_i32_shared_cluster(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i32_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_shared_cluster_param_0];
+; CHECK-NEXT: ld.relaxed.cta.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr addrspace(3) %addr syncscope("cluster") unordered, align 4
+ ret i32 %retval
+}
+
+define i32 @unordered_i32_shared_gpu(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i32_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_shared_gpu_param_0];
+; CHECK-NEXT: ld.relaxed.gpu.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr addrspace(3) %addr syncscope("device") unordered, align 4
+ ret i32 %retval
+}
+
+define i32 @unordered_i32_shared_sys_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i32_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr addrspace(3) %addr syncscope("") unordered, align 4
+ ret i32 %retval
+}
+
+define i32 @unordered_i32_shared_cta_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i32_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr addrspace(3) %addr syncscope("block") unordered, align 4
+ ret i32 %retval
+}
+
+define i32 @unordered_i32_shared_cluster_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i32_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr addrspace(3) %addr syncscope("cluster") unordered, align 4
+ ret i32 %retval
+}
+
+define i32 @unordered_i32_shared_gpu_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i32_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr addrspace(3) %addr syncscope("device") unordered, align 4
+ ret i32 %retval
+}
+
+define i64 @unordered_i64_shared_sys(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i64_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_shared_sys_param_0];
+; CHECK-NEXT: ld.relaxed.sys.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr addrspace(3) %addr syncscope("") unordered, align 8
+ ret i64 %retval
+}
+
+define i64 @unordered_i64_shared_cta(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i64_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_shared_cta_param_0];
+; CHECK-NEXT: ld.relaxed.cta.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr addrspace(3) %addr syncscope("block") unordered, align 8
+ ret i64 %retval
+}
+
+define i64 @unordered_i64_shared_cluster(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i64_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_shared_cluster_param_0];
+; CHECK-NEXT: ld.relaxed.cta.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr addrspace(3) %addr syncscope("cluster") unordered, align 8
+ ret i64 %retval
+}
+
+define i64 @unordered_i64_shared_gpu(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i64_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_shared_gpu_param_0];
+; CHECK-NEXT: ld.relaxed.gpu.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr addrspace(3) %addr syncscope("device") unordered, align 8
+ ret i64 %retval
+}
+
+define i64 @unordered_i64_shared_sys_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i64_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr addrspace(3) %addr syncscope("") unordered, align 8
+ ret i64 %retval
+}
+
+define i64 @unordered_i64_shared_cta_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i64_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr addrspace(3) %addr syncscope("block") unordered, align 8
+ ret i64 %retval
+}
+
+define i64 @unordered_i64_shared_cluster_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i64_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr addrspace(3) %addr syncscope("cluster") unordered, align 8
+ ret i64 %retval
+}
+
+define i64 @unordered_i64_shared_gpu_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i64_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr addrspace(3) %addr syncscope("device") unordered, align 8
+ ret i64 %retval
+}
+
+define half @unordered_half_shared_sys(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_half_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_shared_sys_param_0];
+; CHECK-NEXT: ld.relaxed.sys.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr addrspace(3) %addr syncscope("") unordered, align 2
+ ret half %retval
+}
+
+define half @unordered_half_shared_cta(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_half_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_shared_cta_param_0];
+; CHECK-NEXT: ld.relaxed.cta.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr addrspace(3) %addr syncscope("block") unordered, align 2
+ ret half %retval
+}
+
+define half @unordered_half_shared_cluster(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_half_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_shared_cluster_param_0];
+; CHECK-NEXT: ld.relaxed.cta.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr addrspace(3) %addr syncscope("cluster") unordered, align 2
+ ret half %retval
+}
+
+define half @unordered_half_shared_gpu(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_half_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_shared_gpu_param_0];
+; CHECK-NEXT: ld.relaxed.gpu.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr addrspace(3) %addr syncscope("device") unordered, align 2
+ ret half %retval
+}
+
+define half @unordered_half_shared_sys_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_half_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr addrspace(3) %addr syncscope("") unordered, align 2
+ ret half %retval
+}
+
+define half @unordered_half_shared_cta_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_half_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr addrspace(3) %addr syncscope("block") unordered, align 2
+ ret half %retval
+}
+
+define half @unordered_half_shared_cluster_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_half_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr addrspace(3) %addr syncscope("cluster") unordered, align 2
+ ret half %retval
+}
+
+define half @unordered_half_shared_gpu_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_half_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr addrspace(3) %addr syncscope("device") unordered, align 2
+ ret half %retval
+}
+
+define bfloat @unordered_bfloat_shared_sys(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_bfloat_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_shared_sys_param_0];
+; CHECK-NEXT: ld.relaxed.sys.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr addrspace(3) %addr syncscope("") unordered, align 2
+ ret bfloat %retval
+}
+
+define bfloat @unordered_bfloat_shared_cta(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_bfloat_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_shared_cta_param_0];
+; CHECK-NEXT: ld.relaxed.cta.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr addrspace(3) %addr syncscope("block") unordered, align 2
+ ret bfloat %retval
+}
+
+define bfloat @unordered_bfloat_shared_cluster(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_bfloat_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_shared_cluster_param_0];
+; CHECK-NEXT: ld.relaxed.cta.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr addrspace(3) %addr syncscope("cluster") unordered, align 2
+ ret bfloat %retval
+}
+
+define bfloat @unordered_bfloat_shared_gpu(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_bfloat_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_shared_gpu_param_0];
+; CHECK-NEXT: ld.relaxed.gpu.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr addrspace(3) %addr syncscope("device") unordered, align 2
+ ret bfloat %retval
+}
+
+define bfloat @unordered_bfloat_shared_sys_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_bfloat_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr addrspace(3) %addr syncscope("") unordered, align 2
+ ret bfloat %retval
+}
+
+define bfloat @unordered_bfloat_shared_cta_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_bfloat_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr addrspace(3) %addr syncscope("block") unordered, align 2
+ ret bfloat %retval
+}
+
+define bfloat @unordered_bfloat_shared_cluster_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_bfloat_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr addrspace(3) %addr syncscope("cluster") unordered, align 2
+ ret bfloat %retval
+}
+
+define bfloat @unordered_bfloat_shared_gpu_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_bfloat_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr addrspace(3) %addr syncscope("device") unordered, align 2
+ ret bfloat %retval
+}
+
+define float @unordered_float_shared_sys(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_float_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_shared_sys_param_0];
+; CHECK-NEXT: ld.relaxed.sys.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr addrspace(3) %addr syncscope("") unordered, align 4
+ ret float %retval
+}
+
+define float @unordered_float_shared_cta(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_float_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_shared_cta_param_0];
+; CHECK-NEXT: ld.relaxed.cta.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr addrspace(3) %addr syncscope("block") unordered, align 4
+ ret float %retval
+}
+
+define float @unordered_float_shared_cluster(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_float_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_shared_cluster_param_0];
+; CHECK-NEXT: ld.relaxed.cta.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr addrspace(3) %addr syncscope("cluster") unordered, align 4
+ ret float %retval
+}
+
+define float @unordered_float_shared_gpu(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_float_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_shared_gpu_param_0];
+; CHECK-NEXT: ld.relaxed.gpu.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr addrspace(3) %addr syncscope("device") unordered, align 4
+ ret float %retval
+}
+
+define float @unordered_float_shared_sys_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_float_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr addrspace(3) %addr syncscope("") unordered, align 4
+ ret float %retval
+}
+
+define float @unordered_float_shared_cta_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_float_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr addrspace(3) %addr syncscope("block") unordered, align 4
+ ret float %retval
+}
+
+define float @unordered_float_shared_cluster_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_float_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr addrspace(3) %addr syncscope("cluster") unordered, align 4
+ ret float %retval
+}
+
+define float @unordered_float_shared_gpu_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_float_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr addrspace(3) %addr syncscope("device") unordered, align 4
+ ret float %retval
+}
+
+define double @unordered_double_shared_sys(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_double_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_shared_sys_param_0];
+; CHECK-NEXT: ld.relaxed.sys.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr addrspace(3) %addr syncscope("") unordered, align 8
+ ret double %retval
+}
+
+define double @unordered_double_shared_cta(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_double_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_shared_cta_param_0];
+; CHECK-NEXT: ld.relaxed.cta.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr addrspace(3) %addr syncscope("block") unordered, align 8
+ ret double %retval
+}
+
+define double @unordered_double_shared_cluster(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_double_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_shared_cluster_param_0];
+; CHECK-NEXT: ld.relaxed.cta.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr addrspace(3) %addr syncscope("cluster") unordered, align 8
+ ret double %retval
+}
+
+define double @unordered_double_shared_gpu(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_double_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_shared_gpu_param_0];
+; CHECK-NEXT: ld.relaxed.gpu.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr addrspace(3) %addr syncscope("device") unordered, align 8
+ ret double %retval
+}
+
+define double @unordered_double_shared_sys_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_double_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr addrspace(3) %addr syncscope("") unordered, align 8
+ ret double %retval
+}
+
+define double @unordered_double_shared_cta_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_double_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr addrspace(3) %addr syncscope("block") unordered, align 8
+ ret double %retval
+}
+
+define double @unordered_double_shared_cluster_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_double_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr addrspace(3) %addr syncscope("cluster") unordered, align 8
+ ret double %retval
+}
+
+define double @unordered_double_shared_gpu_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_double_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr addrspace(3) %addr syncscope("device") unordered, align 8
+ ret double %retval
+}
+
+define i8 @unordered_i8_local_sys() {
+; CHECK-LABEL: unordered_i8_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot288[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot288;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b8 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i8, ptr addrspace(5) %slot syncscope("") unordered, align 1
+ ret i8 %retval
+}
+
+define i8 @unordered_i8_local_cta() {
+; CHECK-LABEL: unordered_i8_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot289[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot289;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b8 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i8, ptr addrspace(5) %slot syncscope("block") unordered, align 1
+ ret i8 %retval
+}
+
+define i8 @unordered_i8_local_cluster() {
+; CHECK-LABEL: unordered_i8_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot290[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot290;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b8 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i8, ptr addrspace(5) %slot syncscope("cluster") unordered, align 1
+ ret i8 %retval
+}
+
+define i8 @unordered_i8_local_gpu() {
+; CHECK-LABEL: unordered_i8_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot291[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot291;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b8 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i8, ptr addrspace(5) %slot syncscope("device") unordered, align 1
+ ret i8 %retval
+}
+
+define i8 @unordered_i8_local_sys_volatile() {
+; CHECK-LABEL: unordered_i8_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot292[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot292;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b8 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i8, ptr addrspace(5) %slot syncscope("") unordered, align 1
+ ret i8 %retval
+}
+
+define i8 @unordered_i8_local_cta_volatile() {
+; CHECK-LABEL: unordered_i8_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot293[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot293;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b8 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i8, ptr addrspace(5) %slot syncscope("block") unordered, align 1
+ ret i8 %retval
+}
+
+define i8 @unordered_i8_local_cluster_volatile() {
+; CHECK-LABEL: unordered_i8_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot294[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot294;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b8 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i8, ptr addrspace(5) %slot syncscope("cluster") unordered, align 1
+ ret i8 %retval
+}
+
+define i8 @unordered_i8_local_gpu_volatile() {
+; CHECK-LABEL: unordered_i8_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot295[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot295;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b8 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i8, ptr addrspace(5) %slot syncscope("device") unordered, align 1
+ ret i8 %retval
+}
+
+define i16 @unordered_i16_local_sys() {
+; CHECK-LABEL: unordered_i16_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot296[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot296;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i16, ptr addrspace(5) %slot syncscope("") unordered, align 2
+ ret i16 %retval
+}
+
+define i16 @unordered_i16_local_cta() {
+; CHECK-LABEL: unordered_i16_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot297[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot297;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i16, ptr addrspace(5) %slot syncscope("block") unordered, align 2
+ ret i16 %retval
+}
+
+define i16 @unordered_i16_local_cluster() {
+; CHECK-LABEL: unordered_i16_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot298[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot298;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i16, ptr addrspace(5) %slot syncscope("cluster") unordered, align 2
+ ret i16 %retval
+}
+
+define i16 @unordered_i16_local_gpu() {
+; CHECK-LABEL: unordered_i16_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot299[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot299;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i16, ptr addrspace(5) %slot syncscope("device") unordered, align 2
+ ret i16 %retval
+}
+
+define i16 @unordered_i16_local_sys_volatile() {
+; CHECK-LABEL: unordered_i16_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot300[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot300;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i16, ptr addrspace(5) %slot syncscope("") unordered, align 2
+ ret i16 %retval
+}
+
+define i16 @unordered_i16_local_cta_volatile() {
+; CHECK-LABEL: unordered_i16_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot301[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot301;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i16, ptr addrspace(5) %slot syncscope("block") unordered, align 2
+ ret i16 %retval
+}
+
+define i16 @unordered_i16_local_cluster_volatile() {
+; CHECK-LABEL: unordered_i16_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot302[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot302;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i16, ptr addrspace(5) %slot syncscope("cluster") unordered, align 2
+ ret i16 %retval
+}
+
+define i16 @unordered_i16_local_gpu_volatile() {
+; CHECK-LABEL: unordered_i16_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot303[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot303;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i16, ptr addrspace(5) %slot syncscope("device") unordered, align 2
+ ret i16 %retval
+}
+
+define i32 @unordered_i32_local_sys() {
+; CHECK-LABEL: unordered_i32_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot304[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot304;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i32, ptr addrspace(5) %slot syncscope("") unordered, align 4
+ ret i32 %retval
+}
+
+define i32 @unordered_i32_local_cta() {
+; CHECK-LABEL: unordered_i32_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot305[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot305;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i32, ptr addrspace(5) %slot syncscope("block") unordered, align 4
+ ret i32 %retval
+}
+
+define i32 @unordered_i32_local_cluster() {
+; CHECK-LABEL: unordered_i32_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot306[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot306;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i32, ptr addrspace(5) %slot syncscope("cluster") unordered, align 4
+ ret i32 %retval
+}
+
+define i32 @unordered_i32_local_gpu() {
+; CHECK-LABEL: unordered_i32_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot307[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot307;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i32, ptr addrspace(5) %slot syncscope("device") unordered, align 4
+ ret i32 %retval
+}
+
+define i32 @unordered_i32_local_sys_volatile() {
+; CHECK-LABEL: unordered_i32_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot308[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot308;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i32, ptr addrspace(5) %slot syncscope("") unordered, align 4
+ ret i32 %retval
+}
+
+define i32 @unordered_i32_local_cta_volatile() {
+; CHECK-LABEL: unordered_i32_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot309[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot309;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i32, ptr addrspace(5) %slot syncscope("block") unordered, align 4
+ ret i32 %retval
+}
+
+define i32 @unordered_i32_local_cluster_volatile() {
+; CHECK-LABEL: unordered_i32_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot310[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot310;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i32, ptr addrspace(5) %slot syncscope("cluster") unordered, align 4
+ ret i32 %retval
+}
+
+define i32 @unordered_i32_local_gpu_volatile() {
+; CHECK-LABEL: unordered_i32_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot311[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot311;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i32, ptr addrspace(5) %slot syncscope("device") unordered, align 4
+ ret i32 %retval
+}
+
+define i64 @unordered_i64_local_sys() {
+; CHECK-LABEL: unordered_i64_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot312[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot312;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i64, ptr addrspace(5) %slot syncscope("") unordered, align 8
+ ret i64 %retval
+}
+
+define i64 @unordered_i64_local_cta() {
+; CHECK-LABEL: unordered_i64_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot313[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot313;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i64, ptr addrspace(5) %slot syncscope("block") unordered, align 8
+ ret i64 %retval
+}
+
+define i64 @unordered_i64_local_cluster() {
+; CHECK-LABEL: unordered_i64_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot314[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot314;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i64, ptr addrspace(5) %slot syncscope("cluster") unordered, align 8
+ ret i64 %retval
+}
+
+define i64 @unordered_i64_local_gpu() {
+; CHECK-LABEL: unordered_i64_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot315[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot315;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i64, ptr addrspace(5) %slot syncscope("device") unordered, align 8
+ ret i64 %retval
+}
+
+define i64 @unordered_i64_local_sys_volatile() {
+; CHECK-LABEL: unordered_i64_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot316[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot316;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i64, ptr addrspace(5) %slot syncscope("") unordered, align 8
+ ret i64 %retval
+}
+
+define i64 @unordered_i64_local_cta_volatile() {
+; CHECK-LABEL: unordered_i64_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot317[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot317;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i64, ptr addrspace(5) %slot syncscope("block") unordered, align 8
+ ret i64 %retval
+}
+
+define i64 @unordered_i64_local_cluster_volatile() {
+; CHECK-LABEL: unordered_i64_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot318[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot318;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i64, ptr addrspace(5) %slot syncscope("cluster") unordered, align 8
+ ret i64 %retval
+}
+
+define i64 @unordered_i64_local_gpu_volatile() {
+; CHECK-LABEL: unordered_i64_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot319[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot319;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i64, ptr addrspace(5) %slot syncscope("device") unordered, align 8
+ ret i64 %retval
+}
+
+define half @unordered_half_local_sys() {
+; CHECK-LABEL: unordered_half_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot320[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot320;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic half, ptr addrspace(5) %slot syncscope("") unordered, align 2
+ ret half %retval
+}
+
+define half @unordered_half_local_cta() {
+; CHECK-LABEL: unordered_half_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot321[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot321;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic half, ptr addrspace(5) %slot syncscope("block") unordered, align 2
+ ret half %retval
+}
+
+define half @unordered_half_local_cluster() {
+; CHECK-LABEL: unordered_half_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot322[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot322;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic half, ptr addrspace(5) %slot syncscope("cluster") unordered, align 2
+ ret half %retval
+}
+
+define half @unordered_half_local_gpu() {
+; CHECK-LABEL: unordered_half_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot323[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot323;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic half, ptr addrspace(5) %slot syncscope("device") unordered, align 2
+ ret half %retval
+}
+
+define half @unordered_half_local_sys_volatile() {
+; CHECK-LABEL: unordered_half_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot324[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot324;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile half, ptr addrspace(5) %slot syncscope("") unordered, align 2
+ ret half %retval
+}
+
+define half @unordered_half_local_cta_volatile() {
+; CHECK-LABEL: unordered_half_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot325[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot325;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile half, ptr addrspace(5) %slot syncscope("block") unordered, align 2
+ ret half %retval
+}
+
+define half @unordered_half_local_cluster_volatile() {
+; CHECK-LABEL: unordered_half_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot326[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot326;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile half, ptr addrspace(5) %slot syncscope("cluster") unordered, align 2
+ ret half %retval
+}
+
+define half @unordered_half_local_gpu_volatile() {
+; CHECK-LABEL: unordered_half_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot327[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot327;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile half, ptr addrspace(5) %slot syncscope("device") unordered, align 2
+ ret half %retval
+}
+
+define bfloat @unordered_bfloat_local_sys() {
+; CHECK-LABEL: unordered_bfloat_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot328[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot328;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic bfloat, ptr addrspace(5) %slot syncscope("") unordered, align 2
+ ret bfloat %retval
+}
+
+define bfloat @unordered_bfloat_local_cta() {
+; CHECK-LABEL: unordered_bfloat_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot329[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot329;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic bfloat, ptr addrspace(5) %slot syncscope("block") unordered, align 2
+ ret bfloat %retval
+}
+
+define bfloat @unordered_bfloat_local_cluster() {
+; CHECK-LABEL: unordered_bfloat_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot330[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot330;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic bfloat, ptr addrspace(5) %slot syncscope("cluster") unordered, align 2
+ ret bfloat %retval
+}
+
+define bfloat @unordered_bfloat_local_gpu() {
+; CHECK-LABEL: unordered_bfloat_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot331[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot331;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic bfloat, ptr addrspace(5) %slot syncscope("device") unordered, align 2
+ ret bfloat %retval
+}
+
+define bfloat @unordered_bfloat_local_sys_volatile() {
+; CHECK-LABEL: unordered_bfloat_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot332[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot332;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile bfloat, ptr addrspace(5) %slot syncscope("") unordered, align 2
+ ret bfloat %retval
+}
+
+define bfloat @unordered_bfloat_local_cta_volatile() {
+; CHECK-LABEL: unordered_bfloat_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot333[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot333;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile bfloat, ptr addrspace(5) %slot syncscope("block") unordered, align 2
+ ret bfloat %retval
+}
+
+define bfloat @unordered_bfloat_local_cluster_volatile() {
+; CHECK-LABEL: unordered_bfloat_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot334[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot334;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile bfloat, ptr addrspace(5) %slot syncscope("cluster") unordered, align 2
+ ret bfloat %retval
+}
+
+define bfloat @unordered_bfloat_local_gpu_volatile() {
+; CHECK-LABEL: unordered_bfloat_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot335[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot335;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile bfloat, ptr addrspace(5) %slot syncscope("device") unordered, align 2
+ ret bfloat %retval
+}
+
+define float @unordered_float_local_sys() {
+; CHECK-LABEL: unordered_float_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot336[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot336;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic float, ptr addrspace(5) %slot syncscope("") unordered, align 4
+ ret float %retval
+}
+
+define float @unordered_float_local_cta() {
+; CHECK-LABEL: unordered_float_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot337[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot337;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic float, ptr addrspace(5) %slot syncscope("block") unordered, align 4
+ ret float %retval
+}
+
+define float @unordered_float_local_cluster() {
+; CHECK-LABEL: unordered_float_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot338[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot338;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic float, ptr addrspace(5) %slot syncscope("cluster") unordered, align 4
+ ret float %retval
+}
+
+define float @unordered_float_local_gpu() {
+; CHECK-LABEL: unordered_float_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot339[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot339;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic float, ptr addrspace(5) %slot syncscope("device") unordered, align 4
+ ret float %retval
+}
+
+define float @unordered_float_local_sys_volatile() {
+; CHECK-LABEL: unordered_float_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot340[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot340;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile float, ptr addrspace(5) %slot syncscope("") unordered, align 4
+ ret float %retval
+}
+
+define float @unordered_float_local_cta_volatile() {
+; CHECK-LABEL: unordered_float_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot341[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot341;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile float, ptr addrspace(5) %slot syncscope("block") unordered, align 4
+ ret float %retval
+}
+
+define float @unordered_float_local_cluster_volatile() {
+; CHECK-LABEL: unordered_float_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot342[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot342;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile float, ptr addrspace(5) %slot syncscope("cluster") unordered, align 4
+ ret float %retval
+}
+
+define float @unordered_float_local_gpu_volatile() {
+; CHECK-LABEL: unordered_float_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot343[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot343;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile float, ptr addrspace(5) %slot syncscope("device") unordered, align 4
+ ret float %retval
+}
+
+define double @unordered_double_local_sys() {
+; CHECK-LABEL: unordered_double_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot344[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot344;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic double, ptr addrspace(5) %slot syncscope("") unordered, align 8
+ ret double %retval
+}
+
+define double @unordered_double_local_cta() {
+; CHECK-LABEL: unordered_double_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot345[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot345;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic double, ptr addrspace(5) %slot syncscope("block") unordered, align 8
+ ret double %retval
+}
+
+define double @unordered_double_local_cluster() {
+; CHECK-LABEL: unordered_double_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot346[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot346;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic double, ptr addrspace(5) %slot syncscope("cluster") unordered, align 8
+ ret double %retval
+}
+
+define double @unordered_double_local_gpu() {
+; CHECK-LABEL: unordered_double_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot347[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot347;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic double, ptr addrspace(5) %slot syncscope("device") unordered, align 8
+ ret double %retval
+}
+
+define double @unordered_double_local_sys_volatile() {
+; CHECK-LABEL: unordered_double_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot348[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot348;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile double, ptr addrspace(5) %slot syncscope("") unordered, align 8
+ ret double %retval
+}
+
+define double @unordered_double_local_cta_volatile() {
+; CHECK-LABEL: unordered_double_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot349[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot349;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile double, ptr addrspace(5) %slot syncscope("block") unordered, align 8
+ ret double %retval
+}
+
+define double @unordered_double_local_cluster_volatile() {
+; CHECK-LABEL: unordered_double_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot350[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot350;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile double, ptr addrspace(5) %slot syncscope("cluster") unordered, align 8
+ ret double %retval
+}
+
+define double @unordered_double_local_gpu_volatile() {
+; CHECK-LABEL: unordered_double_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot351[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot351;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile double, ptr addrspace(5) %slot syncscope("device") unordered, align 8
+ ret double %retval
+}
+
+define i8 @monotonic_i8_generic_sys(ptr %addr) {
+; CHECK-LABEL: monotonic_i8_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_generic_sys_param_0];
+; CHECK-NEXT: ld.relaxed.sys.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr %addr syncscope("") monotonic, align 1
+ ret i8 %retval
+}
+
+define i8 @monotonic_i8_generic_cta(ptr %addr) {
+; CHECK-LABEL: monotonic_i8_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_generic_cta_param_0];
+; CHECK-NEXT: ld.relaxed.cta.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr %addr syncscope("block") monotonic, align 1
+ ret i8 %retval
+}
+
+define i8 @monotonic_i8_generic_cluster(ptr %addr) {
+; CHECK-LABEL: monotonic_i8_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_generic_cluster_param_0];
+; CHECK-NEXT: ld.relaxed.cta.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr %addr syncscope("cluster") monotonic, align 1
+ ret i8 %retval
+}
+
+define i8 @monotonic_i8_generic_gpu(ptr %addr) {
+; CHECK-LABEL: monotonic_i8_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_generic_gpu_param_0];
+; CHECK-NEXT: ld.relaxed.gpu.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr %addr syncscope("device") monotonic, align 1
+ ret i8 %retval
+}
+
+define i8 @monotonic_i8_generic_sys_volatile(ptr %addr) {
+; CHECK-LABEL: monotonic_i8_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr %addr syncscope("") monotonic, align 1
+ ret i8 %retval
+}
+
+define i8 @monotonic_i8_generic_cta_volatile(ptr %addr) {
+; CHECK-LABEL: monotonic_i8_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr %addr syncscope("block") monotonic, align 1
+ ret i8 %retval
+}
+
+define i8 @monotonic_i8_generic_cluster_volatile(ptr %addr) {
+; CHECK-LABEL: monotonic_i8_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr %addr syncscope("cluster") monotonic, align 1
+ ret i8 %retval
+}
+
+define i8 @monotonic_i8_generic_gpu_volatile(ptr %addr) {
+; CHECK-LABEL: monotonic_i8_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr %addr syncscope("device") monotonic, align 1
+ ret i8 %retval
+}
+
+define i16 @monotonic_i16_generic_sys(ptr %addr) {
+; CHECK-LABEL: monotonic_i16_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_generic_sys_param_0];
+; CHECK-NEXT: ld.relaxed.sys.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr %addr syncscope("") monotonic, align 2
+ ret i16 %retval
+}
+
+define i16 @monotonic_i16_generic_cta(ptr %addr) {
+; CHECK-LABEL: monotonic_i16_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_generic_cta_param_0];
+; CHECK-NEXT: ld.relaxed.cta.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr %addr syncscope("block") monotonic, align 2
+ ret i16 %retval
+}
+
+define i16 @monotonic_i16_generic_cluster(ptr %addr) {
+; CHECK-LABEL: monotonic_i16_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_generic_cluster_param_0];
+; CHECK-NEXT: ld.relaxed.cta.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr %addr syncscope("cluster") monotonic, align 2
+ ret i16 %retval
+}
+
+define i16 @monotonic_i16_generic_gpu(ptr %addr) {
+; CHECK-LABEL: monotonic_i16_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_generic_gpu_param_0];
+; CHECK-NEXT: ld.relaxed.gpu.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr %addr syncscope("device") monotonic, align 2
+ ret i16 %retval
+}
+
+define i16 @monotonic_i16_generic_sys_volatile(ptr %addr) {
+; CHECK-LABEL: monotonic_i16_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr %addr syncscope("") monotonic, align 2
+ ret i16 %retval
+}
+
+define i16 @monotonic_i16_generic_cta_volatile(ptr %addr) {
+; CHECK-LABEL: monotonic_i16_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr %addr syncscope("block") monotonic, align 2
+ ret i16 %retval
+}
+
+define i16 @monotonic_i16_generic_cluster_volatile(ptr %addr) {
+; CHECK-LABEL: monotonic_i16_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr %addr syncscope("cluster") monotonic, align 2
+ ret i16 %retval
+}
+
+define i16 @monotonic_i16_generic_gpu_volatile(ptr %addr) {
+; CHECK-LABEL: monotonic_i16_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr %addr syncscope("device") monotonic, align 2
+ ret i16 %retval
+}
+
+define i32 @monotonic_i32_generic_sys(ptr %addr) {
+; CHECK-LABEL: monotonic_i32_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_generic_sys_param_0];
+; CHECK-NEXT: ld.relaxed.sys.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr %addr syncscope("") monotonic, align 4
+ ret i32 %retval
+}
+
+define i32 @monotonic_i32_generic_cta(ptr %addr) {
+; CHECK-LABEL: monotonic_i32_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_generic_cta_param_0];
+; CHECK-NEXT: ld.relaxed.cta.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr %addr syncscope("block") monotonic, align 4
+ ret i32 %retval
+}
+
+define i32 @monotonic_i32_generic_cluster(ptr %addr) {
+; CHECK-LABEL: monotonic_i32_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_generic_cluster_param_0];
+; CHECK-NEXT: ld.relaxed.cta.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr %addr syncscope("cluster") monotonic, align 4
+ ret i32 %retval
+}
+
+define i32 @monotonic_i32_generic_gpu(ptr %addr) {
+; CHECK-LABEL: monotonic_i32_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_generic_gpu_param_0];
+; CHECK-NEXT: ld.relaxed.gpu.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr %addr syncscope("device") monotonic, align 4
+ ret i32 %retval
+}
+
+define i32 @monotonic_i32_generic_sys_volatile(ptr %addr) {
+; CHECK-LABEL: monotonic_i32_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr %addr syncscope("") monotonic, align 4
+ ret i32 %retval
+}
+
+define i32 @monotonic_i32_generic_cta_volatile(ptr %addr) {
+; CHECK-LABEL: monotonic_i32_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr %addr syncscope("block") monotonic, align 4
+ ret i32 %retval
+}
+
+define i32 @monotonic_i32_generic_cluster_volatile(ptr %addr) {
+; CHECK-LABEL: monotonic_i32_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr %addr syncscope("cluster") monotonic, align 4
+ ret i32 %retval
+}
+
+define i32 @monotonic_i32_generic_gpu_volatile(ptr %addr) {
+; CHECK-LABEL: monotonic_i32_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr %addr syncscope("device") monotonic, align 4
+ ret i32 %retval
+}
+
+define i64 @monotonic_i64_generic_sys(ptr %addr) {
+; CHECK-LABEL: monotonic_i64_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_generic_sys_param_0];
+; CHECK-NEXT: ld.relaxed.sys.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr %addr syncscope("") monotonic, align 8
+ ret i64 %retval
+}
+
+define i64 @monotonic_i64_generic_cta(ptr %addr) {
+; CHECK-LABEL: monotonic_i64_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_generic_cta_param_0];
+; CHECK-NEXT: ld.relaxed.cta.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr %addr syncscope("block") monotonic, align 8
+ ret i64 %retval
+}
+
+define i64 @monotonic_i64_generic_cluster(ptr %addr) {
+; CHECK-LABEL: monotonic_i64_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_generic_cluster_param_0];
+; CHECK-NEXT: ld.relaxed.cta.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr %addr syncscope("cluster") monotonic, align 8
+ ret i64 %retval
+}
+
+define i64 @monotonic_i64_generic_gpu(ptr %addr) {
+; CHECK-LABEL: monotonic_i64_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_generic_gpu_param_0];
+; CHECK-NEXT: ld.relaxed.gpu.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr %addr syncscope("device") monotonic, align 8
+ ret i64 %retval
+}
+
+define i64 @monotonic_i64_generic_sys_volatile(ptr %addr) {
+; CHECK-LABEL: monotonic_i64_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr %addr syncscope("") monotonic, align 8
+ ret i64 %retval
+}
+
+define i64 @monotonic_i64_generic_cta_volatile(ptr %addr) {
+; CHECK-LABEL: monotonic_i64_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr %addr syncscope("block") monotonic, align 8
+ ret i64 %retval
+}
+
+define i64 @monotonic_i64_generic_cluster_volatile(ptr %addr) {
+; CHECK-LABEL: monotonic_i64_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr %addr syncscope("cluster") monotonic, align 8
+ ret i64 %retval
+}
+
+define i64 @monotonic_i64_generic_gpu_volatile(ptr %addr) {
+; CHECK-LABEL: monotonic_i64_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr %addr syncscope("device") monotonic, align 8
+ ret i64 %retval
+}
+
+define half @monotonic_half_generic_sys(ptr %addr) {
+; CHECK-LABEL: monotonic_half_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_generic_sys_param_0];
+; CHECK-NEXT: ld.relaxed.sys.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr %addr syncscope("") monotonic, align 2
+ ret half %retval
+}
+
+define half @monotonic_half_generic_cta(ptr %addr) {
+; CHECK-LABEL: monotonic_half_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_generic_cta_param_0];
+; CHECK-NEXT: ld.relaxed.cta.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr %addr syncscope("block") monotonic, align 2
+ ret half %retval
+}
+
+define half @monotonic_half_generic_cluster(ptr %addr) {
+; CHECK-LABEL: monotonic_half_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_generic_cluster_param_0];
+; CHECK-NEXT: ld.relaxed.cta.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr %addr syncscope("cluster") monotonic, align 2
+ ret half %retval
+}
+
+define half @monotonic_half_generic_gpu(ptr %addr) {
+; CHECK-LABEL: monotonic_half_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_generic_gpu_param_0];
+; CHECK-NEXT: ld.relaxed.gpu.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr %addr syncscope("device") monotonic, align 2
+ ret half %retval
+}
+
+define half @monotonic_half_generic_sys_volatile(ptr %addr) {
+; CHECK-LABEL: monotonic_half_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr %addr syncscope("") monotonic, align 2
+ ret half %retval
+}
+
+define half @monotonic_half_generic_cta_volatile(ptr %addr) {
+; CHECK-LABEL: monotonic_half_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr %addr syncscope("block") monotonic, align 2
+ ret half %retval
+}
+
+define half @monotonic_half_generic_cluster_volatile(ptr %addr) {
+; CHECK-LABEL: monotonic_half_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr %addr syncscope("cluster") monotonic, align 2
+ ret half %retval
+}
+
+define half @monotonic_half_generic_gpu_volatile(ptr %addr) {
+; CHECK-LABEL: monotonic_half_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr %addr syncscope("device") monotonic, align 2
+ ret half %retval
+}
+
+define bfloat @monotonic_bfloat_generic_sys(ptr %addr) {
+; CHECK-LABEL: monotonic_bfloat_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_generic_sys_param_0];
+; CHECK-NEXT: ld.relaxed.sys.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr %addr syncscope("") monotonic, align 2
+ ret bfloat %retval
+}
+
+define bfloat @monotonic_bfloat_generic_cta(ptr %addr) {
+; CHECK-LABEL: monotonic_bfloat_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_generic_cta_param_0];
+; CHECK-NEXT: ld.relaxed.cta.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr %addr syncscope("block") monotonic, align 2
+ ret bfloat %retval
+}
+
+define bfloat @monotonic_bfloat_generic_cluster(ptr %addr) {
+; CHECK-LABEL: monotonic_bfloat_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_generic_cluster_param_0];
+; CHECK-NEXT: ld.relaxed.cta.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr %addr syncscope("cluster") monotonic, align 2
+ ret bfloat %retval
+}
+
+define bfloat @monotonic_bfloat_generic_gpu(ptr %addr) {
+; CHECK-LABEL: monotonic_bfloat_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_generic_gpu_param_0];
+; CHECK-NEXT: ld.relaxed.gpu.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr %addr syncscope("device") monotonic, align 2
+ ret bfloat %retval
+}
+
+define bfloat @monotonic_bfloat_generic_sys_volatile(ptr %addr) {
+; CHECK-LABEL: monotonic_bfloat_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr %addr syncscope("") monotonic, align 2
+ ret bfloat %retval
+}
+
+define bfloat @monotonic_bfloat_generic_cta_volatile(ptr %addr) {
+; CHECK-LABEL: monotonic_bfloat_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr %addr syncscope("block") monotonic, align 2
+ ret bfloat %retval
+}
+
+define bfloat @monotonic_bfloat_generic_cluster_volatile(ptr %addr) {
+; CHECK-LABEL: monotonic_bfloat_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr %addr syncscope("cluster") monotonic, align 2
+ ret bfloat %retval
+}
+
+define bfloat @monotonic_bfloat_generic_gpu_volatile(ptr %addr) {
+; CHECK-LABEL: monotonic_bfloat_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr %addr syncscope("device") monotonic, align 2
+ ret bfloat %retval
+}
+
+define float @monotonic_float_generic_sys(ptr %addr) {
+; CHECK-LABEL: monotonic_float_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_generic_sys_param_0];
+; CHECK-NEXT: ld.relaxed.sys.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr %addr syncscope("") monotonic, align 4
+ ret float %retval
+}
+
+define float @monotonic_float_generic_cta(ptr %addr) {
+; CHECK-LABEL: monotonic_float_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_generic_cta_param_0];
+; CHECK-NEXT: ld.relaxed.cta.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr %addr syncscope("block") monotonic, align 4
+ ret float %retval
+}
+
+define float @monotonic_float_generic_cluster(ptr %addr) {
+; CHECK-LABEL: monotonic_float_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_generic_cluster_param_0];
+; CHECK-NEXT: ld.relaxed.cta.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr %addr syncscope("cluster") monotonic, align 4
+ ret float %retval
+}
+
+define float @monotonic_float_generic_gpu(ptr %addr) {
+; CHECK-LABEL: monotonic_float_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_generic_gpu_param_0];
+; CHECK-NEXT: ld.relaxed.gpu.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr %addr syncscope("device") monotonic, align 4
+ ret float %retval
+}
+
+define float @monotonic_float_generic_sys_volatile(ptr %addr) {
+; CHECK-LABEL: monotonic_float_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr %addr syncscope("") monotonic, align 4
+ ret float %retval
+}
+
+define float @monotonic_float_generic_cta_volatile(ptr %addr) {
+; CHECK-LABEL: monotonic_float_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr %addr syncscope("block") monotonic, align 4
+ ret float %retval
+}
+
+define float @monotonic_float_generic_cluster_volatile(ptr %addr) {
+; CHECK-LABEL: monotonic_float_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr %addr syncscope("cluster") monotonic, align 4
+ ret float %retval
+}
+
+define float @monotonic_float_generic_gpu_volatile(ptr %addr) {
+; CHECK-LABEL: monotonic_float_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr %addr syncscope("device") monotonic, align 4
+ ret float %retval
+}
+
+define double @monotonic_double_generic_sys(ptr %addr) {
+; CHECK-LABEL: monotonic_double_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_generic_sys_param_0];
+; CHECK-NEXT: ld.relaxed.sys.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr %addr syncscope("") monotonic, align 8
+ ret double %retval
+}
+
+define double @monotonic_double_generic_cta(ptr %addr) {
+; CHECK-LABEL: monotonic_double_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_generic_cta_param_0];
+; CHECK-NEXT: ld.relaxed.cta.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr %addr syncscope("block") monotonic, align 8
+ ret double %retval
+}
+
+define double @monotonic_double_generic_cluster(ptr %addr) {
+; CHECK-LABEL: monotonic_double_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_generic_cluster_param_0];
+; CHECK-NEXT: ld.relaxed.cta.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr %addr syncscope("cluster") monotonic, align 8
+ ret double %retval
+}
+
+define double @monotonic_double_generic_gpu(ptr %addr) {
+; CHECK-LABEL: monotonic_double_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_generic_gpu_param_0];
+; CHECK-NEXT: ld.relaxed.gpu.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr %addr syncscope("device") monotonic, align 8
+ ret double %retval
+}
+
+define double @monotonic_double_generic_sys_volatile(ptr %addr) {
+; CHECK-LABEL: monotonic_double_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr %addr syncscope("") monotonic, align 8
+ ret double %retval
+}
+
+define double @monotonic_double_generic_cta_volatile(ptr %addr) {
+; CHECK-LABEL: monotonic_double_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr %addr syncscope("block") monotonic, align 8
+ ret double %retval
+}
+
+define double @monotonic_double_generic_cluster_volatile(ptr %addr) {
+; CHECK-LABEL: monotonic_double_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr %addr syncscope("cluster") monotonic, align 8
+ ret double %retval
+}
+
+define double @monotonic_double_generic_gpu_volatile(ptr %addr) {
+; CHECK-LABEL: monotonic_double_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr %addr syncscope("device") monotonic, align 8
+ ret double %retval
+}
+
+define i8 @monotonic_i8_global_sys(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i8_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_global_sys_param_0];
+; CHECK-NEXT: ld.relaxed.sys.global.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr addrspace(1) %addr syncscope("") monotonic, align 1
+ ret i8 %retval
+}
+
+define i8 @monotonic_i8_global_cta(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i8_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_global_cta_param_0];
+; CHECK-NEXT: ld.relaxed.cta.global.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr addrspace(1) %addr syncscope("block") monotonic, align 1
+ ret i8 %retval
+}
+
+define i8 @monotonic_i8_global_cluster(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i8_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_global_cluster_param_0];
+; CHECK-NEXT: ld.relaxed.cta.global.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr addrspace(1) %addr syncscope("cluster") monotonic, align 1
+ ret i8 %retval
+}
+
+define i8 @monotonic_i8_global_gpu(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i8_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_global_gpu_param_0];
+; CHECK-NEXT: ld.relaxed.gpu.global.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr addrspace(1) %addr syncscope("device") monotonic, align 1
+ ret i8 %retval
+}
+
+define i8 @monotonic_i8_global_sys_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i8_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr addrspace(1) %addr syncscope("") monotonic, align 1
+ ret i8 %retval
+}
+
+define i8 @monotonic_i8_global_cta_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i8_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr addrspace(1) %addr syncscope("block") monotonic, align 1
+ ret i8 %retval
+}
+
+define i8 @monotonic_i8_global_cluster_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i8_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr addrspace(1) %addr syncscope("cluster") monotonic, align 1
+ ret i8 %retval
+}
+
+define i8 @monotonic_i8_global_gpu_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i8_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr addrspace(1) %addr syncscope("device") monotonic, align 1
+ ret i8 %retval
+}
+
+define i16 @monotonic_i16_global_sys(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i16_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_global_sys_param_0];
+; CHECK-NEXT: ld.relaxed.sys.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr addrspace(1) %addr syncscope("") monotonic, align 2
+ ret i16 %retval
+}
+
+define i16 @monotonic_i16_global_cta(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i16_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_global_cta_param_0];
+; CHECK-NEXT: ld.relaxed.cta.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr addrspace(1) %addr syncscope("block") monotonic, align 2
+ ret i16 %retval
+}
+
+define i16 @monotonic_i16_global_cluster(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i16_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_global_cluster_param_0];
+; CHECK-NEXT: ld.relaxed.cta.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr addrspace(1) %addr syncscope("cluster") monotonic, align 2
+ ret i16 %retval
+}
+
+define i16 @monotonic_i16_global_gpu(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i16_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_global_gpu_param_0];
+; CHECK-NEXT: ld.relaxed.gpu.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr addrspace(1) %addr syncscope("device") monotonic, align 2
+ ret i16 %retval
+}
+
+define i16 @monotonic_i16_global_sys_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i16_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr addrspace(1) %addr syncscope("") monotonic, align 2
+ ret i16 %retval
+}
+
+define i16 @monotonic_i16_global_cta_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i16_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr addrspace(1) %addr syncscope("block") monotonic, align 2
+ ret i16 %retval
+}
+
+define i16 @monotonic_i16_global_cluster_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i16_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr addrspace(1) %addr syncscope("cluster") monotonic, align 2
+ ret i16 %retval
+}
+
+define i16 @monotonic_i16_global_gpu_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i16_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr addrspace(1) %addr syncscope("device") monotonic, align 2
+ ret i16 %retval
+}
+
+define i32 @monotonic_i32_global_sys(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i32_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_global_sys_param_0];
+; CHECK-NEXT: ld.relaxed.sys.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr addrspace(1) %addr syncscope("") monotonic, align 4
+ ret i32 %retval
+}
+
+define i32 @monotonic_i32_global_cta(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i32_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_global_cta_param_0];
+; CHECK-NEXT: ld.relaxed.cta.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr addrspace(1) %addr syncscope("block") monotonic, align 4
+ ret i32 %retval
+}
+
+define i32 @monotonic_i32_global_cluster(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i32_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_global_cluster_param_0];
+; CHECK-NEXT: ld.relaxed.cta.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr addrspace(1) %addr syncscope("cluster") monotonic, align 4
+ ret i32 %retval
+}
+
+define i32 @monotonic_i32_global_gpu(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i32_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_global_gpu_param_0];
+; CHECK-NEXT: ld.relaxed.gpu.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr addrspace(1) %addr syncscope("device") monotonic, align 4
+ ret i32 %retval
+}
+
+define i32 @monotonic_i32_global_sys_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i32_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr addrspace(1) %addr syncscope("") monotonic, align 4
+ ret i32 %retval
+}
+
+define i32 @monotonic_i32_global_cta_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i32_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr addrspace(1) %addr syncscope("block") monotonic, align 4
+ ret i32 %retval
+}
+
+define i32 @monotonic_i32_global_cluster_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i32_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr addrspace(1) %addr syncscope("cluster") monotonic, align 4
+ ret i32 %retval
+}
+
+define i32 @monotonic_i32_global_gpu_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i32_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr addrspace(1) %addr syncscope("device") monotonic, align 4
+ ret i32 %retval
+}
+
+define i64 @monotonic_i64_global_sys(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i64_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_global_sys_param_0];
+; CHECK-NEXT: ld.relaxed.sys.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr addrspace(1) %addr syncscope("") monotonic, align 8
+ ret i64 %retval
+}
+
+define i64 @monotonic_i64_global_cta(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i64_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_global_cta_param_0];
+; CHECK-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr addrspace(1) %addr syncscope("block") monotonic, align 8
+ ret i64 %retval
+}
+
+define i64 @monotonic_i64_global_cluster(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i64_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_global_cluster_param_0];
+; CHECK-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr addrspace(1) %addr syncscope("cluster") monotonic, align 8
+ ret i64 %retval
+}
+
+define i64 @monotonic_i64_global_gpu(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i64_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_global_gpu_param_0];
+; CHECK-NEXT: ld.relaxed.gpu.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr addrspace(1) %addr syncscope("device") monotonic, align 8
+ ret i64 %retval
+}
+
+define i64 @monotonic_i64_global_sys_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i64_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr addrspace(1) %addr syncscope("") monotonic, align 8
+ ret i64 %retval
+}
+
+define i64 @monotonic_i64_global_cta_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i64_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr addrspace(1) %addr syncscope("block") monotonic, align 8
+ ret i64 %retval
+}
+
+define i64 @monotonic_i64_global_cluster_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i64_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr addrspace(1) %addr syncscope("cluster") monotonic, align 8
+ ret i64 %retval
+}
+
+define i64 @monotonic_i64_global_gpu_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i64_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr addrspace(1) %addr syncscope("device") monotonic, align 8
+ ret i64 %retval
+}
+
+define half @monotonic_half_global_sys(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_half_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_global_sys_param_0];
+; CHECK-NEXT: ld.relaxed.sys.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr addrspace(1) %addr syncscope("") monotonic, align 2
+ ret half %retval
+}
+
+define half @monotonic_half_global_cta(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_half_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_global_cta_param_0];
+; CHECK-NEXT: ld.relaxed.cta.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr addrspace(1) %addr syncscope("block") monotonic, align 2
+ ret half %retval
+}
+
+define half @monotonic_half_global_cluster(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_half_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_global_cluster_param_0];
+; CHECK-NEXT: ld.relaxed.cta.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr addrspace(1) %addr syncscope("cluster") monotonic, align 2
+ ret half %retval
+}
+
+define half @monotonic_half_global_gpu(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_half_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_global_gpu_param_0];
+; CHECK-NEXT: ld.relaxed.gpu.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr addrspace(1) %addr syncscope("device") monotonic, align 2
+ ret half %retval
+}
+
+define half @monotonic_half_global_sys_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_half_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr addrspace(1) %addr syncscope("") monotonic, align 2
+ ret half %retval
+}
+
+define half @monotonic_half_global_cta_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_half_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr addrspace(1) %addr syncscope("block") monotonic, align 2
+ ret half %retval
+}
+
+define half @monotonic_half_global_cluster_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_half_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr addrspace(1) %addr syncscope("cluster") monotonic, align 2
+ ret half %retval
+}
+
+define half @monotonic_half_global_gpu_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_half_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr addrspace(1) %addr syncscope("device") monotonic, align 2
+ ret half %retval
+}
+
+define bfloat @monotonic_bfloat_global_sys(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_bfloat_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_global_sys_param_0];
+; CHECK-NEXT: ld.relaxed.sys.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr addrspace(1) %addr syncscope("") monotonic, align 2
+ ret bfloat %retval
+}
+
+define bfloat @monotonic_bfloat_global_cta(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_bfloat_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_global_cta_param_0];
+; CHECK-NEXT: ld.relaxed.cta.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr addrspace(1) %addr syncscope("block") monotonic, align 2
+ ret bfloat %retval
+}
+
+define bfloat @monotonic_bfloat_global_cluster(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_bfloat_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_global_cluster_param_0];
+; CHECK-NEXT: ld.relaxed.cta.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr addrspace(1) %addr syncscope("cluster") monotonic, align 2
+ ret bfloat %retval
+}
+
+define bfloat @monotonic_bfloat_global_gpu(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_bfloat_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_global_gpu_param_0];
+; CHECK-NEXT: ld.relaxed.gpu.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr addrspace(1) %addr syncscope("device") monotonic, align 2
+ ret bfloat %retval
+}
+
+define bfloat @monotonic_bfloat_global_sys_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_bfloat_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr addrspace(1) %addr syncscope("") monotonic, align 2
+ ret bfloat %retval
+}
+
+define bfloat @monotonic_bfloat_global_cta_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_bfloat_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr addrspace(1) %addr syncscope("block") monotonic, align 2
+ ret bfloat %retval
+}
+
+define bfloat @monotonic_bfloat_global_cluster_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_bfloat_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr addrspace(1) %addr syncscope("cluster") monotonic, align 2
+ ret bfloat %retval
+}
+
+define bfloat @monotonic_bfloat_global_gpu_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_bfloat_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr addrspace(1) %addr syncscope("device") monotonic, align 2
+ ret bfloat %retval
+}
+
+define float @monotonic_float_global_sys(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_float_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_global_sys_param_0];
+; CHECK-NEXT: ld.relaxed.sys.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr addrspace(1) %addr syncscope("") monotonic, align 4
+ ret float %retval
+}
+
+define float @monotonic_float_global_cta(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_float_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_global_cta_param_0];
+; CHECK-NEXT: ld.relaxed.cta.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr addrspace(1) %addr syncscope("block") monotonic, align 4
+ ret float %retval
+}
+
+define float @monotonic_float_global_cluster(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_float_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_global_cluster_param_0];
+; CHECK-NEXT: ld.relaxed.cta.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr addrspace(1) %addr syncscope("cluster") monotonic, align 4
+ ret float %retval
+}
+
+define float @monotonic_float_global_gpu(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_float_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_global_gpu_param_0];
+; CHECK-NEXT: ld.relaxed.gpu.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr addrspace(1) %addr syncscope("device") monotonic, align 4
+ ret float %retval
+}
+
+define float @monotonic_float_global_sys_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_float_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr addrspace(1) %addr syncscope("") monotonic, align 4
+ ret float %retval
+}
+
+define float @monotonic_float_global_cta_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_float_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr addrspace(1) %addr syncscope("block") monotonic, align 4
+ ret float %retval
+}
+
+define float @monotonic_float_global_cluster_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_float_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr addrspace(1) %addr syncscope("cluster") monotonic, align 4
+ ret float %retval
+}
+
+define float @monotonic_float_global_gpu_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_float_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr addrspace(1) %addr syncscope("device") monotonic, align 4
+ ret float %retval
+}
+
+define double @monotonic_double_global_sys(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_double_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_global_sys_param_0];
+; CHECK-NEXT: ld.relaxed.sys.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr addrspace(1) %addr syncscope("") monotonic, align 8
+ ret double %retval
+}
+
+define double @monotonic_double_global_cta(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_double_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_global_cta_param_0];
+; CHECK-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr addrspace(1) %addr syncscope("block") monotonic, align 8
+ ret double %retval
+}
+
+define double @monotonic_double_global_cluster(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_double_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_global_cluster_param_0];
+; CHECK-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr addrspace(1) %addr syncscope("cluster") monotonic, align 8
+ ret double %retval
+}
+
+define double @monotonic_double_global_gpu(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_double_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_global_gpu_param_0];
+; CHECK-NEXT: ld.relaxed.gpu.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr addrspace(1) %addr syncscope("device") monotonic, align 8
+ ret double %retval
+}
+
+define double @monotonic_double_global_sys_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_double_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr addrspace(1) %addr syncscope("") monotonic, align 8
+ ret double %retval
+}
+
+define double @monotonic_double_global_cta_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_double_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr addrspace(1) %addr syncscope("block") monotonic, align 8
+ ret double %retval
+}
+
+define double @monotonic_double_global_cluster_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_double_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr addrspace(1) %addr syncscope("cluster") monotonic, align 8
+ ret double %retval
+}
+
+define double @monotonic_double_global_gpu_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_double_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr addrspace(1) %addr syncscope("device") monotonic, align 8
+ ret double %retval
+}
+
+define i8 @monotonic_i8_shared_sys(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i8_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_shared_sys_param_0];
+; CHECK-NEXT: ld.relaxed.sys.shared.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr addrspace(3) %addr syncscope("") monotonic, align 1
+ ret i8 %retval
+}
+
+define i8 @monotonic_i8_shared_cta(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i8_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_shared_cta_param_0];
+; CHECK-NEXT: ld.relaxed.cta.shared.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr addrspace(3) %addr syncscope("block") monotonic, align 1
+ ret i8 %retval
+}
+
+define i8 @monotonic_i8_shared_cluster(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i8_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_shared_cluster_param_0];
+; CHECK-NEXT: ld.relaxed.cta.shared.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr addrspace(3) %addr syncscope("cluster") monotonic, align 1
+ ret i8 %retval
+}
+
+define i8 @monotonic_i8_shared_gpu(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i8_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_shared_gpu_param_0];
+; CHECK-NEXT: ld.relaxed.gpu.shared.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr addrspace(3) %addr syncscope("device") monotonic, align 1
+ ret i8 %retval
+}
+
+define i8 @monotonic_i8_shared_sys_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i8_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr addrspace(3) %addr syncscope("") monotonic, align 1
+ ret i8 %retval
+}
+
+define i8 @monotonic_i8_shared_cta_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i8_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr addrspace(3) %addr syncscope("block") monotonic, align 1
+ ret i8 %retval
+}
+
+define i8 @monotonic_i8_shared_cluster_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i8_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr addrspace(3) %addr syncscope("cluster") monotonic, align 1
+ ret i8 %retval
+}
+
+define i8 @monotonic_i8_shared_gpu_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i8_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr addrspace(3) %addr syncscope("device") monotonic, align 1
+ ret i8 %retval
+}
+
+define i16 @monotonic_i16_shared_sys(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i16_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_shared_sys_param_0];
+; CHECK-NEXT: ld.relaxed.sys.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr addrspace(3) %addr syncscope("") monotonic, align 2
+ ret i16 %retval
+}
+
+define i16 @monotonic_i16_shared_cta(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i16_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_shared_cta_param_0];
+; CHECK-NEXT: ld.relaxed.cta.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr addrspace(3) %addr syncscope("block") monotonic, align 2
+ ret i16 %retval
+}
+
+define i16 @monotonic_i16_shared_cluster(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i16_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_shared_cluster_param_0];
+; CHECK-NEXT: ld.relaxed.cta.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr addrspace(3) %addr syncscope("cluster") monotonic, align 2
+ ret i16 %retval
+}
+
+define i16 @monotonic_i16_shared_gpu(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i16_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_shared_gpu_param_0];
+; CHECK-NEXT: ld.relaxed.gpu.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr addrspace(3) %addr syncscope("device") monotonic, align 2
+ ret i16 %retval
+}
+
+define i16 @monotonic_i16_shared_sys_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i16_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr addrspace(3) %addr syncscope("") monotonic, align 2
+ ret i16 %retval
+}
+
+define i16 @monotonic_i16_shared_cta_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i16_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr addrspace(3) %addr syncscope("block") monotonic, align 2
+ ret i16 %retval
+}
+
+define i16 @monotonic_i16_shared_cluster_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i16_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr addrspace(3) %addr syncscope("cluster") monotonic, align 2
+ ret i16 %retval
+}
+
+define i16 @monotonic_i16_shared_gpu_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i16_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr addrspace(3) %addr syncscope("device") monotonic, align 2
+ ret i16 %retval
+}
+
+define i32 @monotonic_i32_shared_sys(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i32_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_shared_sys_param_0];
+; CHECK-NEXT: ld.relaxed.sys.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr addrspace(3) %addr syncscope("") monotonic, align 4
+ ret i32 %retval
+}
+
+define i32 @monotonic_i32_shared_cta(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i32_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_shared_cta_param_0];
+; CHECK-NEXT: ld.relaxed.cta.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr addrspace(3) %addr syncscope("block") monotonic, align 4
+ ret i32 %retval
+}
+
+define i32 @monotonic_i32_shared_cluster(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i32_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_shared_cluster_param_0];
+; CHECK-NEXT: ld.relaxed.cta.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr addrspace(3) %addr syncscope("cluster") monotonic, align 4
+ ret i32 %retval
+}
+
+define i32 @monotonic_i32_shared_gpu(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i32_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_shared_gpu_param_0];
+; CHECK-NEXT: ld.relaxed.gpu.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr addrspace(3) %addr syncscope("device") monotonic, align 4
+ ret i32 %retval
+}
+
+define i32 @monotonic_i32_shared_sys_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i32_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr addrspace(3) %addr syncscope("") monotonic, align 4
+ ret i32 %retval
+}
+
+define i32 @monotonic_i32_shared_cta_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i32_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr addrspace(3) %addr syncscope("block") monotonic, align 4
+ ret i32 %retval
+}
+
+define i32 @monotonic_i32_shared_cluster_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i32_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr addrspace(3) %addr syncscope("cluster") monotonic, align 4
+ ret i32 %retval
+}
+
+define i32 @monotonic_i32_shared_gpu_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i32_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr addrspace(3) %addr syncscope("device") monotonic, align 4
+ ret i32 %retval
+}
+
+define i64 @monotonic_i64_shared_sys(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i64_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_shared_sys_param_0];
+; CHECK-NEXT: ld.relaxed.sys.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr addrspace(3) %addr syncscope("") monotonic, align 8
+ ret i64 %retval
+}
+
+define i64 @monotonic_i64_shared_cta(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i64_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_shared_cta_param_0];
+; CHECK-NEXT: ld.relaxed.cta.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr addrspace(3) %addr syncscope("block") monotonic, align 8
+ ret i64 %retval
+}
+
+define i64 @monotonic_i64_shared_cluster(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i64_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_shared_cluster_param_0];
+; CHECK-NEXT: ld.relaxed.cta.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr addrspace(3) %addr syncscope("cluster") monotonic, align 8
+ ret i64 %retval
+}
+
+define i64 @monotonic_i64_shared_gpu(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i64_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_shared_gpu_param_0];
+; CHECK-NEXT: ld.relaxed.gpu.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr addrspace(3) %addr syncscope("device") monotonic, align 8
+ ret i64 %retval
+}
+
+define i64 @monotonic_i64_shared_sys_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i64_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr addrspace(3) %addr syncscope("") monotonic, align 8
+ ret i64 %retval
+}
+
+define i64 @monotonic_i64_shared_cta_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i64_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr addrspace(3) %addr syncscope("block") monotonic, align 8
+ ret i64 %retval
+}
+
+define i64 @monotonic_i64_shared_cluster_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i64_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr addrspace(3) %addr syncscope("cluster") monotonic, align 8
+ ret i64 %retval
+}
+
+define i64 @monotonic_i64_shared_gpu_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i64_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr addrspace(3) %addr syncscope("device") monotonic, align 8
+ ret i64 %retval
+}
+
+define half @monotonic_half_shared_sys(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_half_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_shared_sys_param_0];
+; CHECK-NEXT: ld.relaxed.sys.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr addrspace(3) %addr syncscope("") monotonic, align 2
+ ret half %retval
+}
+
+define half @monotonic_half_shared_cta(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_half_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_shared_cta_param_0];
+; CHECK-NEXT: ld.relaxed.cta.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr addrspace(3) %addr syncscope("block") monotonic, align 2
+ ret half %retval
+}
+
+define half @monotonic_half_shared_cluster(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_half_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_shared_cluster_param_0];
+; CHECK-NEXT: ld.relaxed.cta.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr addrspace(3) %addr syncscope("cluster") monotonic, align 2
+ ret half %retval
+}
+
+define half @monotonic_half_shared_gpu(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_half_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_shared_gpu_param_0];
+; CHECK-NEXT: ld.relaxed.gpu.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr addrspace(3) %addr syncscope("device") monotonic, align 2
+ ret half %retval
+}
+
+define half @monotonic_half_shared_sys_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_half_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr addrspace(3) %addr syncscope("") monotonic, align 2
+ ret half %retval
+}
+
+define half @monotonic_half_shared_cta_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_half_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr addrspace(3) %addr syncscope("block") monotonic, align 2
+ ret half %retval
+}
+
+define half @monotonic_half_shared_cluster_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_half_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr addrspace(3) %addr syncscope("cluster") monotonic, align 2
+ ret half %retval
+}
+
+define half @monotonic_half_shared_gpu_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_half_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr addrspace(3) %addr syncscope("device") monotonic, align 2
+ ret half %retval
+}
+
+define bfloat @monotonic_bfloat_shared_sys(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_bfloat_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_shared_sys_param_0];
+; CHECK-NEXT: ld.relaxed.sys.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr addrspace(3) %addr syncscope("") monotonic, align 2
+ ret bfloat %retval
+}
+
+define bfloat @monotonic_bfloat_shared_cta(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_bfloat_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_shared_cta_param_0];
+; CHECK-NEXT: ld.relaxed.cta.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr addrspace(3) %addr syncscope("block") monotonic, align 2
+ ret bfloat %retval
+}
+
+define bfloat @monotonic_bfloat_shared_cluster(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_bfloat_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_shared_cluster_param_0];
+; CHECK-NEXT: ld.relaxed.cta.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr addrspace(3) %addr syncscope("cluster") monotonic, align 2
+ ret bfloat %retval
+}
+
+define bfloat @monotonic_bfloat_shared_gpu(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_bfloat_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_shared_gpu_param_0];
+; CHECK-NEXT: ld.relaxed.gpu.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr addrspace(3) %addr syncscope("device") monotonic, align 2
+ ret bfloat %retval
+}
+
+define bfloat @monotonic_bfloat_shared_sys_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_bfloat_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr addrspace(3) %addr syncscope("") monotonic, align 2
+ ret bfloat %retval
+}
+
+define bfloat @monotonic_bfloat_shared_cta_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_bfloat_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr addrspace(3) %addr syncscope("block") monotonic, align 2
+ ret bfloat %retval
+}
+
+define bfloat @monotonic_bfloat_shared_cluster_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_bfloat_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr addrspace(3) %addr syncscope("cluster") monotonic, align 2
+ ret bfloat %retval
+}
+
+define bfloat @monotonic_bfloat_shared_gpu_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_bfloat_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr addrspace(3) %addr syncscope("device") monotonic, align 2
+ ret bfloat %retval
+}
+
+define float @monotonic_float_shared_sys(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_float_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_shared_sys_param_0];
+; CHECK-NEXT: ld.relaxed.sys.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr addrspace(3) %addr syncscope("") monotonic, align 4
+ ret float %retval
+}
+
+define float @monotonic_float_shared_cta(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_float_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_shared_cta_param_0];
+; CHECK-NEXT: ld.relaxed.cta.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr addrspace(3) %addr syncscope("block") monotonic, align 4
+ ret float %retval
+}
+
+define float @monotonic_float_shared_cluster(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_float_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_shared_cluster_param_0];
+; CHECK-NEXT: ld.relaxed.cta.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr addrspace(3) %addr syncscope("cluster") monotonic, align 4
+ ret float %retval
+}
+
+define float @monotonic_float_shared_gpu(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_float_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_shared_gpu_param_0];
+; CHECK-NEXT: ld.relaxed.gpu.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr addrspace(3) %addr syncscope("device") monotonic, align 4
+ ret float %retval
+}
+
+define float @monotonic_float_shared_sys_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_float_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr addrspace(3) %addr syncscope("") monotonic, align 4
+ ret float %retval
+}
+
+define float @monotonic_float_shared_cta_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_float_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr addrspace(3) %addr syncscope("block") monotonic, align 4
+ ret float %retval
+}
+
+define float @monotonic_float_shared_cluster_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_float_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr addrspace(3) %addr syncscope("cluster") monotonic, align 4
+ ret float %retval
+}
+
+define float @monotonic_float_shared_gpu_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_float_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr addrspace(3) %addr syncscope("device") monotonic, align 4
+ ret float %retval
+}
+
+define double @monotonic_double_shared_sys(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_double_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_shared_sys_param_0];
+; CHECK-NEXT: ld.relaxed.sys.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr addrspace(3) %addr syncscope("") monotonic, align 8
+ ret double %retval
+}
+
+define double @monotonic_double_shared_cta(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_double_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_shared_cta_param_0];
+; CHECK-NEXT: ld.relaxed.cta.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr addrspace(3) %addr syncscope("block") monotonic, align 8
+ ret double %retval
+}
+
+define double @monotonic_double_shared_cluster(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_double_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_shared_cluster_param_0];
+; CHECK-NEXT: ld.relaxed.cta.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr addrspace(3) %addr syncscope("cluster") monotonic, align 8
+ ret double %retval
+}
+
+define double @monotonic_double_shared_gpu(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_double_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_shared_gpu_param_0];
+; CHECK-NEXT: ld.relaxed.gpu.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr addrspace(3) %addr syncscope("device") monotonic, align 8
+ ret double %retval
+}
+
+define double @monotonic_double_shared_sys_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_double_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr addrspace(3) %addr syncscope("") monotonic, align 8
+ ret double %retval
+}
+
+define double @monotonic_double_shared_cta_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_double_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr addrspace(3) %addr syncscope("block") monotonic, align 8
+ ret double %retval
+}
+
+define double @monotonic_double_shared_cluster_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_double_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr addrspace(3) %addr syncscope("cluster") monotonic, align 8
+ ret double %retval
+}
+
+define double @monotonic_double_shared_gpu_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_double_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr addrspace(3) %addr syncscope("device") monotonic, align 8
+ ret double %retval
+}
+
+define i8 @monotonic_i8_local_sys() {
+; CHECK-LABEL: monotonic_i8_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot544[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot544;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b8 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i8, ptr addrspace(5) %slot syncscope("") monotonic, align 1
+ ret i8 %retval
+}
+
+define i8 @monotonic_i8_local_cta() {
+; CHECK-LABEL: monotonic_i8_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot545[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot545;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b8 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i8, ptr addrspace(5) %slot syncscope("block") monotonic, align 1
+ ret i8 %retval
+}
+
+define i8 @monotonic_i8_local_cluster() {
+; CHECK-LABEL: monotonic_i8_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot546[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot546;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b8 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i8, ptr addrspace(5) %slot syncscope("cluster") monotonic, align 1
+ ret i8 %retval
+}
+
+define i8 @monotonic_i8_local_gpu() {
+; CHECK-LABEL: monotonic_i8_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot547[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot547;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b8 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i8, ptr addrspace(5) %slot syncscope("device") monotonic, align 1
+ ret i8 %retval
+}
+
+define i8 @monotonic_i8_local_sys_volatile() {
+; CHECK-LABEL: monotonic_i8_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot548[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot548;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b8 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i8, ptr addrspace(5) %slot syncscope("") monotonic, align 1
+ ret i8 %retval
+}
+
+define i8 @monotonic_i8_local_cta_volatile() {
+; CHECK-LABEL: monotonic_i8_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot549[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot549;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b8 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i8, ptr addrspace(5) %slot syncscope("block") monotonic, align 1
+ ret i8 %retval
+}
+
+define i8 @monotonic_i8_local_cluster_volatile() {
+; CHECK-LABEL: monotonic_i8_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot550[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot550;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b8 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i8, ptr addrspace(5) %slot syncscope("cluster") monotonic, align 1
+ ret i8 %retval
+}
+
+define i8 @monotonic_i8_local_gpu_volatile() {
+; CHECK-LABEL: monotonic_i8_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot551[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot551;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b8 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i8, ptr addrspace(5) %slot syncscope("device") monotonic, align 1
+ ret i8 %retval
+}
+
+define i16 @monotonic_i16_local_sys() {
+; CHECK-LABEL: monotonic_i16_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot552[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot552;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i16, ptr addrspace(5) %slot syncscope("") monotonic, align 2
+ ret i16 %retval
+}
+
+define i16 @monotonic_i16_local_cta() {
+; CHECK-LABEL: monotonic_i16_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot553[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot553;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i16, ptr addrspace(5) %slot syncscope("block") monotonic, align 2
+ ret i16 %retval
+}
+
+define i16 @monotonic_i16_local_cluster() {
+; CHECK-LABEL: monotonic_i16_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot554[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot554;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i16, ptr addrspace(5) %slot syncscope("cluster") monotonic, align 2
+ ret i16 %retval
+}
+
+define i16 @monotonic_i16_local_gpu() {
+; CHECK-LABEL: monotonic_i16_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot555[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot555;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i16, ptr addrspace(5) %slot syncscope("device") monotonic, align 2
+ ret i16 %retval
+}
+
+define i16 @monotonic_i16_local_sys_volatile() {
+; CHECK-LABEL: monotonic_i16_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot556[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot556;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i16, ptr addrspace(5) %slot syncscope("") monotonic, align 2
+ ret i16 %retval
+}
+
+define i16 @monotonic_i16_local_cta_volatile() {
+; CHECK-LABEL: monotonic_i16_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot557[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot557;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i16, ptr addrspace(5) %slot syncscope("block") monotonic, align 2
+ ret i16 %retval
+}
+
+define i16 @monotonic_i16_local_cluster_volatile() {
+; CHECK-LABEL: monotonic_i16_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot558[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot558;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i16, ptr addrspace(5) %slot syncscope("cluster") monotonic, align 2
+ ret i16 %retval
+}
+
+define i16 @monotonic_i16_local_gpu_volatile() {
+; CHECK-LABEL: monotonic_i16_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot559[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot559;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i16, ptr addrspace(5) %slot syncscope("device") monotonic, align 2
+ ret i16 %retval
+}
+
+define i32 @monotonic_i32_local_sys() {
+; CHECK-LABEL: monotonic_i32_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot560[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot560;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i32, ptr addrspace(5) %slot syncscope("") monotonic, align 4
+ ret i32 %retval
+}
+
+define i32 @monotonic_i32_local_cta() {
+; CHECK-LABEL: monotonic_i32_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot561[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot561;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i32, ptr addrspace(5) %slot syncscope("block") monotonic, align 4
+ ret i32 %retval
+}
+
+define i32 @monotonic_i32_local_cluster() {
+; CHECK-LABEL: monotonic_i32_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot562[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot562;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i32, ptr addrspace(5) %slot syncscope("cluster") monotonic, align 4
+ ret i32 %retval
+}
+
+define i32 @monotonic_i32_local_gpu() {
+; CHECK-LABEL: monotonic_i32_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot563[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot563;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i32, ptr addrspace(5) %slot syncscope("device") monotonic, align 4
+ ret i32 %retval
+}
+
+define i32 @monotonic_i32_local_sys_volatile() {
+; CHECK-LABEL: monotonic_i32_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot564[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot564;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i32, ptr addrspace(5) %slot syncscope("") monotonic, align 4
+ ret i32 %retval
+}
+
+define i32 @monotonic_i32_local_cta_volatile() {
+; CHECK-LABEL: monotonic_i32_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot565[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot565;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i32, ptr addrspace(5) %slot syncscope("block") monotonic, align 4
+ ret i32 %retval
+}
+
+define i32 @monotonic_i32_local_cluster_volatile() {
+; CHECK-LABEL: monotonic_i32_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot566[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot566;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i32, ptr addrspace(5) %slot syncscope("cluster") monotonic, align 4
+ ret i32 %retval
+}
+
+define i32 @monotonic_i32_local_gpu_volatile() {
+; CHECK-LABEL: monotonic_i32_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot567[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot567;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i32, ptr addrspace(5) %slot syncscope("device") monotonic, align 4
+ ret i32 %retval
+}
+
+define i64 @monotonic_i64_local_sys() {
+; CHECK-LABEL: monotonic_i64_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot568[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot568;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i64, ptr addrspace(5) %slot syncscope("") monotonic, align 8
+ ret i64 %retval
+}
+
+define i64 @monotonic_i64_local_cta() {
+; CHECK-LABEL: monotonic_i64_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot569[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot569;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i64, ptr addrspace(5) %slot syncscope("block") monotonic, align 8
+ ret i64 %retval
+}
+
+define i64 @monotonic_i64_local_cluster() {
+; CHECK-LABEL: monotonic_i64_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot570[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot570;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i64, ptr addrspace(5) %slot syncscope("cluster") monotonic, align 8
+ ret i64 %retval
+}
+
+define i64 @monotonic_i64_local_gpu() {
+; CHECK-LABEL: monotonic_i64_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot571[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot571;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i64, ptr addrspace(5) %slot syncscope("device") monotonic, align 8
+ ret i64 %retval
+}
+
+define i64 @monotonic_i64_local_sys_volatile() {
+; CHECK-LABEL: monotonic_i64_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot572[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot572;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i64, ptr addrspace(5) %slot syncscope("") monotonic, align 8
+ ret i64 %retval
+}
+
+define i64 @monotonic_i64_local_cta_volatile() {
+; CHECK-LABEL: monotonic_i64_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot573[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot573;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i64, ptr addrspace(5) %slot syncscope("block") monotonic, align 8
+ ret i64 %retval
+}
+
+define i64 @monotonic_i64_local_cluster_volatile() {
+; CHECK-LABEL: monotonic_i64_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot574[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot574;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i64, ptr addrspace(5) %slot syncscope("cluster") monotonic, align 8
+ ret i64 %retval
+}
+
+define i64 @monotonic_i64_local_gpu_volatile() {
+; CHECK-LABEL: monotonic_i64_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot575[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot575;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i64, ptr addrspace(5) %slot syncscope("device") monotonic, align 8
+ ret i64 %retval
+}
+
+define half @monotonic_half_local_sys() {
+; CHECK-LABEL: monotonic_half_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot576[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot576;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic half, ptr addrspace(5) %slot syncscope("") monotonic, align 2
+ ret half %retval
+}
+
+define half @monotonic_half_local_cta() {
+; CHECK-LABEL: monotonic_half_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot577[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot577;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic half, ptr addrspace(5) %slot syncscope("block") monotonic, align 2
+ ret half %retval
+}
+
+define half @monotonic_half_local_cluster() {
+; CHECK-LABEL: monotonic_half_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot578[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot578;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic half, ptr addrspace(5) %slot syncscope("cluster") monotonic, align 2
+ ret half %retval
+}
+
+define half @monotonic_half_local_gpu() {
+; CHECK-LABEL: monotonic_half_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot579[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot579;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic half, ptr addrspace(5) %slot syncscope("device") monotonic, align 2
+ ret half %retval
+}
+
+define half @monotonic_half_local_sys_volatile() {
+; CHECK-LABEL: monotonic_half_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot580[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot580;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile half, ptr addrspace(5) %slot syncscope("") monotonic, align 2
+ ret half %retval
+}
+
+define half @monotonic_half_local_cta_volatile() {
+; CHECK-LABEL: monotonic_half_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot581[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot581;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile half, ptr addrspace(5) %slot syncscope("block") monotonic, align 2
+ ret half %retval
+}
+
+define half @monotonic_half_local_cluster_volatile() {
+; CHECK-LABEL: monotonic_half_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot582[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot582;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile half, ptr addrspace(5) %slot syncscope("cluster") monotonic, align 2
+ ret half %retval
+}
+
+define half @monotonic_half_local_gpu_volatile() {
+; CHECK-LABEL: monotonic_half_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot583[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot583;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile half, ptr addrspace(5) %slot syncscope("device") monotonic, align 2
+ ret half %retval
+}
+
+define bfloat @monotonic_bfloat_local_sys() {
+; CHECK-LABEL: monotonic_bfloat_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot584[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot584;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic bfloat, ptr addrspace(5) %slot syncscope("") monotonic, align 2
+ ret bfloat %retval
+}
+
+define bfloat @monotonic_bfloat_local_cta() {
+; CHECK-LABEL: monotonic_bfloat_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot585[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot585;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic bfloat, ptr addrspace(5) %slot syncscope("block") monotonic, align 2
+ ret bfloat %retval
+}
+
+define bfloat @monotonic_bfloat_local_cluster() {
+; CHECK-LABEL: monotonic_bfloat_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot586[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot586;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic bfloat, ptr addrspace(5) %slot syncscope("cluster") monotonic, align 2
+ ret bfloat %retval
+}
+
+define bfloat @monotonic_bfloat_local_gpu() {
+; CHECK-LABEL: monotonic_bfloat_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot587[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot587;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic bfloat, ptr addrspace(5) %slot syncscope("device") monotonic, align 2
+ ret bfloat %retval
+}
+
+define bfloat @monotonic_bfloat_local_sys_volatile() {
+; CHECK-LABEL: monotonic_bfloat_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot588[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot588;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile bfloat, ptr addrspace(5) %slot syncscope("") monotonic, align 2
+ ret bfloat %retval
+}
+
+define bfloat @monotonic_bfloat_local_cta_volatile() {
+; CHECK-LABEL: monotonic_bfloat_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot589[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot589;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile bfloat, ptr addrspace(5) %slot syncscope("block") monotonic, align 2
+ ret bfloat %retval
+}
+
+define bfloat @monotonic_bfloat_local_cluster_volatile() {
+; CHECK-LABEL: monotonic_bfloat_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot590[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot590;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile bfloat, ptr addrspace(5) %slot syncscope("cluster") monotonic, align 2
+ ret bfloat %retval
+}
+
+define bfloat @monotonic_bfloat_local_gpu_volatile() {
+; CHECK-LABEL: monotonic_bfloat_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot591[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot591;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile bfloat, ptr addrspace(5) %slot syncscope("device") monotonic, align 2
+ ret bfloat %retval
+}
+
+define float @monotonic_float_local_sys() {
+; CHECK-LABEL: monotonic_float_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot592[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot592;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic float, ptr addrspace(5) %slot syncscope("") monotonic, align 4
+ ret float %retval
+}
+
+define float @monotonic_float_local_cta() {
+; CHECK-LABEL: monotonic_float_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot593[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot593;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic float, ptr addrspace(5) %slot syncscope("block") monotonic, align 4
+ ret float %retval
+}
+
+define float @monotonic_float_local_cluster() {
+; CHECK-LABEL: monotonic_float_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot594[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot594;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic float, ptr addrspace(5) %slot syncscope("cluster") monotonic, align 4
+ ret float %retval
+}
+
+define float @monotonic_float_local_gpu() {
+; CHECK-LABEL: monotonic_float_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot595[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot595;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic float, ptr addrspace(5) %slot syncscope("device") monotonic, align 4
+ ret float %retval
+}
+
+define float @monotonic_float_local_sys_volatile() {
+; CHECK-LABEL: monotonic_float_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot596[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot596;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile float, ptr addrspace(5) %slot syncscope("") monotonic, align 4
+ ret float %retval
+}
+
+define float @monotonic_float_local_cta_volatile() {
+; CHECK-LABEL: monotonic_float_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot597[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot597;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile float, ptr addrspace(5) %slot syncscope("block") monotonic, align 4
+ ret float %retval
+}
+
+define float @monotonic_float_local_cluster_volatile() {
+; CHECK-LABEL: monotonic_float_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot598[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot598;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile float, ptr addrspace(5) %slot syncscope("cluster") monotonic, align 4
+ ret float %retval
+}
+
+define float @monotonic_float_local_gpu_volatile() {
+; CHECK-LABEL: monotonic_float_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot599[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot599;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile float, ptr addrspace(5) %slot syncscope("device") monotonic, align 4
+ ret float %retval
+}
+
+define double @monotonic_double_local_sys() {
+; CHECK-LABEL: monotonic_double_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot600[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot600;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic double, ptr addrspace(5) %slot syncscope("") monotonic, align 8
+ ret double %retval
+}
+
+define double @monotonic_double_local_cta() {
+; CHECK-LABEL: monotonic_double_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot601[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot601;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic double, ptr addrspace(5) %slot syncscope("block") monotonic, align 8
+ ret double %retval
+}
+
+define double @monotonic_double_local_cluster() {
+; CHECK-LABEL: monotonic_double_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot602[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot602;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic double, ptr addrspace(5) %slot syncscope("cluster") monotonic, align 8
+ ret double %retval
+}
+
+define double @monotonic_double_local_gpu() {
+; CHECK-LABEL: monotonic_double_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot603[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot603;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic double, ptr addrspace(5) %slot syncscope("device") monotonic, align 8
+ ret double %retval
+}
+
+define double @monotonic_double_local_sys_volatile() {
+; CHECK-LABEL: monotonic_double_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot604[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot604;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile double, ptr addrspace(5) %slot syncscope("") monotonic, align 8
+ ret double %retval
+}
+
+define double @monotonic_double_local_cta_volatile() {
+; CHECK-LABEL: monotonic_double_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot605[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot605;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile double, ptr addrspace(5) %slot syncscope("block") monotonic, align 8
+ ret double %retval
+}
+
+define double @monotonic_double_local_cluster_volatile() {
+; CHECK-LABEL: monotonic_double_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot606[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot606;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile double, ptr addrspace(5) %slot syncscope("cluster") monotonic, align 8
+ ret double %retval
+}
+
+define double @monotonic_double_local_gpu_volatile() {
+; CHECK-LABEL: monotonic_double_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot607[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot607;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile double, ptr addrspace(5) %slot syncscope("device") monotonic, align 8
+ ret double %retval
+}
+
+define i8 @acquire_i8_generic_sys(ptr %addr) {
+; CHECK-LABEL: acquire_i8_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i8_generic_sys_param_0];
+; CHECK-NEXT: ld.acquire.sys.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr %addr syncscope("") acquire, align 1
+ ret i8 %retval
+}
+
+define i8 @acquire_i8_generic_cta(ptr %addr) {
+; CHECK-LABEL: acquire_i8_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i8_generic_cta_param_0];
+; CHECK-NEXT: ld.acquire.cta.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr %addr syncscope("block") acquire, align 1
+ ret i8 %retval
+}
+
+define i8 @acquire_i8_generic_cluster(ptr %addr) {
+; CHECK-LABEL: acquire_i8_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i8_generic_cluster_param_0];
+; CHECK-NEXT: ld.acquire.cta.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr %addr syncscope("cluster") acquire, align 1
+ ret i8 %retval
+}
+
+define i8 @acquire_i8_generic_gpu(ptr %addr) {
+; CHECK-LABEL: acquire_i8_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i8_generic_gpu_param_0];
+; CHECK-NEXT: ld.acquire.gpu.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr %addr syncscope("device") acquire, align 1
+ ret i8 %retval
+}
+
+define i8 @acquire_i8_generic_sys_volatile(ptr %addr) {
+; CHECK-LABEL: acquire_i8_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i8_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr %addr syncscope("") acquire, align 1
+ ret i8 %retval
+}
+
+define i8 @acquire_i8_generic_cta_volatile(ptr %addr) {
+; CHECK-LABEL: acquire_i8_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i8_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr %addr syncscope("block") acquire, align 1
+ ret i8 %retval
+}
+
+define i8 @acquire_i8_generic_cluster_volatile(ptr %addr) {
+; CHECK-LABEL: acquire_i8_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i8_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr %addr syncscope("cluster") acquire, align 1
+ ret i8 %retval
+}
+
+define i8 @acquire_i8_generic_gpu_volatile(ptr %addr) {
+; CHECK-LABEL: acquire_i8_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i8_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr %addr syncscope("device") acquire, align 1
+ ret i8 %retval
+}
+
+define i16 @acquire_i16_generic_sys(ptr %addr) {
+; CHECK-LABEL: acquire_i16_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i16_generic_sys_param_0];
+; CHECK-NEXT: ld.acquire.sys.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr %addr syncscope("") acquire, align 2
+ ret i16 %retval
+}
+
+define i16 @acquire_i16_generic_cta(ptr %addr) {
+; CHECK-LABEL: acquire_i16_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i16_generic_cta_param_0];
+; CHECK-NEXT: ld.acquire.cta.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr %addr syncscope("block") acquire, align 2
+ ret i16 %retval
+}
+
+define i16 @acquire_i16_generic_cluster(ptr %addr) {
+; CHECK-LABEL: acquire_i16_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i16_generic_cluster_param_0];
+; CHECK-NEXT: ld.acquire.cta.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr %addr syncscope("cluster") acquire, align 2
+ ret i16 %retval
+}
+
+define i16 @acquire_i16_generic_gpu(ptr %addr) {
+; CHECK-LABEL: acquire_i16_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i16_generic_gpu_param_0];
+; CHECK-NEXT: ld.acquire.gpu.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr %addr syncscope("device") acquire, align 2
+ ret i16 %retval
+}
+
+define i16 @acquire_i16_generic_sys_volatile(ptr %addr) {
+; CHECK-LABEL: acquire_i16_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i16_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr %addr syncscope("") acquire, align 2
+ ret i16 %retval
+}
+
+define i16 @acquire_i16_generic_cta_volatile(ptr %addr) {
+; CHECK-LABEL: acquire_i16_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i16_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr %addr syncscope("block") acquire, align 2
+ ret i16 %retval
+}
+
+define i16 @acquire_i16_generic_cluster_volatile(ptr %addr) {
+; CHECK-LABEL: acquire_i16_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i16_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr %addr syncscope("cluster") acquire, align 2
+ ret i16 %retval
+}
+
+define i16 @acquire_i16_generic_gpu_volatile(ptr %addr) {
+; CHECK-LABEL: acquire_i16_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i16_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr %addr syncscope("device") acquire, align 2
+ ret i16 %retval
+}
+
+define i32 @acquire_i32_generic_sys(ptr %addr) {
+; CHECK-LABEL: acquire_i32_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i32_generic_sys_param_0];
+; CHECK-NEXT: ld.acquire.sys.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr %addr syncscope("") acquire, align 4
+ ret i32 %retval
+}
+
+define i32 @acquire_i32_generic_cta(ptr %addr) {
+; CHECK-LABEL: acquire_i32_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i32_generic_cta_param_0];
+; CHECK-NEXT: ld.acquire.cta.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr %addr syncscope("block") acquire, align 4
+ ret i32 %retval
+}
+
+define i32 @acquire_i32_generic_cluster(ptr %addr) {
+; CHECK-LABEL: acquire_i32_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i32_generic_cluster_param_0];
+; CHECK-NEXT: ld.acquire.cta.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr %addr syncscope("cluster") acquire, align 4
+ ret i32 %retval
+}
+
+define i32 @acquire_i32_generic_gpu(ptr %addr) {
+; CHECK-LABEL: acquire_i32_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i32_generic_gpu_param_0];
+; CHECK-NEXT: ld.acquire.gpu.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr %addr syncscope("device") acquire, align 4
+ ret i32 %retval
+}
+
+define i32 @acquire_i32_generic_sys_volatile(ptr %addr) {
+; CHECK-LABEL: acquire_i32_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i32_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr %addr syncscope("") acquire, align 4
+ ret i32 %retval
+}
+
+define i32 @acquire_i32_generic_cta_volatile(ptr %addr) {
+; CHECK-LABEL: acquire_i32_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i32_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr %addr syncscope("block") acquire, align 4
+ ret i32 %retval
+}
+
+define i32 @acquire_i32_generic_cluster_volatile(ptr %addr) {
+; CHECK-LABEL: acquire_i32_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i32_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr %addr syncscope("cluster") acquire, align 4
+ ret i32 %retval
+}
+
+define i32 @acquire_i32_generic_gpu_volatile(ptr %addr) {
+; CHECK-LABEL: acquire_i32_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i32_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr %addr syncscope("device") acquire, align 4
+ ret i32 %retval
+}
+
+define i64 @acquire_i64_generic_sys(ptr %addr) {
+; CHECK-LABEL: acquire_i64_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i64_generic_sys_param_0];
+; CHECK-NEXT: ld.acquire.sys.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr %addr syncscope("") acquire, align 8
+ ret i64 %retval
+}
+
+define i64 @acquire_i64_generic_cta(ptr %addr) {
+; CHECK-LABEL: acquire_i64_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i64_generic_cta_param_0];
+; CHECK-NEXT: ld.acquire.cta.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr %addr syncscope("block") acquire, align 8
+ ret i64 %retval
+}
+
+define i64 @acquire_i64_generic_cluster(ptr %addr) {
+; CHECK-LABEL: acquire_i64_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i64_generic_cluster_param_0];
+; CHECK-NEXT: ld.acquire.cta.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr %addr syncscope("cluster") acquire, align 8
+ ret i64 %retval
+}
+
+define i64 @acquire_i64_generic_gpu(ptr %addr) {
+; CHECK-LABEL: acquire_i64_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i64_generic_gpu_param_0];
+; CHECK-NEXT: ld.acquire.gpu.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr %addr syncscope("device") acquire, align 8
+ ret i64 %retval
+}
+
+define i64 @acquire_i64_generic_sys_volatile(ptr %addr) {
+; CHECK-LABEL: acquire_i64_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i64_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr %addr syncscope("") acquire, align 8
+ ret i64 %retval
+}
+
+define i64 @acquire_i64_generic_cta_volatile(ptr %addr) {
+; CHECK-LABEL: acquire_i64_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i64_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr %addr syncscope("block") acquire, align 8
+ ret i64 %retval
+}
+
+define i64 @acquire_i64_generic_cluster_volatile(ptr %addr) {
+; CHECK-LABEL: acquire_i64_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i64_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr %addr syncscope("cluster") acquire, align 8
+ ret i64 %retval
+}
+
+define i64 @acquire_i64_generic_gpu_volatile(ptr %addr) {
+; CHECK-LABEL: acquire_i64_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i64_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr %addr syncscope("device") acquire, align 8
+ ret i64 %retval
+}
+
+define half @acquire_half_generic_sys(ptr %addr) {
+; CHECK-LABEL: acquire_half_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_half_generic_sys_param_0];
+; CHECK-NEXT: ld.acquire.sys.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr %addr syncscope("") acquire, align 2
+ ret half %retval
+}
+
+define half @acquire_half_generic_cta(ptr %addr) {
+; CHECK-LABEL: acquire_half_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_half_generic_cta_param_0];
+; CHECK-NEXT: ld.acquire.cta.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr %addr syncscope("block") acquire, align 2
+ ret half %retval
+}
+
+define half @acquire_half_generic_cluster(ptr %addr) {
+; CHECK-LABEL: acquire_half_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_half_generic_cluster_param_0];
+; CHECK-NEXT: ld.acquire.cta.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr %addr syncscope("cluster") acquire, align 2
+ ret half %retval
+}
+
+define half @acquire_half_generic_gpu(ptr %addr) {
+; CHECK-LABEL: acquire_half_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_half_generic_gpu_param_0];
+; CHECK-NEXT: ld.acquire.gpu.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr %addr syncscope("device") acquire, align 2
+ ret half %retval
+}
+
+define half @acquire_half_generic_sys_volatile(ptr %addr) {
+; CHECK-LABEL: acquire_half_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_half_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr %addr syncscope("") acquire, align 2
+ ret half %retval
+}
+
+define half @acquire_half_generic_cta_volatile(ptr %addr) {
+; CHECK-LABEL: acquire_half_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_half_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr %addr syncscope("block") acquire, align 2
+ ret half %retval
+}
+
+define half @acquire_half_generic_cluster_volatile(ptr %addr) {
+; CHECK-LABEL: acquire_half_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_half_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr %addr syncscope("cluster") acquire, align 2
+ ret half %retval
+}
+
+define half @acquire_half_generic_gpu_volatile(ptr %addr) {
+; CHECK-LABEL: acquire_half_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_half_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr %addr syncscope("device") acquire, align 2
+ ret half %retval
+}
+
+define bfloat @acquire_bfloat_generic_sys(ptr %addr) {
+; CHECK-LABEL: acquire_bfloat_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_bfloat_generic_sys_param_0];
+; CHECK-NEXT: ld.acquire.sys.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr %addr syncscope("") acquire, align 2
+ ret bfloat %retval
+}
+
+define bfloat @acquire_bfloat_generic_cta(ptr %addr) {
+; CHECK-LABEL: acquire_bfloat_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_bfloat_generic_cta_param_0];
+; CHECK-NEXT: ld.acquire.cta.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr %addr syncscope("block") acquire, align 2
+ ret bfloat %retval
+}
+
+define bfloat @acquire_bfloat_generic_cluster(ptr %addr) {
+; CHECK-LABEL: acquire_bfloat_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_bfloat_generic_cluster_param_0];
+; CHECK-NEXT: ld.acquire.cta.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr %addr syncscope("cluster") acquire, align 2
+ ret bfloat %retval
+}
+
+define bfloat @acquire_bfloat_generic_gpu(ptr %addr) {
+; CHECK-LABEL: acquire_bfloat_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_bfloat_generic_gpu_param_0];
+; CHECK-NEXT: ld.acquire.gpu.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr %addr syncscope("device") acquire, align 2
+ ret bfloat %retval
+}
+
+define bfloat @acquire_bfloat_generic_sys_volatile(ptr %addr) {
+; CHECK-LABEL: acquire_bfloat_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_bfloat_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr %addr syncscope("") acquire, align 2
+ ret bfloat %retval
+}
+
+define bfloat @acquire_bfloat_generic_cta_volatile(ptr %addr) {
+; CHECK-LABEL: acquire_bfloat_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_bfloat_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr %addr syncscope("block") acquire, align 2
+ ret bfloat %retval
+}
+
+define bfloat @acquire_bfloat_generic_cluster_volatile(ptr %addr) {
+; CHECK-LABEL: acquire_bfloat_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_bfloat_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr %addr syncscope("cluster") acquire, align 2
+ ret bfloat %retval
+}
+
+define bfloat @acquire_bfloat_generic_gpu_volatile(ptr %addr) {
+; CHECK-LABEL: acquire_bfloat_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_bfloat_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr %addr syncscope("device") acquire, align 2
+ ret bfloat %retval
+}
+
+define float @acquire_float_generic_sys(ptr %addr) {
+; CHECK-LABEL: acquire_float_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_float_generic_sys_param_0];
+; CHECK-NEXT: ld.acquire.sys.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr %addr syncscope("") acquire, align 4
+ ret float %retval
+}
+
+define float @acquire_float_generic_cta(ptr %addr) {
+; CHECK-LABEL: acquire_float_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_float_generic_cta_param_0];
+; CHECK-NEXT: ld.acquire.cta.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr %addr syncscope("block") acquire, align 4
+ ret float %retval
+}
+
+define float @acquire_float_generic_cluster(ptr %addr) {
+; CHECK-LABEL: acquire_float_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_float_generic_cluster_param_0];
+; CHECK-NEXT: ld.acquire.cta.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr %addr syncscope("cluster") acquire, align 4
+ ret float %retval
+}
+
+define float @acquire_float_generic_gpu(ptr %addr) {
+; CHECK-LABEL: acquire_float_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_float_generic_gpu_param_0];
+; CHECK-NEXT: ld.acquire.gpu.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr %addr syncscope("device") acquire, align 4
+ ret float %retval
+}
+
+define float @acquire_float_generic_sys_volatile(ptr %addr) {
+; CHECK-LABEL: acquire_float_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_float_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr %addr syncscope("") acquire, align 4
+ ret float %retval
+}
+
+define float @acquire_float_generic_cta_volatile(ptr %addr) {
+; CHECK-LABEL: acquire_float_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_float_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr %addr syncscope("block") acquire, align 4
+ ret float %retval
+}
+
+define float @acquire_float_generic_cluster_volatile(ptr %addr) {
+; CHECK-LABEL: acquire_float_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_float_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr %addr syncscope("cluster") acquire, align 4
+ ret float %retval
+}
+
+define float @acquire_float_generic_gpu_volatile(ptr %addr) {
+; CHECK-LABEL: acquire_float_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_float_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr %addr syncscope("device") acquire, align 4
+ ret float %retval
+}
+
+define double @acquire_double_generic_sys(ptr %addr) {
+; CHECK-LABEL: acquire_double_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_double_generic_sys_param_0];
+; CHECK-NEXT: ld.acquire.sys.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr %addr syncscope("") acquire, align 8
+ ret double %retval
+}
+
+define double @acquire_double_generic_cta(ptr %addr) {
+; CHECK-LABEL: acquire_double_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_double_generic_cta_param_0];
+; CHECK-NEXT: ld.acquire.cta.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr %addr syncscope("block") acquire, align 8
+ ret double %retval
+}
+
+define double @acquire_double_generic_cluster(ptr %addr) {
+; CHECK-LABEL: acquire_double_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_double_generic_cluster_param_0];
+; CHECK-NEXT: ld.acquire.cta.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr %addr syncscope("cluster") acquire, align 8
+ ret double %retval
+}
+
+define double @acquire_double_generic_gpu(ptr %addr) {
+; CHECK-LABEL: acquire_double_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_double_generic_gpu_param_0];
+; CHECK-NEXT: ld.acquire.gpu.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr %addr syncscope("device") acquire, align 8
+ ret double %retval
+}
+
+define double @acquire_double_generic_sys_volatile(ptr %addr) {
+; CHECK-LABEL: acquire_double_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_double_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr %addr syncscope("") acquire, align 8
+ ret double %retval
+}
+
+define double @acquire_double_generic_cta_volatile(ptr %addr) {
+; CHECK-LABEL: acquire_double_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_double_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr %addr syncscope("block") acquire, align 8
+ ret double %retval
+}
+
+define double @acquire_double_generic_cluster_volatile(ptr %addr) {
+; CHECK-LABEL: acquire_double_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_double_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr %addr syncscope("cluster") acquire, align 8
+ ret double %retval
+}
+
+define double @acquire_double_generic_gpu_volatile(ptr %addr) {
+; CHECK-LABEL: acquire_double_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_double_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr %addr syncscope("device") acquire, align 8
+ ret double %retval
+}
+
+define i8 @acquire_i8_global_sys(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_i8_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i8_global_sys_param_0];
+; CHECK-NEXT: ld.acquire.sys.global.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr addrspace(1) %addr syncscope("") acquire, align 1
+ ret i8 %retval
+}
+
+define i8 @acquire_i8_global_cta(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_i8_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i8_global_cta_param_0];
+; CHECK-NEXT: ld.acquire.cta.global.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr addrspace(1) %addr syncscope("block") acquire, align 1
+ ret i8 %retval
+}
+
+define i8 @acquire_i8_global_cluster(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_i8_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i8_global_cluster_param_0];
+; CHECK-NEXT: ld.acquire.cta.global.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr addrspace(1) %addr syncscope("cluster") acquire, align 1
+ ret i8 %retval
+}
+
+define i8 @acquire_i8_global_gpu(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_i8_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i8_global_gpu_param_0];
+; CHECK-NEXT: ld.acquire.gpu.global.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr addrspace(1) %addr syncscope("device") acquire, align 1
+ ret i8 %retval
+}
+
+define i8 @acquire_i8_global_sys_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_i8_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i8_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.global.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr addrspace(1) %addr syncscope("") acquire, align 1
+ ret i8 %retval
+}
+
+define i8 @acquire_i8_global_cta_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_i8_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i8_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.global.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr addrspace(1) %addr syncscope("block") acquire, align 1
+ ret i8 %retval
+}
+
+define i8 @acquire_i8_global_cluster_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_i8_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i8_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.global.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr addrspace(1) %addr syncscope("cluster") acquire, align 1
+ ret i8 %retval
+}
+
+define i8 @acquire_i8_global_gpu_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_i8_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i8_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.global.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr addrspace(1) %addr syncscope("device") acquire, align 1
+ ret i8 %retval
+}
+
+define i16 @acquire_i16_global_sys(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_i16_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i16_global_sys_param_0];
+; CHECK-NEXT: ld.acquire.sys.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr addrspace(1) %addr syncscope("") acquire, align 2
+ ret i16 %retval
+}
+
+define i16 @acquire_i16_global_cta(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_i16_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i16_global_cta_param_0];
+; CHECK-NEXT: ld.acquire.cta.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr addrspace(1) %addr syncscope("block") acquire, align 2
+ ret i16 %retval
+}
+
+define i16 @acquire_i16_global_cluster(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_i16_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i16_global_cluster_param_0];
+; CHECK-NEXT: ld.acquire.cta.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr addrspace(1) %addr syncscope("cluster") acquire, align 2
+ ret i16 %retval
+}
+
+define i16 @acquire_i16_global_gpu(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_i16_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i16_global_gpu_param_0];
+; CHECK-NEXT: ld.acquire.gpu.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr addrspace(1) %addr syncscope("device") acquire, align 2
+ ret i16 %retval
+}
+
+define i16 @acquire_i16_global_sys_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_i16_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i16_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr addrspace(1) %addr syncscope("") acquire, align 2
+ ret i16 %retval
+}
+
+define i16 @acquire_i16_global_cta_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_i16_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i16_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr addrspace(1) %addr syncscope("block") acquire, align 2
+ ret i16 %retval
+}
+
+define i16 @acquire_i16_global_cluster_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_i16_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i16_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr addrspace(1) %addr syncscope("cluster") acquire, align 2
+ ret i16 %retval
+}
+
+define i16 @acquire_i16_global_gpu_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_i16_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i16_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr addrspace(1) %addr syncscope("device") acquire, align 2
+ ret i16 %retval
+}
+
+define i32 @acquire_i32_global_sys(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_i32_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i32_global_sys_param_0];
+; CHECK-NEXT: ld.acquire.sys.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr addrspace(1) %addr syncscope("") acquire, align 4
+ ret i32 %retval
+}
+
+define i32 @acquire_i32_global_cta(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_i32_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i32_global_cta_param_0];
+; CHECK-NEXT: ld.acquire.cta.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr addrspace(1) %addr syncscope("block") acquire, align 4
+ ret i32 %retval
+}
+
+define i32 @acquire_i32_global_cluster(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_i32_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i32_global_cluster_param_0];
+; CHECK-NEXT: ld.acquire.cta.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr addrspace(1) %addr syncscope("cluster") acquire, align 4
+ ret i32 %retval
+}
+
+define i32 @acquire_i32_global_gpu(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_i32_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i32_global_gpu_param_0];
+; CHECK-NEXT: ld.acquire.gpu.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr addrspace(1) %addr syncscope("device") acquire, align 4
+ ret i32 %retval
+}
+
+define i32 @acquire_i32_global_sys_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_i32_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i32_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr addrspace(1) %addr syncscope("") acquire, align 4
+ ret i32 %retval
+}
+
+define i32 @acquire_i32_global_cta_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_i32_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i32_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr addrspace(1) %addr syncscope("block") acquire, align 4
+ ret i32 %retval
+}
+
+define i32 @acquire_i32_global_cluster_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_i32_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i32_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr addrspace(1) %addr syncscope("cluster") acquire, align 4
+ ret i32 %retval
+}
+
+define i32 @acquire_i32_global_gpu_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_i32_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i32_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr addrspace(1) %addr syncscope("device") acquire, align 4
+ ret i32 %retval
+}
+
+define i64 @acquire_i64_global_sys(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_i64_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i64_global_sys_param_0];
+; CHECK-NEXT: ld.acquire.sys.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr addrspace(1) %addr syncscope("") acquire, align 8
+ ret i64 %retval
+}
+
+define i64 @acquire_i64_global_cta(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_i64_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i64_global_cta_param_0];
+; CHECK-NEXT: ld.acquire.cta.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr addrspace(1) %addr syncscope("block") acquire, align 8
+ ret i64 %retval
+}
+
+define i64 @acquire_i64_global_cluster(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_i64_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i64_global_cluster_param_0];
+; CHECK-NEXT: ld.acquire.cta.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr addrspace(1) %addr syncscope("cluster") acquire, align 8
+ ret i64 %retval
+}
+
+define i64 @acquire_i64_global_gpu(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_i64_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i64_global_gpu_param_0];
+; CHECK-NEXT: ld.acquire.gpu.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr addrspace(1) %addr syncscope("device") acquire, align 8
+ ret i64 %retval
+}
+
+define i64 @acquire_i64_global_sys_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_i64_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i64_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr addrspace(1) %addr syncscope("") acquire, align 8
+ ret i64 %retval
+}
+
+define i64 @acquire_i64_global_cta_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_i64_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i64_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr addrspace(1) %addr syncscope("block") acquire, align 8
+ ret i64 %retval
+}
+
+define i64 @acquire_i64_global_cluster_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_i64_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i64_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr addrspace(1) %addr syncscope("cluster") acquire, align 8
+ ret i64 %retval
+}
+
+define i64 @acquire_i64_global_gpu_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_i64_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i64_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr addrspace(1) %addr syncscope("device") acquire, align 8
+ ret i64 %retval
+}
+
+define half @acquire_half_global_sys(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_half_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_half_global_sys_param_0];
+; CHECK-NEXT: ld.acquire.sys.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr addrspace(1) %addr syncscope("") acquire, align 2
+ ret half %retval
+}
+
+define half @acquire_half_global_cta(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_half_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_half_global_cta_param_0];
+; CHECK-NEXT: ld.acquire.cta.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr addrspace(1) %addr syncscope("block") acquire, align 2
+ ret half %retval
+}
+
+define half @acquire_half_global_cluster(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_half_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_half_global_cluster_param_0];
+; CHECK-NEXT: ld.acquire.cta.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr addrspace(1) %addr syncscope("cluster") acquire, align 2
+ ret half %retval
+}
+
+define half @acquire_half_global_gpu(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_half_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_half_global_gpu_param_0];
+; CHECK-NEXT: ld.acquire.gpu.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr addrspace(1) %addr syncscope("device") acquire, align 2
+ ret half %retval
+}
+
+define half @acquire_half_global_sys_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_half_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_half_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr addrspace(1) %addr syncscope("") acquire, align 2
+ ret half %retval
+}
+
+define half @acquire_half_global_cta_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_half_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_half_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr addrspace(1) %addr syncscope("block") acquire, align 2
+ ret half %retval
+}
+
+define half @acquire_half_global_cluster_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_half_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_half_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr addrspace(1) %addr syncscope("cluster") acquire, align 2
+ ret half %retval
+}
+
+define half @acquire_half_global_gpu_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_half_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_half_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr addrspace(1) %addr syncscope("device") acquire, align 2
+ ret half %retval
+}
+
+define bfloat @acquire_bfloat_global_sys(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_bfloat_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_bfloat_global_sys_param_0];
+; CHECK-NEXT: ld.acquire.sys.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr addrspace(1) %addr syncscope("") acquire, align 2
+ ret bfloat %retval
+}
+
+define bfloat @acquire_bfloat_global_cta(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_bfloat_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_bfloat_global_cta_param_0];
+; CHECK-NEXT: ld.acquire.cta.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr addrspace(1) %addr syncscope("block") acquire, align 2
+ ret bfloat %retval
+}
+
+define bfloat @acquire_bfloat_global_cluster(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_bfloat_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_bfloat_global_cluster_param_0];
+; CHECK-NEXT: ld.acquire.cta.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr addrspace(1) %addr syncscope("cluster") acquire, align 2
+ ret bfloat %retval
+}
+
+define bfloat @acquire_bfloat_global_gpu(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_bfloat_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_bfloat_global_gpu_param_0];
+; CHECK-NEXT: ld.acquire.gpu.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr addrspace(1) %addr syncscope("device") acquire, align 2
+ ret bfloat %retval
+}
+
+define bfloat @acquire_bfloat_global_sys_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_bfloat_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_bfloat_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr addrspace(1) %addr syncscope("") acquire, align 2
+ ret bfloat %retval
+}
+
+define bfloat @acquire_bfloat_global_cta_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_bfloat_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_bfloat_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr addrspace(1) %addr syncscope("block") acquire, align 2
+ ret bfloat %retval
+}
+
+define bfloat @acquire_bfloat_global_cluster_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_bfloat_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_bfloat_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr addrspace(1) %addr syncscope("cluster") acquire, align 2
+ ret bfloat %retval
+}
+
+define bfloat @acquire_bfloat_global_gpu_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_bfloat_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_bfloat_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr addrspace(1) %addr syncscope("device") acquire, align 2
+ ret bfloat %retval
+}
+
+define float @acquire_float_global_sys(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_float_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_float_global_sys_param_0];
+; CHECK-NEXT: ld.acquire.sys.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr addrspace(1) %addr syncscope("") acquire, align 4
+ ret float %retval
+}
+
+define float @acquire_float_global_cta(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_float_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_float_global_cta_param_0];
+; CHECK-NEXT: ld.acquire.cta.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr addrspace(1) %addr syncscope("block") acquire, align 4
+ ret float %retval
+}
+
+define float @acquire_float_global_cluster(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_float_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_float_global_cluster_param_0];
+; CHECK-NEXT: ld.acquire.cta.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr addrspace(1) %addr syncscope("cluster") acquire, align 4
+ ret float %retval
+}
+
+define float @acquire_float_global_gpu(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_float_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_float_global_gpu_param_0];
+; CHECK-NEXT: ld.acquire.gpu.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr addrspace(1) %addr syncscope("device") acquire, align 4
+ ret float %retval
+}
+
+define float @acquire_float_global_sys_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_float_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_float_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr addrspace(1) %addr syncscope("") acquire, align 4
+ ret float %retval
+}
+
+define float @acquire_float_global_cta_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_float_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_float_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr addrspace(1) %addr syncscope("block") acquire, align 4
+ ret float %retval
+}
+
+define float @acquire_float_global_cluster_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_float_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_float_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr addrspace(1) %addr syncscope("cluster") acquire, align 4
+ ret float %retval
+}
+
+define float @acquire_float_global_gpu_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_float_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_float_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr addrspace(1) %addr syncscope("device") acquire, align 4
+ ret float %retval
+}
+
+define double @acquire_double_global_sys(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_double_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_double_global_sys_param_0];
+; CHECK-NEXT: ld.acquire.sys.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr addrspace(1) %addr syncscope("") acquire, align 8
+ ret double %retval
+}
+
+define double @acquire_double_global_cta(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_double_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_double_global_cta_param_0];
+; CHECK-NEXT: ld.acquire.cta.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr addrspace(1) %addr syncscope("block") acquire, align 8
+ ret double %retval
+}
+
+define double @acquire_double_global_cluster(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_double_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_double_global_cluster_param_0];
+; CHECK-NEXT: ld.acquire.cta.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr addrspace(1) %addr syncscope("cluster") acquire, align 8
+ ret double %retval
+}
+
+define double @acquire_double_global_gpu(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_double_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_double_global_gpu_param_0];
+; CHECK-NEXT: ld.acquire.gpu.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr addrspace(1) %addr syncscope("device") acquire, align 8
+ ret double %retval
+}
+
+define double @acquire_double_global_sys_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_double_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_double_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr addrspace(1) %addr syncscope("") acquire, align 8
+ ret double %retval
+}
+
+define double @acquire_double_global_cta_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_double_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_double_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr addrspace(1) %addr syncscope("block") acquire, align 8
+ ret double %retval
+}
+
+define double @acquire_double_global_cluster_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_double_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_double_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr addrspace(1) %addr syncscope("cluster") acquire, align 8
+ ret double %retval
+}
+
+define double @acquire_double_global_gpu_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_double_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_double_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr addrspace(1) %addr syncscope("device") acquire, align 8
+ ret double %retval
+}
+
+define i8 @acquire_i8_shared_sys(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_i8_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i8_shared_sys_param_0];
+; CHECK-NEXT: ld.acquire.sys.shared.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr addrspace(3) %addr syncscope("") acquire, align 1
+ ret i8 %retval
+}
+
+define i8 @acquire_i8_shared_cta(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_i8_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i8_shared_cta_param_0];
+; CHECK-NEXT: ld.acquire.cta.shared.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr addrspace(3) %addr syncscope("block") acquire, align 1
+ ret i8 %retval
+}
+
+define i8 @acquire_i8_shared_cluster(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_i8_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i8_shared_cluster_param_0];
+; CHECK-NEXT: ld.acquire.cta.shared.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr addrspace(3) %addr syncscope("cluster") acquire, align 1
+ ret i8 %retval
+}
+
+define i8 @acquire_i8_shared_gpu(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_i8_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i8_shared_gpu_param_0];
+; CHECK-NEXT: ld.acquire.gpu.shared.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr addrspace(3) %addr syncscope("device") acquire, align 1
+ ret i8 %retval
+}
+
+define i8 @acquire_i8_shared_sys_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_i8_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i8_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.shared.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr addrspace(3) %addr syncscope("") acquire, align 1
+ ret i8 %retval
+}
+
+define i8 @acquire_i8_shared_cta_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_i8_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i8_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.shared.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr addrspace(3) %addr syncscope("block") acquire, align 1
+ ret i8 %retval
+}
+
+define i8 @acquire_i8_shared_cluster_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_i8_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i8_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.shared.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr addrspace(3) %addr syncscope("cluster") acquire, align 1
+ ret i8 %retval
+}
+
+define i8 @acquire_i8_shared_gpu_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_i8_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i8_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.shared.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr addrspace(3) %addr syncscope("device") acquire, align 1
+ ret i8 %retval
+}
+
+define i16 @acquire_i16_shared_sys(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_i16_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i16_shared_sys_param_0];
+; CHECK-NEXT: ld.acquire.sys.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr addrspace(3) %addr syncscope("") acquire, align 2
+ ret i16 %retval
+}
+
+define i16 @acquire_i16_shared_cta(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_i16_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i16_shared_cta_param_0];
+; CHECK-NEXT: ld.acquire.cta.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr addrspace(3) %addr syncscope("block") acquire, align 2
+ ret i16 %retval
+}
+
+define i16 @acquire_i16_shared_cluster(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_i16_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i16_shared_cluster_param_0];
+; CHECK-NEXT: ld.acquire.cta.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr addrspace(3) %addr syncscope("cluster") acquire, align 2
+ ret i16 %retval
+}
+
+define i16 @acquire_i16_shared_gpu(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_i16_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i16_shared_gpu_param_0];
+; CHECK-NEXT: ld.acquire.gpu.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr addrspace(3) %addr syncscope("device") acquire, align 2
+ ret i16 %retval
+}
+
+define i16 @acquire_i16_shared_sys_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_i16_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i16_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr addrspace(3) %addr syncscope("") acquire, align 2
+ ret i16 %retval
+}
+
+define i16 @acquire_i16_shared_cta_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_i16_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i16_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr addrspace(3) %addr syncscope("block") acquire, align 2
+ ret i16 %retval
+}
+
+define i16 @acquire_i16_shared_cluster_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_i16_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i16_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr addrspace(3) %addr syncscope("cluster") acquire, align 2
+ ret i16 %retval
+}
+
+define i16 @acquire_i16_shared_gpu_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_i16_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i16_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr addrspace(3) %addr syncscope("device") acquire, align 2
+ ret i16 %retval
+}
+
+define i32 @acquire_i32_shared_sys(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_i32_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i32_shared_sys_param_0];
+; CHECK-NEXT: ld.acquire.sys.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr addrspace(3) %addr syncscope("") acquire, align 4
+ ret i32 %retval
+}
+
+define i32 @acquire_i32_shared_cta(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_i32_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i32_shared_cta_param_0];
+; CHECK-NEXT: ld.acquire.cta.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr addrspace(3) %addr syncscope("block") acquire, align 4
+ ret i32 %retval
+}
+
+define i32 @acquire_i32_shared_cluster(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_i32_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i32_shared_cluster_param_0];
+; CHECK-NEXT: ld.acquire.cta.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr addrspace(3) %addr syncscope("cluster") acquire, align 4
+ ret i32 %retval
+}
+
+define i32 @acquire_i32_shared_gpu(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_i32_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i32_shared_gpu_param_0];
+; CHECK-NEXT: ld.acquire.gpu.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr addrspace(3) %addr syncscope("device") acquire, align 4
+ ret i32 %retval
+}
+
+define i32 @acquire_i32_shared_sys_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_i32_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i32_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr addrspace(3) %addr syncscope("") acquire, align 4
+ ret i32 %retval
+}
+
+define i32 @acquire_i32_shared_cta_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_i32_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i32_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr addrspace(3) %addr syncscope("block") acquire, align 4
+ ret i32 %retval
+}
+
+define i32 @acquire_i32_shared_cluster_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_i32_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i32_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr addrspace(3) %addr syncscope("cluster") acquire, align 4
+ ret i32 %retval
+}
+
+define i32 @acquire_i32_shared_gpu_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_i32_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i32_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr addrspace(3) %addr syncscope("device") acquire, align 4
+ ret i32 %retval
+}
+
+define i64 @acquire_i64_shared_sys(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_i64_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i64_shared_sys_param_0];
+; CHECK-NEXT: ld.acquire.sys.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr addrspace(3) %addr syncscope("") acquire, align 8
+ ret i64 %retval
+}
+
+define i64 @acquire_i64_shared_cta(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_i64_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i64_shared_cta_param_0];
+; CHECK-NEXT: ld.acquire.cta.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr addrspace(3) %addr syncscope("block") acquire, align 8
+ ret i64 %retval
+}
+
+define i64 @acquire_i64_shared_cluster(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_i64_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i64_shared_cluster_param_0];
+; CHECK-NEXT: ld.acquire.cta.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr addrspace(3) %addr syncscope("cluster") acquire, align 8
+ ret i64 %retval
+}
+
+define i64 @acquire_i64_shared_gpu(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_i64_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i64_shared_gpu_param_0];
+; CHECK-NEXT: ld.acquire.gpu.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr addrspace(3) %addr syncscope("device") acquire, align 8
+ ret i64 %retval
+}
+
+define i64 @acquire_i64_shared_sys_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_i64_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i64_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr addrspace(3) %addr syncscope("") acquire, align 8
+ ret i64 %retval
+}
+
+define i64 @acquire_i64_shared_cta_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_i64_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i64_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr addrspace(3) %addr syncscope("block") acquire, align 8
+ ret i64 %retval
+}
+
+define i64 @acquire_i64_shared_cluster_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_i64_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i64_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr addrspace(3) %addr syncscope("cluster") acquire, align 8
+ ret i64 %retval
+}
+
+define i64 @acquire_i64_shared_gpu_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_i64_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i64_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr addrspace(3) %addr syncscope("device") acquire, align 8
+ ret i64 %retval
+}
+
+define half @acquire_half_shared_sys(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_half_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_half_shared_sys_param_0];
+; CHECK-NEXT: ld.acquire.sys.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr addrspace(3) %addr syncscope("") acquire, align 2
+ ret half %retval
+}
+
+define half @acquire_half_shared_cta(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_half_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_half_shared_cta_param_0];
+; CHECK-NEXT: ld.acquire.cta.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr addrspace(3) %addr syncscope("block") acquire, align 2
+ ret half %retval
+}
+
+define half @acquire_half_shared_cluster(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_half_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_half_shared_cluster_param_0];
+; CHECK-NEXT: ld.acquire.cta.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr addrspace(3) %addr syncscope("cluster") acquire, align 2
+ ret half %retval
+}
+
+define half @acquire_half_shared_gpu(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_half_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_half_shared_gpu_param_0];
+; CHECK-NEXT: ld.acquire.gpu.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr addrspace(3) %addr syncscope("device") acquire, align 2
+ ret half %retval
+}
+
+define half @acquire_half_shared_sys_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_half_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_half_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr addrspace(3) %addr syncscope("") acquire, align 2
+ ret half %retval
+}
+
+define half @acquire_half_shared_cta_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_half_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_half_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr addrspace(3) %addr syncscope("block") acquire, align 2
+ ret half %retval
+}
+
+define half @acquire_half_shared_cluster_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_half_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_half_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr addrspace(3) %addr syncscope("cluster") acquire, align 2
+ ret half %retval
+}
+
+define half @acquire_half_shared_gpu_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_half_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_half_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr addrspace(3) %addr syncscope("device") acquire, align 2
+ ret half %retval
+}
+
+define bfloat @acquire_bfloat_shared_sys(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_bfloat_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_bfloat_shared_sys_param_0];
+; CHECK-NEXT: ld.acquire.sys.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr addrspace(3) %addr syncscope("") acquire, align 2
+ ret bfloat %retval
+}
+
+define bfloat @acquire_bfloat_shared_cta(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_bfloat_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_bfloat_shared_cta_param_0];
+; CHECK-NEXT: ld.acquire.cta.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr addrspace(3) %addr syncscope("block") acquire, align 2
+ ret bfloat %retval
+}
+
+define bfloat @acquire_bfloat_shared_cluster(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_bfloat_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_bfloat_shared_cluster_param_0];
+; CHECK-NEXT: ld.acquire.cta.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr addrspace(3) %addr syncscope("cluster") acquire, align 2
+ ret bfloat %retval
+}
+
+define bfloat @acquire_bfloat_shared_gpu(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_bfloat_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_bfloat_shared_gpu_param_0];
+; CHECK-NEXT: ld.acquire.gpu.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr addrspace(3) %addr syncscope("device") acquire, align 2
+ ret bfloat %retval
+}
+
+define bfloat @acquire_bfloat_shared_sys_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_bfloat_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_bfloat_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr addrspace(3) %addr syncscope("") acquire, align 2
+ ret bfloat %retval
+}
+
+define bfloat @acquire_bfloat_shared_cta_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_bfloat_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_bfloat_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr addrspace(3) %addr syncscope("block") acquire, align 2
+ ret bfloat %retval
+}
+
+define bfloat @acquire_bfloat_shared_cluster_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_bfloat_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_bfloat_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr addrspace(3) %addr syncscope("cluster") acquire, align 2
+ ret bfloat %retval
+}
+
+define bfloat @acquire_bfloat_shared_gpu_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_bfloat_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_bfloat_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr addrspace(3) %addr syncscope("device") acquire, align 2
+ ret bfloat %retval
+}
+
+define float @acquire_float_shared_sys(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_float_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_float_shared_sys_param_0];
+; CHECK-NEXT: ld.acquire.sys.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr addrspace(3) %addr syncscope("") acquire, align 4
+ ret float %retval
+}
+
+define float @acquire_float_shared_cta(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_float_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_float_shared_cta_param_0];
+; CHECK-NEXT: ld.acquire.cta.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr addrspace(3) %addr syncscope("block") acquire, align 4
+ ret float %retval
+}
+
+define float @acquire_float_shared_cluster(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_float_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_float_shared_cluster_param_0];
+; CHECK-NEXT: ld.acquire.cta.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr addrspace(3) %addr syncscope("cluster") acquire, align 4
+ ret float %retval
+}
+
+define float @acquire_float_shared_gpu(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_float_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_float_shared_gpu_param_0];
+; CHECK-NEXT: ld.acquire.gpu.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr addrspace(3) %addr syncscope("device") acquire, align 4
+ ret float %retval
+}
+
+define float @acquire_float_shared_sys_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_float_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_float_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr addrspace(3) %addr syncscope("") acquire, align 4
+ ret float %retval
+}
+
+define float @acquire_float_shared_cta_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_float_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_float_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr addrspace(3) %addr syncscope("block") acquire, align 4
+ ret float %retval
+}
+
+define float @acquire_float_shared_cluster_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_float_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_float_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr addrspace(3) %addr syncscope("cluster") acquire, align 4
+ ret float %retval
+}
+
+define float @acquire_float_shared_gpu_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_float_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_float_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr addrspace(3) %addr syncscope("device") acquire, align 4
+ ret float %retval
+}
+
+define double @acquire_double_shared_sys(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_double_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_double_shared_sys_param_0];
+; CHECK-NEXT: ld.acquire.sys.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr addrspace(3) %addr syncscope("") acquire, align 8
+ ret double %retval
+}
+
+define double @acquire_double_shared_cta(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_double_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_double_shared_cta_param_0];
+; CHECK-NEXT: ld.acquire.cta.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr addrspace(3) %addr syncscope("block") acquire, align 8
+ ret double %retval
+}
+
+define double @acquire_double_shared_cluster(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_double_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_double_shared_cluster_param_0];
+; CHECK-NEXT: ld.acquire.cta.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr addrspace(3) %addr syncscope("cluster") acquire, align 8
+ ret double %retval
+}
+
+define double @acquire_double_shared_gpu(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_double_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_double_shared_gpu_param_0];
+; CHECK-NEXT: ld.acquire.gpu.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr addrspace(3) %addr syncscope("device") acquire, align 8
+ ret double %retval
+}
+
+define double @acquire_double_shared_sys_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_double_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_double_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr addrspace(3) %addr syncscope("") acquire, align 8
+ ret double %retval
+}
+
+define double @acquire_double_shared_cta_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_double_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_double_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr addrspace(3) %addr syncscope("block") acquire, align 8
+ ret double %retval
+}
+
+define double @acquire_double_shared_cluster_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_double_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_double_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr addrspace(3) %addr syncscope("cluster") acquire, align 8
+ ret double %retval
+}
+
+define double @acquire_double_shared_gpu_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_double_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_double_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr addrspace(3) %addr syncscope("device") acquire, align 8
+ ret double %retval
+}
+
+define i8 @acquire_i8_local_sys() {
+; CHECK-LABEL: acquire_i8_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot800[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot800;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b8 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i8, ptr addrspace(5) %slot syncscope("") acquire, align 1
+ ret i8 %retval
+}
+
+define i8 @acquire_i8_local_cta() {
+; CHECK-LABEL: acquire_i8_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot801[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot801;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b8 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i8, ptr addrspace(5) %slot syncscope("block") acquire, align 1
+ ret i8 %retval
+}
+
+define i8 @acquire_i8_local_cluster() {
+; CHECK-LABEL: acquire_i8_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot802[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot802;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b8 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i8, ptr addrspace(5) %slot syncscope("cluster") acquire, align 1
+ ret i8 %retval
+}
+
+define i8 @acquire_i8_local_gpu() {
+; CHECK-LABEL: acquire_i8_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot803[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot803;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b8 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i8, ptr addrspace(5) %slot syncscope("device") acquire, align 1
+ ret i8 %retval
+}
+
+define i8 @acquire_i8_local_sys_volatile() {
+; CHECK-LABEL: acquire_i8_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot804[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot804;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b8 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i8, ptr addrspace(5) %slot syncscope("") acquire, align 1
+ ret i8 %retval
+}
+
+define i8 @acquire_i8_local_cta_volatile() {
+; CHECK-LABEL: acquire_i8_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot805[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot805;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b8 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i8, ptr addrspace(5) %slot syncscope("block") acquire, align 1
+ ret i8 %retval
+}
+
+define i8 @acquire_i8_local_cluster_volatile() {
+; CHECK-LABEL: acquire_i8_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot806[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot806;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b8 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i8, ptr addrspace(5) %slot syncscope("cluster") acquire, align 1
+ ret i8 %retval
+}
+
+define i8 @acquire_i8_local_gpu_volatile() {
+; CHECK-LABEL: acquire_i8_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot807[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot807;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b8 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i8, ptr addrspace(5) %slot syncscope("device") acquire, align 1
+ ret i8 %retval
+}
+
+define i16 @acquire_i16_local_sys() {
+; CHECK-LABEL: acquire_i16_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot808[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot808;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i16, ptr addrspace(5) %slot syncscope("") acquire, align 2
+ ret i16 %retval
+}
+
+define i16 @acquire_i16_local_cta() {
+; CHECK-LABEL: acquire_i16_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot809[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot809;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i16, ptr addrspace(5) %slot syncscope("block") acquire, align 2
+ ret i16 %retval
+}
+
+define i16 @acquire_i16_local_cluster() {
+; CHECK-LABEL: acquire_i16_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot810[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot810;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i16, ptr addrspace(5) %slot syncscope("cluster") acquire, align 2
+ ret i16 %retval
+}
+
+define i16 @acquire_i16_local_gpu() {
+; CHECK-LABEL: acquire_i16_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot811[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot811;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i16, ptr addrspace(5) %slot syncscope("device") acquire, align 2
+ ret i16 %retval
+}
+
+define i16 @acquire_i16_local_sys_volatile() {
+; CHECK-LABEL: acquire_i16_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot812[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot812;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i16, ptr addrspace(5) %slot syncscope("") acquire, align 2
+ ret i16 %retval
+}
+
+define i16 @acquire_i16_local_cta_volatile() {
+; CHECK-LABEL: acquire_i16_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot813[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot813;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i16, ptr addrspace(5) %slot syncscope("block") acquire, align 2
+ ret i16 %retval
+}
+
+define i16 @acquire_i16_local_cluster_volatile() {
+; CHECK-LABEL: acquire_i16_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot814[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot814;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i16, ptr addrspace(5) %slot syncscope("cluster") acquire, align 2
+ ret i16 %retval
+}
+
+define i16 @acquire_i16_local_gpu_volatile() {
+; CHECK-LABEL: acquire_i16_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot815[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot815;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i16, ptr addrspace(5) %slot syncscope("device") acquire, align 2
+ ret i16 %retval
+}
+
+define i32 @acquire_i32_local_sys() {
+; CHECK-LABEL: acquire_i32_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot816[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot816;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i32, ptr addrspace(5) %slot syncscope("") acquire, align 4
+ ret i32 %retval
+}
+
+define i32 @acquire_i32_local_cta() {
+; CHECK-LABEL: acquire_i32_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot817[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot817;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i32, ptr addrspace(5) %slot syncscope("block") acquire, align 4
+ ret i32 %retval
+}
+
+define i32 @acquire_i32_local_cluster() {
+; CHECK-LABEL: acquire_i32_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot818[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot818;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i32, ptr addrspace(5) %slot syncscope("cluster") acquire, align 4
+ ret i32 %retval
+}
+
+define i32 @acquire_i32_local_gpu() {
+; CHECK-LABEL: acquire_i32_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot819[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot819;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i32, ptr addrspace(5) %slot syncscope("device") acquire, align 4
+ ret i32 %retval
+}
+
+define i32 @acquire_i32_local_sys_volatile() {
+; CHECK-LABEL: acquire_i32_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot820[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot820;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i32, ptr addrspace(5) %slot syncscope("") acquire, align 4
+ ret i32 %retval
+}
+
+define i32 @acquire_i32_local_cta_volatile() {
+; CHECK-LABEL: acquire_i32_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot821[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot821;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i32, ptr addrspace(5) %slot syncscope("block") acquire, align 4
+ ret i32 %retval
+}
+
+define i32 @acquire_i32_local_cluster_volatile() {
+; CHECK-LABEL: acquire_i32_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot822[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot822;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i32, ptr addrspace(5) %slot syncscope("cluster") acquire, align 4
+ ret i32 %retval
+}
+
+define i32 @acquire_i32_local_gpu_volatile() {
+; CHECK-LABEL: acquire_i32_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot823[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot823;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i32, ptr addrspace(5) %slot syncscope("device") acquire, align 4
+ ret i32 %retval
+}
+
+define i64 @acquire_i64_local_sys() {
+; CHECK-LABEL: acquire_i64_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot824[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot824;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i64, ptr addrspace(5) %slot syncscope("") acquire, align 8
+ ret i64 %retval
+}
+
+define i64 @acquire_i64_local_cta() {
+; CHECK-LABEL: acquire_i64_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot825[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot825;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i64, ptr addrspace(5) %slot syncscope("block") acquire, align 8
+ ret i64 %retval
+}
+
+define i64 @acquire_i64_local_cluster() {
+; CHECK-LABEL: acquire_i64_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot826[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot826;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i64, ptr addrspace(5) %slot syncscope("cluster") acquire, align 8
+ ret i64 %retval
+}
+
+define i64 @acquire_i64_local_gpu() {
+; CHECK-LABEL: acquire_i64_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot827[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot827;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i64, ptr addrspace(5) %slot syncscope("device") acquire, align 8
+ ret i64 %retval
+}
+
+define i64 @acquire_i64_local_sys_volatile() {
+; CHECK-LABEL: acquire_i64_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot828[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot828;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i64, ptr addrspace(5) %slot syncscope("") acquire, align 8
+ ret i64 %retval
+}
+
+define i64 @acquire_i64_local_cta_volatile() {
+; CHECK-LABEL: acquire_i64_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot829[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot829;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i64, ptr addrspace(5) %slot syncscope("block") acquire, align 8
+ ret i64 %retval
+}
+
+define i64 @acquire_i64_local_cluster_volatile() {
+; CHECK-LABEL: acquire_i64_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot830[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot830;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i64, ptr addrspace(5) %slot syncscope("cluster") acquire, align 8
+ ret i64 %retval
+}
+
+define i64 @acquire_i64_local_gpu_volatile() {
+; CHECK-LABEL: acquire_i64_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot831[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot831;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i64, ptr addrspace(5) %slot syncscope("device") acquire, align 8
+ ret i64 %retval
+}
+
+define half @acquire_half_local_sys() {
+; CHECK-LABEL: acquire_half_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot832[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot832;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic half, ptr addrspace(5) %slot syncscope("") acquire, align 2
+ ret half %retval
+}
+
+define half @acquire_half_local_cta() {
+; CHECK-LABEL: acquire_half_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot833[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot833;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic half, ptr addrspace(5) %slot syncscope("block") acquire, align 2
+ ret half %retval
+}
+
+define half @acquire_half_local_cluster() {
+; CHECK-LABEL: acquire_half_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot834[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot834;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic half, ptr addrspace(5) %slot syncscope("cluster") acquire, align 2
+ ret half %retval
+}
+
+define half @acquire_half_local_gpu() {
+; CHECK-LABEL: acquire_half_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot835[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot835;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic half, ptr addrspace(5) %slot syncscope("device") acquire, align 2
+ ret half %retval
+}
+
+define half @acquire_half_local_sys_volatile() {
+; CHECK-LABEL: acquire_half_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot836[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot836;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile half, ptr addrspace(5) %slot syncscope("") acquire, align 2
+ ret half %retval
+}
+
+define half @acquire_half_local_cta_volatile() {
+; CHECK-LABEL: acquire_half_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot837[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot837;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile half, ptr addrspace(5) %slot syncscope("block") acquire, align 2
+ ret half %retval
+}
+
+define half @acquire_half_local_cluster_volatile() {
+; CHECK-LABEL: acquire_half_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot838[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot838;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile half, ptr addrspace(5) %slot syncscope("cluster") acquire, align 2
+ ret half %retval
+}
+
+define half @acquire_half_local_gpu_volatile() {
+; CHECK-LABEL: acquire_half_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot839[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot839;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile half, ptr addrspace(5) %slot syncscope("device") acquire, align 2
+ ret half %retval
+}
+
+define bfloat @acquire_bfloat_local_sys() {
+; CHECK-LABEL: acquire_bfloat_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot840[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot840;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic bfloat, ptr addrspace(5) %slot syncscope("") acquire, align 2
+ ret bfloat %retval
+}
+
+define bfloat @acquire_bfloat_local_cta() {
+; CHECK-LABEL: acquire_bfloat_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot841[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot841;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic bfloat, ptr addrspace(5) %slot syncscope("block") acquire, align 2
+ ret bfloat %retval
+}
+
+define bfloat @acquire_bfloat_local_cluster() {
+; CHECK-LABEL: acquire_bfloat_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot842[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot842;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic bfloat, ptr addrspace(5) %slot syncscope("cluster") acquire, align 2
+ ret bfloat %retval
+}
+
+define bfloat @acquire_bfloat_local_gpu() {
+; CHECK-LABEL: acquire_bfloat_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot843[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot843;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic bfloat, ptr addrspace(5) %slot syncscope("device") acquire, align 2
+ ret bfloat %retval
+}
+
+define bfloat @acquire_bfloat_local_sys_volatile() {
+; CHECK-LABEL: acquire_bfloat_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot844[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot844;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile bfloat, ptr addrspace(5) %slot syncscope("") acquire, align 2
+ ret bfloat %retval
+}
+
+define bfloat @acquire_bfloat_local_cta_volatile() {
+; CHECK-LABEL: acquire_bfloat_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot845[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot845;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile bfloat, ptr addrspace(5) %slot syncscope("block") acquire, align 2
+ ret bfloat %retval
+}
+
+define bfloat @acquire_bfloat_local_cluster_volatile() {
+; CHECK-LABEL: acquire_bfloat_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot846[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot846;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile bfloat, ptr addrspace(5) %slot syncscope("cluster") acquire, align 2
+ ret bfloat %retval
+}
+
+define bfloat @acquire_bfloat_local_gpu_volatile() {
+; CHECK-LABEL: acquire_bfloat_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot847[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot847;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile bfloat, ptr addrspace(5) %slot syncscope("device") acquire, align 2
+ ret bfloat %retval
+}
+
+define float @acquire_float_local_sys() {
+; CHECK-LABEL: acquire_float_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot848[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot848;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic float, ptr addrspace(5) %slot syncscope("") acquire, align 4
+ ret float %retval
+}
+
+define float @acquire_float_local_cta() {
+; CHECK-LABEL: acquire_float_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot849[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot849;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic float, ptr addrspace(5) %slot syncscope("block") acquire, align 4
+ ret float %retval
+}
+
+define float @acquire_float_local_cluster() {
+; CHECK-LABEL: acquire_float_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot850[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot850;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic float, ptr addrspace(5) %slot syncscope("cluster") acquire, align 4
+ ret float %retval
+}
+
+define float @acquire_float_local_gpu() {
+; CHECK-LABEL: acquire_float_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot851[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot851;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic float, ptr addrspace(5) %slot syncscope("device") acquire, align 4
+ ret float %retval
+}
+
+define float @acquire_float_local_sys_volatile() {
+; CHECK-LABEL: acquire_float_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot852[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot852;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile float, ptr addrspace(5) %slot syncscope("") acquire, align 4
+ ret float %retval
+}
+
+define float @acquire_float_local_cta_volatile() {
+; CHECK-LABEL: acquire_float_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot853[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot853;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile float, ptr addrspace(5) %slot syncscope("block") acquire, align 4
+ ret float %retval
+}
+
+define float @acquire_float_local_cluster_volatile() {
+; CHECK-LABEL: acquire_float_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot854[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot854;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile float, ptr addrspace(5) %slot syncscope("cluster") acquire, align 4
+ ret float %retval
+}
+
+define float @acquire_float_local_gpu_volatile() {
+; CHECK-LABEL: acquire_float_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot855[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot855;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile float, ptr addrspace(5) %slot syncscope("device") acquire, align 4
+ ret float %retval
+}
+
+define double @acquire_double_local_sys() {
+; CHECK-LABEL: acquire_double_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot856[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot856;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic double, ptr addrspace(5) %slot syncscope("") acquire, align 8
+ ret double %retval
+}
+
+define double @acquire_double_local_cta() {
+; CHECK-LABEL: acquire_double_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot857[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot857;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic double, ptr addrspace(5) %slot syncscope("block") acquire, align 8
+ ret double %retval
+}
+
+define double @acquire_double_local_cluster() {
+; CHECK-LABEL: acquire_double_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot858[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot858;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic double, ptr addrspace(5) %slot syncscope("cluster") acquire, align 8
+ ret double %retval
+}
+
+define double @acquire_double_local_gpu() {
+; CHECK-LABEL: acquire_double_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot859[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot859;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic double, ptr addrspace(5) %slot syncscope("device") acquire, align 8
+ ret double %retval
+}
+
+define double @acquire_double_local_sys_volatile() {
+; CHECK-LABEL: acquire_double_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot860[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot860;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile double, ptr addrspace(5) %slot syncscope("") acquire, align 8
+ ret double %retval
+}
+
+define double @acquire_double_local_cta_volatile() {
+; CHECK-LABEL: acquire_double_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot861[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot861;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile double, ptr addrspace(5) %slot syncscope("block") acquire, align 8
+ ret double %retval
+}
+
+define double @acquire_double_local_cluster_volatile() {
+; CHECK-LABEL: acquire_double_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot862[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot862;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile double, ptr addrspace(5) %slot syncscope("cluster") acquire, align 8
+ ret double %retval
+}
+
+define double @acquire_double_local_gpu_volatile() {
+; CHECK-LABEL: acquire_double_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot863[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot863;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile double, ptr addrspace(5) %slot syncscope("device") acquire, align 8
+ ret double %retval
+}
+
+define i8 @seq_cst_i8_generic_sys(ptr %addr) {
+; CHECK-LABEL: seq_cst_i8_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_generic_sys_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr %addr syncscope("") seq_cst, align 1
+ ret i8 %retval
+}
+
+define i8 @seq_cst_i8_generic_cta(ptr %addr) {
+; CHECK-LABEL: seq_cst_i8_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_generic_cta_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.acquire.cta.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr %addr syncscope("block") seq_cst, align 1
+ ret i8 %retval
+}
+
+define i8 @seq_cst_i8_generic_cluster(ptr %addr) {
+; CHECK-LABEL: seq_cst_i8_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_generic_cluster_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.acquire.cta.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr %addr syncscope("cluster") seq_cst, align 1
+ ret i8 %retval
+}
+
+define i8 @seq_cst_i8_generic_gpu(ptr %addr) {
+; CHECK-LABEL: seq_cst_i8_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_generic_gpu_param_0];
+; CHECK-NEXT: fence.sc.gpu;
+; CHECK-NEXT: ld.acquire.gpu.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr %addr syncscope("device") seq_cst, align 1
+ ret i8 %retval
+}
+
+define i8 @seq_cst_i8_generic_sys_volatile(ptr %addr) {
+; CHECK-LABEL: seq_cst_i8_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_generic_sys_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr %addr syncscope("") seq_cst, align 1
+ ret i8 %retval
+}
+
+define i8 @seq_cst_i8_generic_cta_volatile(ptr %addr) {
+; CHECK-LABEL: seq_cst_i8_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_generic_cta_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr %addr syncscope("block") seq_cst, align 1
+ ret i8 %retval
+}
+
+define i8 @seq_cst_i8_generic_cluster_volatile(ptr %addr) {
+; CHECK-LABEL: seq_cst_i8_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_generic_cluster_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr %addr syncscope("cluster") seq_cst, align 1
+ ret i8 %retval
+}
+
+define i8 @seq_cst_i8_generic_gpu_volatile(ptr %addr) {
+; CHECK-LABEL: seq_cst_i8_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_generic_gpu_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr %addr syncscope("device") seq_cst, align 1
+ ret i8 %retval
+}
+
+define i16 @seq_cst_i16_generic_sys(ptr %addr) {
+; CHECK-LABEL: seq_cst_i16_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_generic_sys_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr %addr syncscope("") seq_cst, align 2
+ ret i16 %retval
+}
+
+define i16 @seq_cst_i16_generic_cta(ptr %addr) {
+; CHECK-LABEL: seq_cst_i16_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_generic_cta_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.acquire.cta.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr %addr syncscope("block") seq_cst, align 2
+ ret i16 %retval
+}
+
+define i16 @seq_cst_i16_generic_cluster(ptr %addr) {
+; CHECK-LABEL: seq_cst_i16_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_generic_cluster_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.acquire.cta.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr %addr syncscope("cluster") seq_cst, align 2
+ ret i16 %retval
+}
+
+define i16 @seq_cst_i16_generic_gpu(ptr %addr) {
+; CHECK-LABEL: seq_cst_i16_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_generic_gpu_param_0];
+; CHECK-NEXT: fence.sc.gpu;
+; CHECK-NEXT: ld.acquire.gpu.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr %addr syncscope("device") seq_cst, align 2
+ ret i16 %retval
+}
+
+define i16 @seq_cst_i16_generic_sys_volatile(ptr %addr) {
+; CHECK-LABEL: seq_cst_i16_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_generic_sys_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr %addr syncscope("") seq_cst, align 2
+ ret i16 %retval
+}
+
+define i16 @seq_cst_i16_generic_cta_volatile(ptr %addr) {
+; CHECK-LABEL: seq_cst_i16_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_generic_cta_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr %addr syncscope("block") seq_cst, align 2
+ ret i16 %retval
+}
+
+define i16 @seq_cst_i16_generic_cluster_volatile(ptr %addr) {
+; CHECK-LABEL: seq_cst_i16_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_generic_cluster_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr %addr syncscope("cluster") seq_cst, align 2
+ ret i16 %retval
+}
+
+define i16 @seq_cst_i16_generic_gpu_volatile(ptr %addr) {
+; CHECK-LABEL: seq_cst_i16_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_generic_gpu_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr %addr syncscope("device") seq_cst, align 2
+ ret i16 %retval
+}
+
+define i32 @seq_cst_i32_generic_sys(ptr %addr) {
+; CHECK-LABEL: seq_cst_i32_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_generic_sys_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr %addr syncscope("") seq_cst, align 4
+ ret i32 %retval
+}
+
+define i32 @seq_cst_i32_generic_cta(ptr %addr) {
+; CHECK-LABEL: seq_cst_i32_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_generic_cta_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.acquire.cta.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr %addr syncscope("block") seq_cst, align 4
+ ret i32 %retval
+}
+
+define i32 @seq_cst_i32_generic_cluster(ptr %addr) {
+; CHECK-LABEL: seq_cst_i32_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_generic_cluster_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.acquire.cta.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr %addr syncscope("cluster") seq_cst, align 4
+ ret i32 %retval
+}
+
+define i32 @seq_cst_i32_generic_gpu(ptr %addr) {
+; CHECK-LABEL: seq_cst_i32_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_generic_gpu_param_0];
+; CHECK-NEXT: fence.sc.gpu;
+; CHECK-NEXT: ld.acquire.gpu.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr %addr syncscope("device") seq_cst, align 4
+ ret i32 %retval
+}
+
+define i32 @seq_cst_i32_generic_sys_volatile(ptr %addr) {
+; CHECK-LABEL: seq_cst_i32_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_generic_sys_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr %addr syncscope("") seq_cst, align 4
+ ret i32 %retval
+}
+
+define i32 @seq_cst_i32_generic_cta_volatile(ptr %addr) {
+; CHECK-LABEL: seq_cst_i32_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_generic_cta_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr %addr syncscope("block") seq_cst, align 4
+ ret i32 %retval
+}
+
+define i32 @seq_cst_i32_generic_cluster_volatile(ptr %addr) {
+; CHECK-LABEL: seq_cst_i32_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_generic_cluster_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr %addr syncscope("cluster") seq_cst, align 4
+ ret i32 %retval
+}
+
+define i32 @seq_cst_i32_generic_gpu_volatile(ptr %addr) {
+; CHECK-LABEL: seq_cst_i32_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_generic_gpu_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr %addr syncscope("device") seq_cst, align 4
+ ret i32 %retval
+}
+
+define i64 @seq_cst_i64_generic_sys(ptr %addr) {
+; CHECK-LABEL: seq_cst_i64_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_generic_sys_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr %addr syncscope("") seq_cst, align 8
+ ret i64 %retval
+}
+
+define i64 @seq_cst_i64_generic_cta(ptr %addr) {
+; CHECK-LABEL: seq_cst_i64_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_generic_cta_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.acquire.cta.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr %addr syncscope("block") seq_cst, align 8
+ ret i64 %retval
+}
+
+define i64 @seq_cst_i64_generic_cluster(ptr %addr) {
+; CHECK-LABEL: seq_cst_i64_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_generic_cluster_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.acquire.cta.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr %addr syncscope("cluster") seq_cst, align 8
+ ret i64 %retval
+}
+
+define i64 @seq_cst_i64_generic_gpu(ptr %addr) {
+; CHECK-LABEL: seq_cst_i64_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_generic_gpu_param_0];
+; CHECK-NEXT: fence.sc.gpu;
+; CHECK-NEXT: ld.acquire.gpu.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr %addr syncscope("device") seq_cst, align 8
+ ret i64 %retval
+}
+
+define i64 @seq_cst_i64_generic_sys_volatile(ptr %addr) {
+; CHECK-LABEL: seq_cst_i64_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_generic_sys_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr %addr syncscope("") seq_cst, align 8
+ ret i64 %retval
+}
+
+define i64 @seq_cst_i64_generic_cta_volatile(ptr %addr) {
+; CHECK-LABEL: seq_cst_i64_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_generic_cta_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr %addr syncscope("block") seq_cst, align 8
+ ret i64 %retval
+}
+
+define i64 @seq_cst_i64_generic_cluster_volatile(ptr %addr) {
+; CHECK-LABEL: seq_cst_i64_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_generic_cluster_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr %addr syncscope("cluster") seq_cst, align 8
+ ret i64 %retval
+}
+
+define i64 @seq_cst_i64_generic_gpu_volatile(ptr %addr) {
+; CHECK-LABEL: seq_cst_i64_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_generic_gpu_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr %addr syncscope("device") seq_cst, align 8
+ ret i64 %retval
+}
+
+define half @seq_cst_half_generic_sys(ptr %addr) {
+; CHECK-LABEL: seq_cst_half_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_generic_sys_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr %addr syncscope("") seq_cst, align 2
+ ret half %retval
+}
+
+define half @seq_cst_half_generic_cta(ptr %addr) {
+; CHECK-LABEL: seq_cst_half_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_generic_cta_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.acquire.cta.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr %addr syncscope("block") seq_cst, align 2
+ ret half %retval
+}
+
+define half @seq_cst_half_generic_cluster(ptr %addr) {
+; CHECK-LABEL: seq_cst_half_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_generic_cluster_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.acquire.cta.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr %addr syncscope("cluster") seq_cst, align 2
+ ret half %retval
+}
+
+define half @seq_cst_half_generic_gpu(ptr %addr) {
+; CHECK-LABEL: seq_cst_half_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_generic_gpu_param_0];
+; CHECK-NEXT: fence.sc.gpu;
+; CHECK-NEXT: ld.acquire.gpu.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr %addr syncscope("device") seq_cst, align 2
+ ret half %retval
+}
+
+define half @seq_cst_half_generic_sys_volatile(ptr %addr) {
+; CHECK-LABEL: seq_cst_half_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_generic_sys_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr %addr syncscope("") seq_cst, align 2
+ ret half %retval
+}
+
+define half @seq_cst_half_generic_cta_volatile(ptr %addr) {
+; CHECK-LABEL: seq_cst_half_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_generic_cta_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr %addr syncscope("block") seq_cst, align 2
+ ret half %retval
+}
+
+define half @seq_cst_half_generic_cluster_volatile(ptr %addr) {
+; CHECK-LABEL: seq_cst_half_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_generic_cluster_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr %addr syncscope("cluster") seq_cst, align 2
+ ret half %retval
+}
+
+define half @seq_cst_half_generic_gpu_volatile(ptr %addr) {
+; CHECK-LABEL: seq_cst_half_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_generic_gpu_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr %addr syncscope("device") seq_cst, align 2
+ ret half %retval
+}
+
+define bfloat @seq_cst_bfloat_generic_sys(ptr %addr) {
+; CHECK-LABEL: seq_cst_bfloat_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_generic_sys_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr %addr syncscope("") seq_cst, align 2
+ ret bfloat %retval
+}
+
+define bfloat @seq_cst_bfloat_generic_cta(ptr %addr) {
+; CHECK-LABEL: seq_cst_bfloat_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_generic_cta_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.acquire.cta.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr %addr syncscope("block") seq_cst, align 2
+ ret bfloat %retval
+}
+
+define bfloat @seq_cst_bfloat_generic_cluster(ptr %addr) {
+; CHECK-LABEL: seq_cst_bfloat_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_generic_cluster_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.acquire.cta.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr %addr syncscope("cluster") seq_cst, align 2
+ ret bfloat %retval
+}
+
+define bfloat @seq_cst_bfloat_generic_gpu(ptr %addr) {
+; CHECK-LABEL: seq_cst_bfloat_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_generic_gpu_param_0];
+; CHECK-NEXT: fence.sc.gpu;
+; CHECK-NEXT: ld.acquire.gpu.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr %addr syncscope("device") seq_cst, align 2
+ ret bfloat %retval
+}
+
+define bfloat @seq_cst_bfloat_generic_sys_volatile(ptr %addr) {
+; CHECK-LABEL: seq_cst_bfloat_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_generic_sys_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr %addr syncscope("") seq_cst, align 2
+ ret bfloat %retval
+}
+
+define bfloat @seq_cst_bfloat_generic_cta_volatile(ptr %addr) {
+; CHECK-LABEL: seq_cst_bfloat_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_generic_cta_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr %addr syncscope("block") seq_cst, align 2
+ ret bfloat %retval
+}
+
+define bfloat @seq_cst_bfloat_generic_cluster_volatile(ptr %addr) {
+; CHECK-LABEL: seq_cst_bfloat_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_generic_cluster_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr %addr syncscope("cluster") seq_cst, align 2
+ ret bfloat %retval
+}
+
+define bfloat @seq_cst_bfloat_generic_gpu_volatile(ptr %addr) {
+; CHECK-LABEL: seq_cst_bfloat_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_generic_gpu_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr %addr syncscope("device") seq_cst, align 2
+ ret bfloat %retval
+}
+
+define float @seq_cst_float_generic_sys(ptr %addr) {
+; CHECK-LABEL: seq_cst_float_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_generic_sys_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr %addr syncscope("") seq_cst, align 4
+ ret float %retval
+}
+
+define float @seq_cst_float_generic_cta(ptr %addr) {
+; CHECK-LABEL: seq_cst_float_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_generic_cta_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.acquire.cta.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr %addr syncscope("block") seq_cst, align 4
+ ret float %retval
+}
+
+define float @seq_cst_float_generic_cluster(ptr %addr) {
+; CHECK-LABEL: seq_cst_float_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_generic_cluster_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.acquire.cta.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr %addr syncscope("cluster") seq_cst, align 4
+ ret float %retval
+}
+
+define float @seq_cst_float_generic_gpu(ptr %addr) {
+; CHECK-LABEL: seq_cst_float_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_generic_gpu_param_0];
+; CHECK-NEXT: fence.sc.gpu;
+; CHECK-NEXT: ld.acquire.gpu.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr %addr syncscope("device") seq_cst, align 4
+ ret float %retval
+}
+
+define float @seq_cst_float_generic_sys_volatile(ptr %addr) {
+; CHECK-LABEL: seq_cst_float_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_generic_sys_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr %addr syncscope("") seq_cst, align 4
+ ret float %retval
+}
+
+define float @seq_cst_float_generic_cta_volatile(ptr %addr) {
+; CHECK-LABEL: seq_cst_float_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_generic_cta_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr %addr syncscope("block") seq_cst, align 4
+ ret float %retval
+}
+
+define float @seq_cst_float_generic_cluster_volatile(ptr %addr) {
+; CHECK-LABEL: seq_cst_float_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_generic_cluster_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr %addr syncscope("cluster") seq_cst, align 4
+ ret float %retval
+}
+
+define float @seq_cst_float_generic_gpu_volatile(ptr %addr) {
+; CHECK-LABEL: seq_cst_float_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_generic_gpu_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr %addr syncscope("device") seq_cst, align 4
+ ret float %retval
+}
+
+define double @seq_cst_double_generic_sys(ptr %addr) {
+; CHECK-LABEL: seq_cst_double_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_generic_sys_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr %addr syncscope("") seq_cst, align 8
+ ret double %retval
+}
+
+define double @seq_cst_double_generic_cta(ptr %addr) {
+; CHECK-LABEL: seq_cst_double_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_generic_cta_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.acquire.cta.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr %addr syncscope("block") seq_cst, align 8
+ ret double %retval
+}
+
+define double @seq_cst_double_generic_cluster(ptr %addr) {
+; CHECK-LABEL: seq_cst_double_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_generic_cluster_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.acquire.cta.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr %addr syncscope("cluster") seq_cst, align 8
+ ret double %retval
+}
+
+define double @seq_cst_double_generic_gpu(ptr %addr) {
+; CHECK-LABEL: seq_cst_double_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_generic_gpu_param_0];
+; CHECK-NEXT: fence.sc.gpu;
+; CHECK-NEXT: ld.acquire.gpu.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr %addr syncscope("device") seq_cst, align 8
+ ret double %retval
+}
+
+define double @seq_cst_double_generic_sys_volatile(ptr %addr) {
+; CHECK-LABEL: seq_cst_double_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_generic_sys_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr %addr syncscope("") seq_cst, align 8
+ ret double %retval
+}
+
+define double @seq_cst_double_generic_cta_volatile(ptr %addr) {
+; CHECK-LABEL: seq_cst_double_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_generic_cta_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr %addr syncscope("block") seq_cst, align 8
+ ret double %retval
+}
+
+define double @seq_cst_double_generic_cluster_volatile(ptr %addr) {
+; CHECK-LABEL: seq_cst_double_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_generic_cluster_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr %addr syncscope("cluster") seq_cst, align 8
+ ret double %retval
+}
+
+define double @seq_cst_double_generic_gpu_volatile(ptr %addr) {
+; CHECK-LABEL: seq_cst_double_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_generic_gpu_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr %addr syncscope("device") seq_cst, align 8
+ ret double %retval
+}
+
+define i8 @seq_cst_i8_global_sys(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i8_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_global_sys_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.global.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr addrspace(1) %addr syncscope("") seq_cst, align 1
+ ret i8 %retval
+}
+
+define i8 @seq_cst_i8_global_cta(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i8_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_global_cta_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.acquire.cta.global.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr addrspace(1) %addr syncscope("block") seq_cst, align 1
+ ret i8 %retval
+}
+
+define i8 @seq_cst_i8_global_cluster(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i8_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_global_cluster_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.acquire.cta.global.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr addrspace(1) %addr syncscope("cluster") seq_cst, align 1
+ ret i8 %retval
+}
+
+define i8 @seq_cst_i8_global_gpu(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i8_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_global_gpu_param_0];
+; CHECK-NEXT: fence.sc.gpu;
+; CHECK-NEXT: ld.acquire.gpu.global.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr addrspace(1) %addr syncscope("device") seq_cst, align 1
+ ret i8 %retval
+}
+
+define i8 @seq_cst_i8_global_sys_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i8_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_global_sys_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.global.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr addrspace(1) %addr syncscope("") seq_cst, align 1
+ ret i8 %retval
+}
+
+define i8 @seq_cst_i8_global_cta_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i8_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_global_cta_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.global.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr addrspace(1) %addr syncscope("block") seq_cst, align 1
+ ret i8 %retval
+}
+
+define i8 @seq_cst_i8_global_cluster_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i8_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_global_cluster_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.global.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr addrspace(1) %addr syncscope("cluster") seq_cst, align 1
+ ret i8 %retval
+}
+
+define i8 @seq_cst_i8_global_gpu_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i8_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_global_gpu_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.global.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr addrspace(1) %addr syncscope("device") seq_cst, align 1
+ ret i8 %retval
+}
+
+define i16 @seq_cst_i16_global_sys(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i16_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_global_sys_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr addrspace(1) %addr syncscope("") seq_cst, align 2
+ ret i16 %retval
+}
+
+define i16 @seq_cst_i16_global_cta(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i16_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_global_cta_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.acquire.cta.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr addrspace(1) %addr syncscope("block") seq_cst, align 2
+ ret i16 %retval
+}
+
+define i16 @seq_cst_i16_global_cluster(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i16_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_global_cluster_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.acquire.cta.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr addrspace(1) %addr syncscope("cluster") seq_cst, align 2
+ ret i16 %retval
+}
+
+define i16 @seq_cst_i16_global_gpu(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i16_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_global_gpu_param_0];
+; CHECK-NEXT: fence.sc.gpu;
+; CHECK-NEXT: ld.acquire.gpu.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr addrspace(1) %addr syncscope("device") seq_cst, align 2
+ ret i16 %retval
+}
+
+define i16 @seq_cst_i16_global_sys_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i16_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_global_sys_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr addrspace(1) %addr syncscope("") seq_cst, align 2
+ ret i16 %retval
+}
+
+define i16 @seq_cst_i16_global_cta_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i16_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_global_cta_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr addrspace(1) %addr syncscope("block") seq_cst, align 2
+ ret i16 %retval
+}
+
+define i16 @seq_cst_i16_global_cluster_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i16_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_global_cluster_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr addrspace(1) %addr syncscope("cluster") seq_cst, align 2
+ ret i16 %retval
+}
+
+define i16 @seq_cst_i16_global_gpu_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i16_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_global_gpu_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr addrspace(1) %addr syncscope("device") seq_cst, align 2
+ ret i16 %retval
+}
+
+define i32 @seq_cst_i32_global_sys(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i32_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_global_sys_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr addrspace(1) %addr syncscope("") seq_cst, align 4
+ ret i32 %retval
+}
+
+define i32 @seq_cst_i32_global_cta(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i32_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_global_cta_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.acquire.cta.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr addrspace(1) %addr syncscope("block") seq_cst, align 4
+ ret i32 %retval
+}
+
+define i32 @seq_cst_i32_global_cluster(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i32_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_global_cluster_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.acquire.cta.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr addrspace(1) %addr syncscope("cluster") seq_cst, align 4
+ ret i32 %retval
+}
+
+define i32 @seq_cst_i32_global_gpu(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i32_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_global_gpu_param_0];
+; CHECK-NEXT: fence.sc.gpu;
+; CHECK-NEXT: ld.acquire.gpu.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr addrspace(1) %addr syncscope("device") seq_cst, align 4
+ ret i32 %retval
+}
+
+define i32 @seq_cst_i32_global_sys_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i32_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_global_sys_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr addrspace(1) %addr syncscope("") seq_cst, align 4
+ ret i32 %retval
+}
+
+define i32 @seq_cst_i32_global_cta_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i32_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_global_cta_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr addrspace(1) %addr syncscope("block") seq_cst, align 4
+ ret i32 %retval
+}
+
+define i32 @seq_cst_i32_global_cluster_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i32_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_global_cluster_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr addrspace(1) %addr syncscope("cluster") seq_cst, align 4
+ ret i32 %retval
+}
+
+define i32 @seq_cst_i32_global_gpu_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i32_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_global_gpu_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr addrspace(1) %addr syncscope("device") seq_cst, align 4
+ ret i32 %retval
+}
+
+define i64 @seq_cst_i64_global_sys(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i64_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_global_sys_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr addrspace(1) %addr syncscope("") seq_cst, align 8
+ ret i64 %retval
+}
+
+define i64 @seq_cst_i64_global_cta(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i64_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_global_cta_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.acquire.cta.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr addrspace(1) %addr syncscope("block") seq_cst, align 8
+ ret i64 %retval
+}
+
+define i64 @seq_cst_i64_global_cluster(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i64_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_global_cluster_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.acquire.cta.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr addrspace(1) %addr syncscope("cluster") seq_cst, align 8
+ ret i64 %retval
+}
+
+define i64 @seq_cst_i64_global_gpu(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i64_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_global_gpu_param_0];
+; CHECK-NEXT: fence.sc.gpu;
+; CHECK-NEXT: ld.acquire.gpu.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr addrspace(1) %addr syncscope("device") seq_cst, align 8
+ ret i64 %retval
+}
+
+define i64 @seq_cst_i64_global_sys_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i64_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_global_sys_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr addrspace(1) %addr syncscope("") seq_cst, align 8
+ ret i64 %retval
+}
+
+define i64 @seq_cst_i64_global_cta_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i64_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_global_cta_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr addrspace(1) %addr syncscope("block") seq_cst, align 8
+ ret i64 %retval
+}
+
+define i64 @seq_cst_i64_global_cluster_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i64_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_global_cluster_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr addrspace(1) %addr syncscope("cluster") seq_cst, align 8
+ ret i64 %retval
+}
+
+define i64 @seq_cst_i64_global_gpu_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i64_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_global_gpu_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr addrspace(1) %addr syncscope("device") seq_cst, align 8
+ ret i64 %retval
+}
+
+define half @seq_cst_half_global_sys(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_half_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_global_sys_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr addrspace(1) %addr syncscope("") seq_cst, align 2
+ ret half %retval
+}
+
+define half @seq_cst_half_global_cta(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_half_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_global_cta_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.acquire.cta.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr addrspace(1) %addr syncscope("block") seq_cst, align 2
+ ret half %retval
+}
+
+define half @seq_cst_half_global_cluster(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_half_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_global_cluster_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.acquire.cta.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr addrspace(1) %addr syncscope("cluster") seq_cst, align 2
+ ret half %retval
+}
+
+define half @seq_cst_half_global_gpu(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_half_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_global_gpu_param_0];
+; CHECK-NEXT: fence.sc.gpu;
+; CHECK-NEXT: ld.acquire.gpu.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr addrspace(1) %addr syncscope("device") seq_cst, align 2
+ ret half %retval
+}
+
+define half @seq_cst_half_global_sys_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_half_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_global_sys_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr addrspace(1) %addr syncscope("") seq_cst, align 2
+ ret half %retval
+}
+
+define half @seq_cst_half_global_cta_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_half_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_global_cta_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr addrspace(1) %addr syncscope("block") seq_cst, align 2
+ ret half %retval
+}
+
+define half @seq_cst_half_global_cluster_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_half_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_global_cluster_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr addrspace(1) %addr syncscope("cluster") seq_cst, align 2
+ ret half %retval
+}
+
+define half @seq_cst_half_global_gpu_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_half_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_global_gpu_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr addrspace(1) %addr syncscope("device") seq_cst, align 2
+ ret half %retval
+}
+
+define bfloat @seq_cst_bfloat_global_sys(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_bfloat_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_global_sys_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr addrspace(1) %addr syncscope("") seq_cst, align 2
+ ret bfloat %retval
+}
+
+define bfloat @seq_cst_bfloat_global_cta(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_bfloat_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_global_cta_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.acquire.cta.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr addrspace(1) %addr syncscope("block") seq_cst, align 2
+ ret bfloat %retval
+}
+
+define bfloat @seq_cst_bfloat_global_cluster(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_bfloat_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_global_cluster_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.acquire.cta.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr addrspace(1) %addr syncscope("cluster") seq_cst, align 2
+ ret bfloat %retval
+}
+
+define bfloat @seq_cst_bfloat_global_gpu(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_bfloat_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_global_gpu_param_0];
+; CHECK-NEXT: fence.sc.gpu;
+; CHECK-NEXT: ld.acquire.gpu.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr addrspace(1) %addr syncscope("device") seq_cst, align 2
+ ret bfloat %retval
+}
+
+define bfloat @seq_cst_bfloat_global_sys_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_bfloat_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_global_sys_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr addrspace(1) %addr syncscope("") seq_cst, align 2
+ ret bfloat %retval
+}
+
+define bfloat @seq_cst_bfloat_global_cta_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_bfloat_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_global_cta_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr addrspace(1) %addr syncscope("block") seq_cst, align 2
+ ret bfloat %retval
+}
+
+define bfloat @seq_cst_bfloat_global_cluster_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_bfloat_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_global_cluster_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr addrspace(1) %addr syncscope("cluster") seq_cst, align 2
+ ret bfloat %retval
+}
+
+define bfloat @seq_cst_bfloat_global_gpu_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_bfloat_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_global_gpu_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr addrspace(1) %addr syncscope("device") seq_cst, align 2
+ ret bfloat %retval
+}
+
+define float @seq_cst_float_global_sys(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_float_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_global_sys_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr addrspace(1) %addr syncscope("") seq_cst, align 4
+ ret float %retval
+}
+
+define float @seq_cst_float_global_cta(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_float_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_global_cta_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.acquire.cta.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr addrspace(1) %addr syncscope("block") seq_cst, align 4
+ ret float %retval
+}
+
+define float @seq_cst_float_global_cluster(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_float_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_global_cluster_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.acquire.cta.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr addrspace(1) %addr syncscope("cluster") seq_cst, align 4
+ ret float %retval
+}
+
+define float @seq_cst_float_global_gpu(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_float_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_global_gpu_param_0];
+; CHECK-NEXT: fence.sc.gpu;
+; CHECK-NEXT: ld.acquire.gpu.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr addrspace(1) %addr syncscope("device") seq_cst, align 4
+ ret float %retval
+}
+
+define float @seq_cst_float_global_sys_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_float_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_global_sys_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr addrspace(1) %addr syncscope("") seq_cst, align 4
+ ret float %retval
+}
+
+define float @seq_cst_float_global_cta_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_float_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_global_cta_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr addrspace(1) %addr syncscope("block") seq_cst, align 4
+ ret float %retval
+}
+
+define float @seq_cst_float_global_cluster_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_float_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_global_cluster_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr addrspace(1) %addr syncscope("cluster") seq_cst, align 4
+ ret float %retval
+}
+
+define float @seq_cst_float_global_gpu_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_float_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_global_gpu_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr addrspace(1) %addr syncscope("device") seq_cst, align 4
+ ret float %retval
+}
+
+define double @seq_cst_double_global_sys(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_double_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_global_sys_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr addrspace(1) %addr syncscope("") seq_cst, align 8
+ ret double %retval
+}
+
+define double @seq_cst_double_global_cta(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_double_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_global_cta_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.acquire.cta.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr addrspace(1) %addr syncscope("block") seq_cst, align 8
+ ret double %retval
+}
+
+define double @seq_cst_double_global_cluster(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_double_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_global_cluster_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.acquire.cta.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr addrspace(1) %addr syncscope("cluster") seq_cst, align 8
+ ret double %retval
+}
+
+define double @seq_cst_double_global_gpu(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_double_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_global_gpu_param_0];
+; CHECK-NEXT: fence.sc.gpu;
+; CHECK-NEXT: ld.acquire.gpu.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr addrspace(1) %addr syncscope("device") seq_cst, align 8
+ ret double %retval
+}
+
+define double @seq_cst_double_global_sys_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_double_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_global_sys_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr addrspace(1) %addr syncscope("") seq_cst, align 8
+ ret double %retval
+}
+
+define double @seq_cst_double_global_cta_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_double_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_global_cta_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr addrspace(1) %addr syncscope("block") seq_cst, align 8
+ ret double %retval
+}
+
+define double @seq_cst_double_global_cluster_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_double_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_global_cluster_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr addrspace(1) %addr syncscope("cluster") seq_cst, align 8
+ ret double %retval
+}
+
+define double @seq_cst_double_global_gpu_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_double_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_global_gpu_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr addrspace(1) %addr syncscope("device") seq_cst, align 8
+ ret double %retval
+}
+
+define i8 @seq_cst_i8_shared_sys(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i8_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_shared_sys_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.shared.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr addrspace(3) %addr syncscope("") seq_cst, align 1
+ ret i8 %retval
+}
+
+define i8 @seq_cst_i8_shared_cta(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i8_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_shared_cta_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.acquire.cta.shared.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr addrspace(3) %addr syncscope("block") seq_cst, align 1
+ ret i8 %retval
+}
+
+define i8 @seq_cst_i8_shared_cluster(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i8_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_shared_cluster_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.acquire.cta.shared.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr addrspace(3) %addr syncscope("cluster") seq_cst, align 1
+ ret i8 %retval
+}
+
+define i8 @seq_cst_i8_shared_gpu(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i8_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_shared_gpu_param_0];
+; CHECK-NEXT: fence.sc.gpu;
+; CHECK-NEXT: ld.acquire.gpu.shared.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr addrspace(3) %addr syncscope("device") seq_cst, align 1
+ ret i8 %retval
+}
+
+define i8 @seq_cst_i8_shared_sys_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i8_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_shared_sys_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.shared.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr addrspace(3) %addr syncscope("") seq_cst, align 1
+ ret i8 %retval
+}
+
+define i8 @seq_cst_i8_shared_cta_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i8_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_shared_cta_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.shared.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr addrspace(3) %addr syncscope("block") seq_cst, align 1
+ ret i8 %retval
+}
+
+define i8 @seq_cst_i8_shared_cluster_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i8_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_shared_cluster_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.shared.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr addrspace(3) %addr syncscope("cluster") seq_cst, align 1
+ ret i8 %retval
+}
+
+define i8 @seq_cst_i8_shared_gpu_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i8_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_shared_gpu_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.shared.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr addrspace(3) %addr syncscope("device") seq_cst, align 1
+ ret i8 %retval
+}
+
+define i16 @seq_cst_i16_shared_sys(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i16_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_shared_sys_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr addrspace(3) %addr syncscope("") seq_cst, align 2
+ ret i16 %retval
+}
+
+define i16 @seq_cst_i16_shared_cta(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i16_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_shared_cta_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.acquire.cta.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr addrspace(3) %addr syncscope("block") seq_cst, align 2
+ ret i16 %retval
+}
+
+define i16 @seq_cst_i16_shared_cluster(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i16_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_shared_cluster_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.acquire.cta.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr addrspace(3) %addr syncscope("cluster") seq_cst, align 2
+ ret i16 %retval
+}
+
+define i16 @seq_cst_i16_shared_gpu(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i16_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_shared_gpu_param_0];
+; CHECK-NEXT: fence.sc.gpu;
+; CHECK-NEXT: ld.acquire.gpu.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr addrspace(3) %addr syncscope("device") seq_cst, align 2
+ ret i16 %retval
+}
+
+define i16 @seq_cst_i16_shared_sys_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i16_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_shared_sys_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr addrspace(3) %addr syncscope("") seq_cst, align 2
+ ret i16 %retval
+}
+
+define i16 @seq_cst_i16_shared_cta_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i16_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_shared_cta_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr addrspace(3) %addr syncscope("block") seq_cst, align 2
+ ret i16 %retval
+}
+
+define i16 @seq_cst_i16_shared_cluster_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i16_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_shared_cluster_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr addrspace(3) %addr syncscope("cluster") seq_cst, align 2
+ ret i16 %retval
+}
+
+define i16 @seq_cst_i16_shared_gpu_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i16_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_shared_gpu_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr addrspace(3) %addr syncscope("device") seq_cst, align 2
+ ret i16 %retval
+}
+
+define i32 @seq_cst_i32_shared_sys(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i32_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_shared_sys_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr addrspace(3) %addr syncscope("") seq_cst, align 4
+ ret i32 %retval
+}
+
+define i32 @seq_cst_i32_shared_cta(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i32_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_shared_cta_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.acquire.cta.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr addrspace(3) %addr syncscope("block") seq_cst, align 4
+ ret i32 %retval
+}
+
+define i32 @seq_cst_i32_shared_cluster(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i32_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_shared_cluster_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.acquire.cta.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr addrspace(3) %addr syncscope("cluster") seq_cst, align 4
+ ret i32 %retval
+}
+
+define i32 @seq_cst_i32_shared_gpu(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i32_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_shared_gpu_param_0];
+; CHECK-NEXT: fence.sc.gpu;
+; CHECK-NEXT: ld.acquire.gpu.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr addrspace(3) %addr syncscope("device") seq_cst, align 4
+ ret i32 %retval
+}
+
+define i32 @seq_cst_i32_shared_sys_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i32_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_shared_sys_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr addrspace(3) %addr syncscope("") seq_cst, align 4
+ ret i32 %retval
+}
+
+define i32 @seq_cst_i32_shared_cta_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i32_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_shared_cta_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr addrspace(3) %addr syncscope("block") seq_cst, align 4
+ ret i32 %retval
+}
+
+define i32 @seq_cst_i32_shared_cluster_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i32_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_shared_cluster_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr addrspace(3) %addr syncscope("cluster") seq_cst, align 4
+ ret i32 %retval
+}
+
+define i32 @seq_cst_i32_shared_gpu_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i32_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_shared_gpu_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr addrspace(3) %addr syncscope("device") seq_cst, align 4
+ ret i32 %retval
+}
+
+define i64 @seq_cst_i64_shared_sys(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i64_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_shared_sys_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr addrspace(3) %addr syncscope("") seq_cst, align 8
+ ret i64 %retval
+}
+
+define i64 @seq_cst_i64_shared_cta(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i64_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_shared_cta_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.acquire.cta.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr addrspace(3) %addr syncscope("block") seq_cst, align 8
+ ret i64 %retval
+}
+
+define i64 @seq_cst_i64_shared_cluster(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i64_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_shared_cluster_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.acquire.cta.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr addrspace(3) %addr syncscope("cluster") seq_cst, align 8
+ ret i64 %retval
+}
+
+define i64 @seq_cst_i64_shared_gpu(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i64_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_shared_gpu_param_0];
+; CHECK-NEXT: fence.sc.gpu;
+; CHECK-NEXT: ld.acquire.gpu.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr addrspace(3) %addr syncscope("device") seq_cst, align 8
+ ret i64 %retval
+}
+
+define i64 @seq_cst_i64_shared_sys_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i64_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_shared_sys_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr addrspace(3) %addr syncscope("") seq_cst, align 8
+ ret i64 %retval
+}
+
+define i64 @seq_cst_i64_shared_cta_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i64_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_shared_cta_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr addrspace(3) %addr syncscope("block") seq_cst, align 8
+ ret i64 %retval
+}
+
+define i64 @seq_cst_i64_shared_cluster_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i64_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_shared_cluster_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr addrspace(3) %addr syncscope("cluster") seq_cst, align 8
+ ret i64 %retval
+}
+
+define i64 @seq_cst_i64_shared_gpu_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i64_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_shared_gpu_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr addrspace(3) %addr syncscope("device") seq_cst, align 8
+ ret i64 %retval
+}
+
+define half @seq_cst_half_shared_sys(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_half_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_shared_sys_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr addrspace(3) %addr syncscope("") seq_cst, align 2
+ ret half %retval
+}
+
+define half @seq_cst_half_shared_cta(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_half_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_shared_cta_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.acquire.cta.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr addrspace(3) %addr syncscope("block") seq_cst, align 2
+ ret half %retval
+}
+
+define half @seq_cst_half_shared_cluster(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_half_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_shared_cluster_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.acquire.cta.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr addrspace(3) %addr syncscope("cluster") seq_cst, align 2
+ ret half %retval
+}
+
+define half @seq_cst_half_shared_gpu(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_half_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_shared_gpu_param_0];
+; CHECK-NEXT: fence.sc.gpu;
+; CHECK-NEXT: ld.acquire.gpu.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr addrspace(3) %addr syncscope("device") seq_cst, align 2
+ ret half %retval
+}
+
+define half @seq_cst_half_shared_sys_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_half_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_shared_sys_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr addrspace(3) %addr syncscope("") seq_cst, align 2
+ ret half %retval
+}
+
+define half @seq_cst_half_shared_cta_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_half_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_shared_cta_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr addrspace(3) %addr syncscope("block") seq_cst, align 2
+ ret half %retval
+}
+
+define half @seq_cst_half_shared_cluster_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_half_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_shared_cluster_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr addrspace(3) %addr syncscope("cluster") seq_cst, align 2
+ ret half %retval
+}
+
+define half @seq_cst_half_shared_gpu_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_half_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_shared_gpu_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr addrspace(3) %addr syncscope("device") seq_cst, align 2
+ ret half %retval
+}
+
+define bfloat @seq_cst_bfloat_shared_sys(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_bfloat_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_shared_sys_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr addrspace(3) %addr syncscope("") seq_cst, align 2
+ ret bfloat %retval
+}
+
+define bfloat @seq_cst_bfloat_shared_cta(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_bfloat_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_shared_cta_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.acquire.cta.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr addrspace(3) %addr syncscope("block") seq_cst, align 2
+ ret bfloat %retval
+}
+
+define bfloat @seq_cst_bfloat_shared_cluster(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_bfloat_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_shared_cluster_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.acquire.cta.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr addrspace(3) %addr syncscope("cluster") seq_cst, align 2
+ ret bfloat %retval
+}
+
+define bfloat @seq_cst_bfloat_shared_gpu(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_bfloat_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_shared_gpu_param_0];
+; CHECK-NEXT: fence.sc.gpu;
+; CHECK-NEXT: ld.acquire.gpu.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr addrspace(3) %addr syncscope("device") seq_cst, align 2
+ ret bfloat %retval
+}
+
+define bfloat @seq_cst_bfloat_shared_sys_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_bfloat_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_shared_sys_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr addrspace(3) %addr syncscope("") seq_cst, align 2
+ ret bfloat %retval
+}
+
+define bfloat @seq_cst_bfloat_shared_cta_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_bfloat_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_shared_cta_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr addrspace(3) %addr syncscope("block") seq_cst, align 2
+ ret bfloat %retval
+}
+
+define bfloat @seq_cst_bfloat_shared_cluster_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_bfloat_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_shared_cluster_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr addrspace(3) %addr syncscope("cluster") seq_cst, align 2
+ ret bfloat %retval
+}
+
+define bfloat @seq_cst_bfloat_shared_gpu_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_bfloat_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_shared_gpu_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr addrspace(3) %addr syncscope("device") seq_cst, align 2
+ ret bfloat %retval
+}
+
+define float @seq_cst_float_shared_sys(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_float_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_shared_sys_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr addrspace(3) %addr syncscope("") seq_cst, align 4
+ ret float %retval
+}
+
+define float @seq_cst_float_shared_cta(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_float_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_shared_cta_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.acquire.cta.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr addrspace(3) %addr syncscope("block") seq_cst, align 4
+ ret float %retval
+}
+
+define float @seq_cst_float_shared_cluster(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_float_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_shared_cluster_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.acquire.cta.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr addrspace(3) %addr syncscope("cluster") seq_cst, align 4
+ ret float %retval
+}
+
+define float @seq_cst_float_shared_gpu(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_float_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_shared_gpu_param_0];
+; CHECK-NEXT: fence.sc.gpu;
+; CHECK-NEXT: ld.acquire.gpu.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr addrspace(3) %addr syncscope("device") seq_cst, align 4
+ ret float %retval
+}
+
+define float @seq_cst_float_shared_sys_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_float_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_shared_sys_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr addrspace(3) %addr syncscope("") seq_cst, align 4
+ ret float %retval
+}
+
+define float @seq_cst_float_shared_cta_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_float_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_shared_cta_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr addrspace(3) %addr syncscope("block") seq_cst, align 4
+ ret float %retval
+}
+
+define float @seq_cst_float_shared_cluster_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_float_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_shared_cluster_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr addrspace(3) %addr syncscope("cluster") seq_cst, align 4
+ ret float %retval
+}
+
+define float @seq_cst_float_shared_gpu_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_float_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_shared_gpu_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr addrspace(3) %addr syncscope("device") seq_cst, align 4
+ ret float %retval
+}
+
+define double @seq_cst_double_shared_sys(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_double_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_shared_sys_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr addrspace(3) %addr syncscope("") seq_cst, align 8
+ ret double %retval
+}
+
+define double @seq_cst_double_shared_cta(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_double_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_shared_cta_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.acquire.cta.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr addrspace(3) %addr syncscope("block") seq_cst, align 8
+ ret double %retval
+}
+
+define double @seq_cst_double_shared_cluster(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_double_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_shared_cluster_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.acquire.cta.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr addrspace(3) %addr syncscope("cluster") seq_cst, align 8
+ ret double %retval
+}
+
+define double @seq_cst_double_shared_gpu(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_double_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_shared_gpu_param_0];
+; CHECK-NEXT: fence.sc.gpu;
+; CHECK-NEXT: ld.acquire.gpu.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr addrspace(3) %addr syncscope("device") seq_cst, align 8
+ ret double %retval
+}
+
+define double @seq_cst_double_shared_sys_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_double_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_shared_sys_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr addrspace(3) %addr syncscope("") seq_cst, align 8
+ ret double %retval
+}
+
+define double @seq_cst_double_shared_cta_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_double_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_shared_cta_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr addrspace(3) %addr syncscope("block") seq_cst, align 8
+ ret double %retval
+}
+
+define double @seq_cst_double_shared_cluster_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_double_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_shared_cluster_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr addrspace(3) %addr syncscope("cluster") seq_cst, align 8
+ ret double %retval
+}
+
+define double @seq_cst_double_shared_gpu_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_double_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_shared_gpu_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr addrspace(3) %addr syncscope("device") seq_cst, align 8
+ ret double %retval
+}
+
+define i8 @seq_cst_i8_local_sys() {
+; CHECK-LABEL: seq_cst_i8_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot1056[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1056;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b8 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i8, ptr addrspace(5) %slot syncscope("") seq_cst, align 1
+ ret i8 %retval
+}
+
+define i8 @seq_cst_i8_local_cta() {
+; CHECK-LABEL: seq_cst_i8_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot1057[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1057;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b8 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i8, ptr addrspace(5) %slot syncscope("block") seq_cst, align 1
+ ret i8 %retval
+}
+
+define i8 @seq_cst_i8_local_cluster() {
+; CHECK-LABEL: seq_cst_i8_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot1058[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1058;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b8 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i8, ptr addrspace(5) %slot syncscope("cluster") seq_cst, align 1
+ ret i8 %retval
+}
+
+define i8 @seq_cst_i8_local_gpu() {
+; CHECK-LABEL: seq_cst_i8_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot1059[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1059;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b8 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i8, ptr addrspace(5) %slot syncscope("device") seq_cst, align 1
+ ret i8 %retval
+}
+
+define i8 @seq_cst_i8_local_sys_volatile() {
+; CHECK-LABEL: seq_cst_i8_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot1060[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1060;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b8 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i8, ptr addrspace(5) %slot syncscope("") seq_cst, align 1
+ ret i8 %retval
+}
+
+define i8 @seq_cst_i8_local_cta_volatile() {
+; CHECK-LABEL: seq_cst_i8_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot1061[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1061;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b8 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i8, ptr addrspace(5) %slot syncscope("block") seq_cst, align 1
+ ret i8 %retval
+}
+
+define i8 @seq_cst_i8_local_cluster_volatile() {
+; CHECK-LABEL: seq_cst_i8_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot1062[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1062;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b8 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i8, ptr addrspace(5) %slot syncscope("cluster") seq_cst, align 1
+ ret i8 %retval
+}
+
+define i8 @seq_cst_i8_local_gpu_volatile() {
+; CHECK-LABEL: seq_cst_i8_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot1063[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1063;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b8 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i8, ptr addrspace(5) %slot syncscope("device") seq_cst, align 1
+ ret i8 %retval
+}
+
+define i16 @seq_cst_i16_local_sys() {
+; CHECK-LABEL: seq_cst_i16_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1064[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1064;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i16, ptr addrspace(5) %slot syncscope("") seq_cst, align 2
+ ret i16 %retval
+}
+
+define i16 @seq_cst_i16_local_cta() {
+; CHECK-LABEL: seq_cst_i16_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1065[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1065;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i16, ptr addrspace(5) %slot syncscope("block") seq_cst, align 2
+ ret i16 %retval
+}
+
+define i16 @seq_cst_i16_local_cluster() {
+; CHECK-LABEL: seq_cst_i16_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1066[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1066;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i16, ptr addrspace(5) %slot syncscope("cluster") seq_cst, align 2
+ ret i16 %retval
+}
+
+define i16 @seq_cst_i16_local_gpu() {
+; CHECK-LABEL: seq_cst_i16_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1067[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1067;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i16, ptr addrspace(5) %slot syncscope("device") seq_cst, align 2
+ ret i16 %retval
+}
+
+define i16 @seq_cst_i16_local_sys_volatile() {
+; CHECK-LABEL: seq_cst_i16_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1068[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1068;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i16, ptr addrspace(5) %slot syncscope("") seq_cst, align 2
+ ret i16 %retval
+}
+
+define i16 @seq_cst_i16_local_cta_volatile() {
+; CHECK-LABEL: seq_cst_i16_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1069[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1069;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i16, ptr addrspace(5) %slot syncscope("block") seq_cst, align 2
+ ret i16 %retval
+}
+
+define i16 @seq_cst_i16_local_cluster_volatile() {
+; CHECK-LABEL: seq_cst_i16_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1070[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1070;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i16, ptr addrspace(5) %slot syncscope("cluster") seq_cst, align 2
+ ret i16 %retval
+}
+
+define i16 @seq_cst_i16_local_gpu_volatile() {
+; CHECK-LABEL: seq_cst_i16_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1071[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1071;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i16, ptr addrspace(5) %slot syncscope("device") seq_cst, align 2
+ ret i16 %retval
+}
+
+define i32 @seq_cst_i32_local_sys() {
+; CHECK-LABEL: seq_cst_i32_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot1072[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1072;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i32, ptr addrspace(5) %slot syncscope("") seq_cst, align 4
+ ret i32 %retval
+}
+
+define i32 @seq_cst_i32_local_cta() {
+; CHECK-LABEL: seq_cst_i32_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot1073[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1073;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i32, ptr addrspace(5) %slot syncscope("block") seq_cst, align 4
+ ret i32 %retval
+}
+
+define i32 @seq_cst_i32_local_cluster() {
+; CHECK-LABEL: seq_cst_i32_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot1074[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1074;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i32, ptr addrspace(5) %slot syncscope("cluster") seq_cst, align 4
+ ret i32 %retval
+}
+
+define i32 @seq_cst_i32_local_gpu() {
+; CHECK-LABEL: seq_cst_i32_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot1075[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1075;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i32, ptr addrspace(5) %slot syncscope("device") seq_cst, align 4
+ ret i32 %retval
+}
+
+define i32 @seq_cst_i32_local_sys_volatile() {
+; CHECK-LABEL: seq_cst_i32_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot1076[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1076;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i32, ptr addrspace(5) %slot syncscope("") seq_cst, align 4
+ ret i32 %retval
+}
+
+define i32 @seq_cst_i32_local_cta_volatile() {
+; CHECK-LABEL: seq_cst_i32_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot1077[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1077;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i32, ptr addrspace(5) %slot syncscope("block") seq_cst, align 4
+ ret i32 %retval
+}
+
+define i32 @seq_cst_i32_local_cluster_volatile() {
+; CHECK-LABEL: seq_cst_i32_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot1078[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1078;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i32, ptr addrspace(5) %slot syncscope("cluster") seq_cst, align 4
+ ret i32 %retval
+}
+
+define i32 @seq_cst_i32_local_gpu_volatile() {
+; CHECK-LABEL: seq_cst_i32_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot1079[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1079;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i32, ptr addrspace(5) %slot syncscope("device") seq_cst, align 4
+ ret i32 %retval
+}
+
+define i64 @seq_cst_i64_local_sys() {
+; CHECK-LABEL: seq_cst_i64_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot1080[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1080;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i64, ptr addrspace(5) %slot syncscope("") seq_cst, align 8
+ ret i64 %retval
+}
+
+define i64 @seq_cst_i64_local_cta() {
+; CHECK-LABEL: seq_cst_i64_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot1081[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1081;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i64, ptr addrspace(5) %slot syncscope("block") seq_cst, align 8
+ ret i64 %retval
+}
+
+define i64 @seq_cst_i64_local_cluster() {
+; CHECK-LABEL: seq_cst_i64_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot1082[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1082;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i64, ptr addrspace(5) %slot syncscope("cluster") seq_cst, align 8
+ ret i64 %retval
+}
+
+define i64 @seq_cst_i64_local_gpu() {
+; CHECK-LABEL: seq_cst_i64_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot1083[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1083;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i64, ptr addrspace(5) %slot syncscope("device") seq_cst, align 8
+ ret i64 %retval
+}
+
+define i64 @seq_cst_i64_local_sys_volatile() {
+; CHECK-LABEL: seq_cst_i64_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot1084[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1084;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i64, ptr addrspace(5) %slot syncscope("") seq_cst, align 8
+ ret i64 %retval
+}
+
+define i64 @seq_cst_i64_local_cta_volatile() {
+; CHECK-LABEL: seq_cst_i64_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot1085[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1085;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i64, ptr addrspace(5) %slot syncscope("block") seq_cst, align 8
+ ret i64 %retval
+}
+
+define i64 @seq_cst_i64_local_cluster_volatile() {
+; CHECK-LABEL: seq_cst_i64_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot1086[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1086;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i64, ptr addrspace(5) %slot syncscope("cluster") seq_cst, align 8
+ ret i64 %retval
+}
+
+define i64 @seq_cst_i64_local_gpu_volatile() {
+; CHECK-LABEL: seq_cst_i64_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot1087[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1087;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i64, ptr addrspace(5) %slot syncscope("device") seq_cst, align 8
+ ret i64 %retval
+}
+
+define half @seq_cst_half_local_sys() {
+; CHECK-LABEL: seq_cst_half_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1088[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1088;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic half, ptr addrspace(5) %slot syncscope("") seq_cst, align 2
+ ret half %retval
+}
+
+define half @seq_cst_half_local_cta() {
+; CHECK-LABEL: seq_cst_half_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1089[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1089;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic half, ptr addrspace(5) %slot syncscope("block") seq_cst, align 2
+ ret half %retval
+}
+
+define half @seq_cst_half_local_cluster() {
+; CHECK-LABEL: seq_cst_half_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1090[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1090;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic half, ptr addrspace(5) %slot syncscope("cluster") seq_cst, align 2
+ ret half %retval
+}
+
+define half @seq_cst_half_local_gpu() {
+; CHECK-LABEL: seq_cst_half_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1091[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1091;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic half, ptr addrspace(5) %slot syncscope("device") seq_cst, align 2
+ ret half %retval
+}
+
+define half @seq_cst_half_local_sys_volatile() {
+; CHECK-LABEL: seq_cst_half_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1092[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1092;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile half, ptr addrspace(5) %slot syncscope("") seq_cst, align 2
+ ret half %retval
+}
+
+define half @seq_cst_half_local_cta_volatile() {
+; CHECK-LABEL: seq_cst_half_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1093[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1093;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile half, ptr addrspace(5) %slot syncscope("block") seq_cst, align 2
+ ret half %retval
+}
+
+define half @seq_cst_half_local_cluster_volatile() {
+; CHECK-LABEL: seq_cst_half_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1094[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1094;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile half, ptr addrspace(5) %slot syncscope("cluster") seq_cst, align 2
+ ret half %retval
+}
+
+define half @seq_cst_half_local_gpu_volatile() {
+; CHECK-LABEL: seq_cst_half_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1095[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1095;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile half, ptr addrspace(5) %slot syncscope("device") seq_cst, align 2
+ ret half %retval
+}
+
+define bfloat @seq_cst_bfloat_local_sys() {
+; CHECK-LABEL: seq_cst_bfloat_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1096[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1096;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic bfloat, ptr addrspace(5) %slot syncscope("") seq_cst, align 2
+ ret bfloat %retval
+}
+
+define bfloat @seq_cst_bfloat_local_cta() {
+; CHECK-LABEL: seq_cst_bfloat_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1097[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1097;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic bfloat, ptr addrspace(5) %slot syncscope("block") seq_cst, align 2
+ ret bfloat %retval
+}
+
+define bfloat @seq_cst_bfloat_local_cluster() {
+; CHECK-LABEL: seq_cst_bfloat_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1098[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1098;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic bfloat, ptr addrspace(5) %slot syncscope("cluster") seq_cst, align 2
+ ret bfloat %retval
+}
+
+define bfloat @seq_cst_bfloat_local_gpu() {
+; CHECK-LABEL: seq_cst_bfloat_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1099[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1099;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic bfloat, ptr addrspace(5) %slot syncscope("device") seq_cst, align 2
+ ret bfloat %retval
+}
+
+define bfloat @seq_cst_bfloat_local_sys_volatile() {
+; CHECK-LABEL: seq_cst_bfloat_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1100[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1100;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile bfloat, ptr addrspace(5) %slot syncscope("") seq_cst, align 2
+ ret bfloat %retval
+}
+
+define bfloat @seq_cst_bfloat_local_cta_volatile() {
+; CHECK-LABEL: seq_cst_bfloat_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1101[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1101;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile bfloat, ptr addrspace(5) %slot syncscope("block") seq_cst, align 2
+ ret bfloat %retval
+}
+
+define bfloat @seq_cst_bfloat_local_cluster_volatile() {
+; CHECK-LABEL: seq_cst_bfloat_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1102[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1102;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile bfloat, ptr addrspace(5) %slot syncscope("cluster") seq_cst, align 2
+ ret bfloat %retval
+}
+
+define bfloat @seq_cst_bfloat_local_gpu_volatile() {
+; CHECK-LABEL: seq_cst_bfloat_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1103[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1103;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile bfloat, ptr addrspace(5) %slot syncscope("device") seq_cst, align 2
+ ret bfloat %retval
+}
+
+define float @seq_cst_float_local_sys() {
+; CHECK-LABEL: seq_cst_float_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot1104[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1104;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic float, ptr addrspace(5) %slot syncscope("") seq_cst, align 4
+ ret float %retval
+}
+
+define float @seq_cst_float_local_cta() {
+; CHECK-LABEL: seq_cst_float_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot1105[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1105;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic float, ptr addrspace(5) %slot syncscope("block") seq_cst, align 4
+ ret float %retval
+}
+
+define float @seq_cst_float_local_cluster() {
+; CHECK-LABEL: seq_cst_float_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot1106[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1106;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic float, ptr addrspace(5) %slot syncscope("cluster") seq_cst, align 4
+ ret float %retval
+}
+
+define float @seq_cst_float_local_gpu() {
+; CHECK-LABEL: seq_cst_float_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot1107[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1107;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic float, ptr addrspace(5) %slot syncscope("device") seq_cst, align 4
+ ret float %retval
+}
+
+define float @seq_cst_float_local_sys_volatile() {
+; CHECK-LABEL: seq_cst_float_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot1108[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1108;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile float, ptr addrspace(5) %slot syncscope("") seq_cst, align 4
+ ret float %retval
+}
+
+define float @seq_cst_float_local_cta_volatile() {
+; CHECK-LABEL: seq_cst_float_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot1109[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1109;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile float, ptr addrspace(5) %slot syncscope("block") seq_cst, align 4
+ ret float %retval
+}
+
+define float @seq_cst_float_local_cluster_volatile() {
+; CHECK-LABEL: seq_cst_float_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot1110[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1110;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile float, ptr addrspace(5) %slot syncscope("cluster") seq_cst, align 4
+ ret float %retval
+}
+
+define float @seq_cst_float_local_gpu_volatile() {
+; CHECK-LABEL: seq_cst_float_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot1111[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1111;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile float, ptr addrspace(5) %slot syncscope("device") seq_cst, align 4
+ ret float %retval
+}
+
+define double @seq_cst_double_local_sys() {
+; CHECK-LABEL: seq_cst_double_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot1112[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1112;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic double, ptr addrspace(5) %slot syncscope("") seq_cst, align 8
+ ret double %retval
+}
+
+define double @seq_cst_double_local_cta() {
+; CHECK-LABEL: seq_cst_double_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot1113[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1113;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic double, ptr addrspace(5) %slot syncscope("block") seq_cst, align 8
+ ret double %retval
+}
+
+define double @seq_cst_double_local_cluster() {
+; CHECK-LABEL: seq_cst_double_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot1114[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1114;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic double, ptr addrspace(5) %slot syncscope("cluster") seq_cst, align 8
+ ret double %retval
+}
+
+define double @seq_cst_double_local_gpu() {
+; CHECK-LABEL: seq_cst_double_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot1115[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1115;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic double, ptr addrspace(5) %slot syncscope("device") seq_cst, align 8
+ ret double %retval
+}
+
+define double @seq_cst_double_local_sys_volatile() {
+; CHECK-LABEL: seq_cst_double_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot1116[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1116;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile double, ptr addrspace(5) %slot syncscope("") seq_cst, align 8
+ ret double %retval
+}
+
+define double @seq_cst_double_local_cta_volatile() {
+; CHECK-LABEL: seq_cst_double_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot1117[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1117;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile double, ptr addrspace(5) %slot syncscope("block") seq_cst, align 8
+ ret double %retval
+}
+
+define double @seq_cst_double_local_cluster_volatile() {
+; CHECK-LABEL: seq_cst_double_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot1118[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1118;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile double, ptr addrspace(5) %slot syncscope("cluster") seq_cst, align 8
+ ret double %retval
+}
+
+define double @seq_cst_double_local_gpu_volatile() {
+; CHECK-LABEL: seq_cst_double_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot1119[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1119;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile double, ptr addrspace(5) %slot syncscope("device") seq_cst, align 8
+ ret double %retval
+}
+
diff --git a/llvm/test/CodeGen/NVPTX/load-sm90.ll b/llvm/test/CodeGen/NVPTX/load-sm90.ll
new file mode 100644
index 0000000000000..f09f777c70292
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/load-sm90.ll
@@ -0,0 +1,22986 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_90 -mattr=+ptx87 | FileCheck %s
+; RUN: %if ptxas-sm_90 && ptxas-isa-8.7 %{ llc < %s -march=nvptx64 -mcpu=sm_90 -mattr=+ptx87 | %ptxas-verify -arch=sm_90 %}
+
+define i8 @notatomic_i8_generic(ptr %addr) {
+; CHECK-LABEL: notatomic_i8_generic(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i8_generic_param_0];
+; CHECK-NEXT: ld.b8 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load i8, ptr %addr
+ ret i8 %retval
+}
+
+define i8 @notatomic_i8_generic_volatile(ptr %addr) {
+; CHECK-LABEL: notatomic_i8_generic_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i8_generic_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b8 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load volatile i8, ptr %addr
+ ret i8 %retval
+}
+
+define i16 @notatomic_i16_generic(ptr %addr) {
+; CHECK-LABEL: notatomic_i16_generic(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i16_generic_param_0];
+; CHECK-NEXT: ld.b16 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load i16, ptr %addr
+ ret i16 %retval
+}
+
+define i16 @notatomic_i16_generic_volatile(ptr %addr) {
+; CHECK-LABEL: notatomic_i16_generic_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i16_generic_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b16 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load volatile i16, ptr %addr
+ ret i16 %retval
+}
+
+define i32 @notatomic_i32_generic(ptr %addr) {
+; CHECK-LABEL: notatomic_i32_generic(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i32_generic_param_0];
+; CHECK-NEXT: ld.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load i32, ptr %addr
+ ret i32 %retval
+}
+
+define i32 @notatomic_i32_generic_volatile(ptr %addr) {
+; CHECK-LABEL: notatomic_i32_generic_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i32_generic_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load volatile i32, ptr %addr
+ ret i32 %retval
+}
+
+define i64 @notatomic_i64_generic(ptr %addr) {
+; CHECK-LABEL: notatomic_i64_generic(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i64_generic_param_0];
+; CHECK-NEXT: ld.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load i64, ptr %addr
+ ret i64 %retval
+}
+
+define i64 @notatomic_i64_generic_volatile(ptr %addr) {
+; CHECK-LABEL: notatomic_i64_generic_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i64_generic_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load volatile i64, ptr %addr
+ ret i64 %retval
+}
+
+define i128 @notatomic_i128_generic(ptr %addr) {
+; CHECK-LABEL: notatomic_i128_generic(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i128_generic_param_0];
+; CHECK-NEXT: ld.v2.b64 {%rd2, %rd3}, [%rd1];
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd2, %rd3};
+; CHECK-NEXT: ret;
+ %retval = load i128, ptr %addr
+ ret i128 %retval
+}
+
+define i128 @notatomic_i128_generic_volatile(ptr %addr) {
+; CHECK-LABEL: notatomic_i128_generic_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i128_generic_volatile_param_0];
+; CHECK-NEXT: ld.volatile.v2.b64 {%rd2, %rd3}, [%rd1];
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd2, %rd3};
+; CHECK-NEXT: ret;
+ %retval = load volatile i128, ptr %addr
+ ret i128 %retval
+}
+
+define half @notatomic_half_generic(ptr %addr) {
+; CHECK-LABEL: notatomic_half_generic(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_half_generic_param_0];
+; CHECK-NEXT: ld.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load half, ptr %addr
+ ret half %retval
+}
+
+define half @notatomic_half_generic_volatile(ptr %addr) {
+; CHECK-LABEL: notatomic_half_generic_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_half_generic_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load volatile half, ptr %addr
+ ret half %retval
+}
+
+define bfloat @notatomic_bfloat_generic(ptr %addr) {
+; CHECK-LABEL: notatomic_bfloat_generic(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_bfloat_generic_param_0];
+; CHECK-NEXT: ld.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load bfloat, ptr %addr
+ ret bfloat %retval
+}
+
+define bfloat @notatomic_bfloat_generic_volatile(ptr %addr) {
+; CHECK-LABEL: notatomic_bfloat_generic_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_bfloat_generic_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load volatile bfloat, ptr %addr
+ ret bfloat %retval
+}
+
+define float @notatomic_float_generic(ptr %addr) {
+; CHECK-LABEL: notatomic_float_generic(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_float_generic_param_0];
+; CHECK-NEXT: ld.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load float, ptr %addr
+ ret float %retval
+}
+
+define float @notatomic_float_generic_volatile(ptr %addr) {
+; CHECK-LABEL: notatomic_float_generic_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_float_generic_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load volatile float, ptr %addr
+ ret float %retval
+}
+
+define double @notatomic_double_generic(ptr %addr) {
+; CHECK-LABEL: notatomic_double_generic(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_double_generic_param_0];
+; CHECK-NEXT: ld.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load double, ptr %addr
+ ret double %retval
+}
+
+define double @notatomic_double_generic_volatile(ptr %addr) {
+; CHECK-LABEL: notatomic_double_generic_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_double_generic_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load volatile double, ptr %addr
+ ret double %retval
+}
+
+define i8 @notatomic_i8_global(ptr addrspace(1) %addr) {
+; CHECK-LABEL: notatomic_i8_global(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i8_global_param_0];
+; CHECK-NEXT: ld.global.b8 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load i8, ptr addrspace(1) %addr
+ ret i8 %retval
+}
+
+define i8 @notatomic_i8_global_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: notatomic_i8_global_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i8_global_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b8 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load volatile i8, ptr addrspace(1) %addr
+ ret i8 %retval
+}
+
+define i16 @notatomic_i16_global(ptr addrspace(1) %addr) {
+; CHECK-LABEL: notatomic_i16_global(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i16_global_param_0];
+; CHECK-NEXT: ld.global.b16 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load i16, ptr addrspace(1) %addr
+ ret i16 %retval
+}
+
+define i16 @notatomic_i16_global_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: notatomic_i16_global_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i16_global_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b16 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load volatile i16, ptr addrspace(1) %addr
+ ret i16 %retval
+}
+
+define i32 @notatomic_i32_global(ptr addrspace(1) %addr) {
+; CHECK-LABEL: notatomic_i32_global(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i32_global_param_0];
+; CHECK-NEXT: ld.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load i32, ptr addrspace(1) %addr
+ ret i32 %retval
+}
+
+define i32 @notatomic_i32_global_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: notatomic_i32_global_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i32_global_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load volatile i32, ptr addrspace(1) %addr
+ ret i32 %retval
+}
+
+define i64 @notatomic_i64_global(ptr addrspace(1) %addr) {
+; CHECK-LABEL: notatomic_i64_global(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i64_global_param_0];
+; CHECK-NEXT: ld.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load i64, ptr addrspace(1) %addr
+ ret i64 %retval
+}
+
+define i64 @notatomic_i64_global_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: notatomic_i64_global_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i64_global_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load volatile i64, ptr addrspace(1) %addr
+ ret i64 %retval
+}
+
+define i128 @notatomic_i128_global(ptr addrspace(1) %addr) {
+; CHECK-LABEL: notatomic_i128_global(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i128_global_param_0];
+; CHECK-NEXT: ld.global.v2.b64 {%rd2, %rd3}, [%rd1];
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd2, %rd3};
+; CHECK-NEXT: ret;
+ %retval = load i128, ptr addrspace(1) %addr
+ ret i128 %retval
+}
+
+define i128 @notatomic_i128_global_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: notatomic_i128_global_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i128_global_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.v2.b64 {%rd2, %rd3}, [%rd1];
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd2, %rd3};
+; CHECK-NEXT: ret;
+ %retval = load volatile i128, ptr addrspace(1) %addr
+ ret i128 %retval
+}
+
+define half @notatomic_half_global(ptr addrspace(1) %addr) {
+; CHECK-LABEL: notatomic_half_global(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_half_global_param_0];
+; CHECK-NEXT: ld.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load half, ptr addrspace(1) %addr
+ ret half %retval
+}
+
+define half @notatomic_half_global_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: notatomic_half_global_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_half_global_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load volatile half, ptr addrspace(1) %addr
+ ret half %retval
+}
+
+define bfloat @notatomic_bfloat_global(ptr addrspace(1) %addr) {
+; CHECK-LABEL: notatomic_bfloat_global(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_bfloat_global_param_0];
+; CHECK-NEXT: ld.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load bfloat, ptr addrspace(1) %addr
+ ret bfloat %retval
+}
+
+define bfloat @notatomic_bfloat_global_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: notatomic_bfloat_global_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_bfloat_global_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load volatile bfloat, ptr addrspace(1) %addr
+ ret bfloat %retval
+}
+
+define float @notatomic_float_global(ptr addrspace(1) %addr) {
+; CHECK-LABEL: notatomic_float_global(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_float_global_param_0];
+; CHECK-NEXT: ld.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load float, ptr addrspace(1) %addr
+ ret float %retval
+}
+
+define float @notatomic_float_global_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: notatomic_float_global_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_float_global_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load volatile float, ptr addrspace(1) %addr
+ ret float %retval
+}
+
+define double @notatomic_double_global(ptr addrspace(1) %addr) {
+; CHECK-LABEL: notatomic_double_global(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_double_global_param_0];
+; CHECK-NEXT: ld.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load double, ptr addrspace(1) %addr
+ ret double %retval
+}
+
+define double @notatomic_double_global_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: notatomic_double_global_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_double_global_volatile_param_0];
+; CHECK-NEXT: ld.volatile.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load volatile double, ptr addrspace(1) %addr
+ ret double %retval
+}
+
+define i8 @notatomic_i8_shared(ptr addrspace(3) %addr) {
+; CHECK-LABEL: notatomic_i8_shared(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i8_shared_param_0];
+; CHECK-NEXT: ld.shared.b8 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load i8, ptr addrspace(3) %addr
+ ret i8 %retval
+}
+
+define i8 @notatomic_i8_shared_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: notatomic_i8_shared_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i8_shared_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b8 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load volatile i8, ptr addrspace(3) %addr
+ ret i8 %retval
+}
+
+define i16 @notatomic_i16_shared(ptr addrspace(3) %addr) {
+; CHECK-LABEL: notatomic_i16_shared(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i16_shared_param_0];
+; CHECK-NEXT: ld.shared.b16 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load i16, ptr addrspace(3) %addr
+ ret i16 %retval
+}
+
+define i16 @notatomic_i16_shared_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: notatomic_i16_shared_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i16_shared_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load volatile i16, ptr addrspace(3) %addr
+ ret i16 %retval
+}
+
+define i32 @notatomic_i32_shared(ptr addrspace(3) %addr) {
+; CHECK-LABEL: notatomic_i32_shared(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i32_shared_param_0];
+; CHECK-NEXT: ld.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load i32, ptr addrspace(3) %addr
+ ret i32 %retval
+}
+
+define i32 @notatomic_i32_shared_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: notatomic_i32_shared_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i32_shared_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load volatile i32, ptr addrspace(3) %addr
+ ret i32 %retval
+}
+
+define i64 @notatomic_i64_shared(ptr addrspace(3) %addr) {
+; CHECK-LABEL: notatomic_i64_shared(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i64_shared_param_0];
+; CHECK-NEXT: ld.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load i64, ptr addrspace(3) %addr
+ ret i64 %retval
+}
+
+define i64 @notatomic_i64_shared_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: notatomic_i64_shared_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i64_shared_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load volatile i64, ptr addrspace(3) %addr
+ ret i64 %retval
+}
+
+define i128 @notatomic_i128_shared(ptr addrspace(3) %addr) {
+; CHECK-LABEL: notatomic_i128_shared(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i128_shared_param_0];
+; CHECK-NEXT: ld.shared.v2.b64 {%rd2, %rd3}, [%rd1];
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd2, %rd3};
+; CHECK-NEXT: ret;
+ %retval = load i128, ptr addrspace(3) %addr
+ ret i128 %retval
+}
+
+define i128 @notatomic_i128_shared_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: notatomic_i128_shared_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i128_shared_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.v2.b64 {%rd2, %rd3}, [%rd1];
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd2, %rd3};
+; CHECK-NEXT: ret;
+ %retval = load volatile i128, ptr addrspace(3) %addr
+ ret i128 %retval
+}
+
+define half @notatomic_half_shared(ptr addrspace(3) %addr) {
+; CHECK-LABEL: notatomic_half_shared(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_half_shared_param_0];
+; CHECK-NEXT: ld.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load half, ptr addrspace(3) %addr
+ ret half %retval
+}
+
+define half @notatomic_half_shared_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: notatomic_half_shared_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_half_shared_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load volatile half, ptr addrspace(3) %addr
+ ret half %retval
+}
+
+define bfloat @notatomic_bfloat_shared(ptr addrspace(3) %addr) {
+; CHECK-LABEL: notatomic_bfloat_shared(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_bfloat_shared_param_0];
+; CHECK-NEXT: ld.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load bfloat, ptr addrspace(3) %addr
+ ret bfloat %retval
+}
+
+define bfloat @notatomic_bfloat_shared_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: notatomic_bfloat_shared_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_bfloat_shared_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load volatile bfloat, ptr addrspace(3) %addr
+ ret bfloat %retval
+}
+
+define float @notatomic_float_shared(ptr addrspace(3) %addr) {
+; CHECK-LABEL: notatomic_float_shared(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_float_shared_param_0];
+; CHECK-NEXT: ld.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load float, ptr addrspace(3) %addr
+ ret float %retval
+}
+
+define float @notatomic_float_shared_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: notatomic_float_shared_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_float_shared_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load volatile float, ptr addrspace(3) %addr
+ ret float %retval
+}
+
+define double @notatomic_double_shared(ptr addrspace(3) %addr) {
+; CHECK-LABEL: notatomic_double_shared(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_double_shared_param_0];
+; CHECK-NEXT: ld.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load double, ptr addrspace(3) %addr
+ ret double %retval
+}
+
+define double @notatomic_double_shared_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: notatomic_double_shared_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_double_shared_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load volatile double, ptr addrspace(3) %addr
+ ret double %retval
+}
+
+define i8 @notatomic_i8_const(ptr addrspace(4) %addr) {
+; CHECK-LABEL: notatomic_i8_const(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i8_const_param_0];
+; CHECK-NEXT: ld.const.b8 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load i8, ptr addrspace(4) %addr
+ ret i8 %retval
+}
+
+define i8 @notatomic_i8_const_volatile(ptr addrspace(4) %addr) {
+; CHECK-LABEL: notatomic_i8_const_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i8_const_volatile_param_0];
+; CHECK-NEXT: ld.const.b8 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load volatile i8, ptr addrspace(4) %addr
+ ret i8 %retval
+}
+
+define i16 @notatomic_i16_const(ptr addrspace(4) %addr) {
+; CHECK-LABEL: notatomic_i16_const(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i16_const_param_0];
+; CHECK-NEXT: ld.const.b16 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load i16, ptr addrspace(4) %addr
+ ret i16 %retval
+}
+
+define i16 @notatomic_i16_const_volatile(ptr addrspace(4) %addr) {
+; CHECK-LABEL: notatomic_i16_const_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i16_const_volatile_param_0];
+; CHECK-NEXT: ld.const.b16 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load volatile i16, ptr addrspace(4) %addr
+ ret i16 %retval
+}
+
+define i32 @notatomic_i32_const(ptr addrspace(4) %addr) {
+; CHECK-LABEL: notatomic_i32_const(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i32_const_param_0];
+; CHECK-NEXT: ld.const.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load i32, ptr addrspace(4) %addr
+ ret i32 %retval
+}
+
+define i32 @notatomic_i32_const_volatile(ptr addrspace(4) %addr) {
+; CHECK-LABEL: notatomic_i32_const_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i32_const_volatile_param_0];
+; CHECK-NEXT: ld.const.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load volatile i32, ptr addrspace(4) %addr
+ ret i32 %retval
+}
+
+define i64 @notatomic_i64_const(ptr addrspace(4) %addr) {
+; CHECK-LABEL: notatomic_i64_const(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i64_const_param_0];
+; CHECK-NEXT: ld.const.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load i64, ptr addrspace(4) %addr
+ ret i64 %retval
+}
+
+define i64 @notatomic_i64_const_volatile(ptr addrspace(4) %addr) {
+; CHECK-LABEL: notatomic_i64_const_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i64_const_volatile_param_0];
+; CHECK-NEXT: ld.const.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load volatile i64, ptr addrspace(4) %addr
+ ret i64 %retval
+}
+
+define i128 @notatomic_i128_const(ptr addrspace(4) %addr) {
+; CHECK-LABEL: notatomic_i128_const(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i128_const_param_0];
+; CHECK-NEXT: ld.const.v2.b64 {%rd2, %rd3}, [%rd1];
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd2, %rd3};
+; CHECK-NEXT: ret;
+ %retval = load i128, ptr addrspace(4) %addr
+ ret i128 %retval
+}
+
+define i128 @notatomic_i128_const_volatile(ptr addrspace(4) %addr) {
+; CHECK-LABEL: notatomic_i128_const_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i128_const_volatile_param_0];
+; CHECK-NEXT: ld.const.v2.b64 {%rd2, %rd3}, [%rd1];
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd2, %rd3};
+; CHECK-NEXT: ret;
+ %retval = load volatile i128, ptr addrspace(4) %addr
+ ret i128 %retval
+}
+
+define half @notatomic_half_const(ptr addrspace(4) %addr) {
+; CHECK-LABEL: notatomic_half_const(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_half_const_param_0];
+; CHECK-NEXT: ld.const.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load half, ptr addrspace(4) %addr
+ ret half %retval
+}
+
+define half @notatomic_half_const_volatile(ptr addrspace(4) %addr) {
+; CHECK-LABEL: notatomic_half_const_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_half_const_volatile_param_0];
+; CHECK-NEXT: ld.const.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load volatile half, ptr addrspace(4) %addr
+ ret half %retval
+}
+
+define bfloat @notatomic_bfloat_const(ptr addrspace(4) %addr) {
+; CHECK-LABEL: notatomic_bfloat_const(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_bfloat_const_param_0];
+; CHECK-NEXT: ld.const.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load bfloat, ptr addrspace(4) %addr
+ ret bfloat %retval
+}
+
+define bfloat @notatomic_bfloat_const_volatile(ptr addrspace(4) %addr) {
+; CHECK-LABEL: notatomic_bfloat_const_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_bfloat_const_volatile_param_0];
+; CHECK-NEXT: ld.const.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load volatile bfloat, ptr addrspace(4) %addr
+ ret bfloat %retval
+}
+
+define float @notatomic_float_const(ptr addrspace(4) %addr) {
+; CHECK-LABEL: notatomic_float_const(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_float_const_param_0];
+; CHECK-NEXT: ld.const.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load float, ptr addrspace(4) %addr
+ ret float %retval
+}
+
+define float @notatomic_float_const_volatile(ptr addrspace(4) %addr) {
+; CHECK-LABEL: notatomic_float_const_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_float_const_volatile_param_0];
+; CHECK-NEXT: ld.const.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load volatile float, ptr addrspace(4) %addr
+ ret float %retval
+}
+
+define double @notatomic_double_const(ptr addrspace(4) %addr) {
+; CHECK-LABEL: notatomic_double_const(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_double_const_param_0];
+; CHECK-NEXT: ld.const.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load double, ptr addrspace(4) %addr
+ ret double %retval
+}
+
+define double @notatomic_double_const_volatile(ptr addrspace(4) %addr) {
+; CHECK-LABEL: notatomic_double_const_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_double_const_volatile_param_0];
+; CHECK-NEXT: ld.const.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load volatile double, ptr addrspace(4) %addr
+ ret double %retval
+}
+
+define i8 @notatomic_i8_local() {
+; CHECK-LABEL: notatomic_i8_local(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot72[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot72;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b8 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load i8, ptr addrspace(5) %slot
+ ret i8 %retval
+}
+
+define i8 @notatomic_i8_local_volatile() {
+; CHECK-LABEL: notatomic_i8_local_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot73[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot73;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b8 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load volatile i8, ptr addrspace(5) %slot
+ ret i8 %retval
+}
+
+define i16 @notatomic_i16_local() {
+; CHECK-LABEL: notatomic_i16_local(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot74[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot74;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load i16, ptr addrspace(5) %slot
+ ret i16 %retval
+}
+
+define i16 @notatomic_i16_local_volatile() {
+; CHECK-LABEL: notatomic_i16_local_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot75[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot75;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load volatile i16, ptr addrspace(5) %slot
+ ret i16 %retval
+}
+
+define i32 @notatomic_i32_local() {
+; CHECK-LABEL: notatomic_i32_local(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot76[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot76;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load i32, ptr addrspace(5) %slot
+ ret i32 %retval
+}
+
+define i32 @notatomic_i32_local_volatile() {
+; CHECK-LABEL: notatomic_i32_local_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot77[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot77;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load volatile i32, ptr addrspace(5) %slot
+ ret i32 %retval
+}
+
+define i64 @notatomic_i64_local() {
+; CHECK-LABEL: notatomic_i64_local(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot78[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot78;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load i64, ptr addrspace(5) %slot
+ ret i64 %retval
+}
+
+define i64 @notatomic_i64_local_volatile() {
+; CHECK-LABEL: notatomic_i64_local_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot79[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot79;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load volatile i64, ptr addrspace(5) %slot
+ ret i64 %retval
+}
+
+define i128 @notatomic_i128_local() {
+; CHECK-LABEL: notatomic_i128_local(
+; CHECK: {
+; CHECK-NEXT: .local .align 16 .b8 __local_depot80[16];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot80;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.v2.b64 {%rd2, %rd3}, [%SP];
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd2, %rd3};
+; CHECK-NEXT: ret;
+ %slot = alloca i128, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load i128, ptr addrspace(5) %slot
+ ret i128 %retval
+}
+
+define i128 @notatomic_i128_local_volatile() {
+; CHECK-LABEL: notatomic_i128_local_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 16 .b8 __local_depot81[16];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot81;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.v2.b64 {%rd2, %rd3}, [%SP];
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd2, %rd3};
+; CHECK-NEXT: ret;
+ %slot = alloca i128, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load volatile i128, ptr addrspace(5) %slot
+ ret i128 %retval
+}
+
+define half @notatomic_half_local() {
+; CHECK-LABEL: notatomic_half_local(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot82[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot82;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load half, ptr addrspace(5) %slot
+ ret half %retval
+}
+
+define half @notatomic_half_local_volatile() {
+; CHECK-LABEL: notatomic_half_local_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot83[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot83;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load volatile half, ptr addrspace(5) %slot
+ ret half %retval
+}
+
+define bfloat @notatomic_bfloat_local() {
+; CHECK-LABEL: notatomic_bfloat_local(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot84[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot84;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load bfloat, ptr addrspace(5) %slot
+ ret bfloat %retval
+}
+
+define bfloat @notatomic_bfloat_local_volatile() {
+; CHECK-LABEL: notatomic_bfloat_local_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot85[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot85;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load volatile bfloat, ptr addrspace(5) %slot
+ ret bfloat %retval
+}
+
+define float @notatomic_float_local() {
+; CHECK-LABEL: notatomic_float_local(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot86[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot86;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load float, ptr addrspace(5) %slot
+ ret float %retval
+}
+
+define float @notatomic_float_local_volatile() {
+; CHECK-LABEL: notatomic_float_local_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot87[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot87;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load volatile float, ptr addrspace(5) %slot
+ ret float %retval
+}
+
+define double @notatomic_double_local() {
+; CHECK-LABEL: notatomic_double_local(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot88[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot88;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca double, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load double, ptr addrspace(5) %slot
+ ret double %retval
+}
+
+define double @notatomic_double_local_volatile() {
+; CHECK-LABEL: notatomic_double_local_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot89[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot89;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca double, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load volatile double, ptr addrspace(5) %slot
+ ret double %retval
+}
+
+define i8 @notatomic_i8_param(ptr addrspace(101) %addr) {
+; CHECK-LABEL: notatomic_i8_param(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i8_param_param_0];
+; CHECK-NEXT: ld.param.b8 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load i8, ptr addrspace(101) %addr
+ ret i8 %retval
+}
+
+define i8 @notatomic_i8_param_volatile(ptr addrspace(101) %addr) {
+; CHECK-LABEL: notatomic_i8_param_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i8_param_volatile_param_0];
+; CHECK-NEXT: ld.param.b8 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load volatile i8, ptr addrspace(101) %addr
+ ret i8 %retval
+}
+
+define i16 @notatomic_i16_param(ptr addrspace(101) %addr) {
+; CHECK-LABEL: notatomic_i16_param(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i16_param_param_0];
+; CHECK-NEXT: ld.param.b16 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load i16, ptr addrspace(101) %addr
+ ret i16 %retval
+}
+
+define i16 @notatomic_i16_param_volatile(ptr addrspace(101) %addr) {
+; CHECK-LABEL: notatomic_i16_param_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i16_param_volatile_param_0];
+; CHECK-NEXT: ld.param.b16 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load volatile i16, ptr addrspace(101) %addr
+ ret i16 %retval
+}
+
+define i32 @notatomic_i32_param(ptr addrspace(101) %addr) {
+; CHECK-LABEL: notatomic_i32_param(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i32_param_param_0];
+; CHECK-NEXT: ld.param.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load i32, ptr addrspace(101) %addr
+ ret i32 %retval
+}
+
+define i32 @notatomic_i32_param_volatile(ptr addrspace(101) %addr) {
+; CHECK-LABEL: notatomic_i32_param_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i32_param_volatile_param_0];
+; CHECK-NEXT: ld.param.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load volatile i32, ptr addrspace(101) %addr
+ ret i32 %retval
+}
+
+define i64 @notatomic_i64_param(ptr addrspace(101) %addr) {
+; CHECK-LABEL: notatomic_i64_param(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i64_param_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load i64, ptr addrspace(101) %addr
+ ret i64 %retval
+}
+
+define i64 @notatomic_i64_param_volatile(ptr addrspace(101) %addr) {
+; CHECK-LABEL: notatomic_i64_param_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i64_param_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load volatile i64, ptr addrspace(101) %addr
+ ret i64 %retval
+}
+
+define i128 @notatomic_i128_param(ptr addrspace(101) %addr) {
+; CHECK-LABEL: notatomic_i128_param(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i128_param_param_0];
+; CHECK-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [%rd1];
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd2, %rd3};
+; CHECK-NEXT: ret;
+ %retval = load i128, ptr addrspace(101) %addr
+ ret i128 %retval
+}
+
+define i128 @notatomic_i128_param_volatile(ptr addrspace(101) %addr) {
+; CHECK-LABEL: notatomic_i128_param_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i128_param_volatile_param_0];
+; CHECK-NEXT: ld.param.v2.b64 {%rd2, %rd3}, [%rd1];
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd2, %rd3};
+; CHECK-NEXT: ret;
+ %retval = load volatile i128, ptr addrspace(101) %addr
+ ret i128 %retval
+}
+
+define half @notatomic_half_param(ptr addrspace(101) %addr) {
+; CHECK-LABEL: notatomic_half_param(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_half_param_param_0];
+; CHECK-NEXT: ld.param.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load half, ptr addrspace(101) %addr
+ ret half %retval
+}
+
+define half @notatomic_half_param_volatile(ptr addrspace(101) %addr) {
+; CHECK-LABEL: notatomic_half_param_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_half_param_volatile_param_0];
+; CHECK-NEXT: ld.param.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load volatile half, ptr addrspace(101) %addr
+ ret half %retval
+}
+
+define bfloat @notatomic_bfloat_param(ptr addrspace(101) %addr) {
+; CHECK-LABEL: notatomic_bfloat_param(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_bfloat_param_param_0];
+; CHECK-NEXT: ld.param.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load bfloat, ptr addrspace(101) %addr
+ ret bfloat %retval
+}
+
+define bfloat @notatomic_bfloat_param_volatile(ptr addrspace(101) %addr) {
+; CHECK-LABEL: notatomic_bfloat_param_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_bfloat_param_volatile_param_0];
+; CHECK-NEXT: ld.param.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load volatile bfloat, ptr addrspace(101) %addr
+ ret bfloat %retval
+}
+
+define float @notatomic_float_param(ptr addrspace(101) %addr) {
+; CHECK-LABEL: notatomic_float_param(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_float_param_param_0];
+; CHECK-NEXT: ld.param.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load float, ptr addrspace(101) %addr
+ ret float %retval
+}
+
+define float @notatomic_float_param_volatile(ptr addrspace(101) %addr) {
+; CHECK-LABEL: notatomic_float_param_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_float_param_volatile_param_0];
+; CHECK-NEXT: ld.param.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load volatile float, ptr addrspace(101) %addr
+ ret float %retval
+}
+
+define double @notatomic_double_param(ptr addrspace(101) %addr) {
+; CHECK-LABEL: notatomic_double_param(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_double_param_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load double, ptr addrspace(101) %addr
+ ret double %retval
+}
+
+define double @notatomic_double_param_volatile(ptr addrspace(101) %addr) {
+; CHECK-LABEL: notatomic_double_param_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_double_param_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load volatile double, ptr addrspace(101) %addr
+ ret double %retval
+}
+
+define i8 @unordered_i8_generic_sys(ptr %addr) {
+; CHECK-LABEL: unordered_i8_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_generic_sys_param_0];
+; CHECK-NEXT: ld.relaxed.sys.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr %addr syncscope("") unordered, align 1
+ ret i8 %retval
+}
+
+define i8 @unordered_i8_generic_cta(ptr %addr) {
+; CHECK-LABEL: unordered_i8_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_generic_cta_param_0];
+; CHECK-NEXT: ld.relaxed.cta.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr %addr syncscope("block") unordered, align 1
+ ret i8 %retval
+}
+
+define i8 @unordered_i8_generic_cluster(ptr %addr) {
+; CHECK-LABEL: unordered_i8_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_generic_cluster_param_0];
+; CHECK-NEXT: ld.relaxed.cluster.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr %addr syncscope("cluster") unordered, align 1
+ ret i8 %retval
+}
+
+define i8 @unordered_i8_generic_gpu(ptr %addr) {
+; CHECK-LABEL: unordered_i8_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_generic_gpu_param_0];
+; CHECK-NEXT: ld.relaxed.gpu.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr %addr syncscope("device") unordered, align 1
+ ret i8 %retval
+}
+
+define i8 @unordered_i8_generic_sys_volatile(ptr %addr) {
+; CHECK-LABEL: unordered_i8_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr %addr syncscope("") unordered, align 1
+ ret i8 %retval
+}
+
+define i8 @unordered_i8_generic_cta_volatile(ptr %addr) {
+; CHECK-LABEL: unordered_i8_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr %addr syncscope("block") unordered, align 1
+ ret i8 %retval
+}
+
+define i8 @unordered_i8_generic_cluster_volatile(ptr %addr) {
+; CHECK-LABEL: unordered_i8_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr %addr syncscope("cluster") unordered, align 1
+ ret i8 %retval
+}
+
+define i8 @unordered_i8_generic_gpu_volatile(ptr %addr) {
+; CHECK-LABEL: unordered_i8_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr %addr syncscope("device") unordered, align 1
+ ret i8 %retval
+}
+
+define i16 @unordered_i16_generic_sys(ptr %addr) {
+; CHECK-LABEL: unordered_i16_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_generic_sys_param_0];
+; CHECK-NEXT: ld.relaxed.sys.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr %addr syncscope("") unordered, align 2
+ ret i16 %retval
+}
+
+define i16 @unordered_i16_generic_cta(ptr %addr) {
+; CHECK-LABEL: unordered_i16_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_generic_cta_param_0];
+; CHECK-NEXT: ld.relaxed.cta.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr %addr syncscope("block") unordered, align 2
+ ret i16 %retval
+}
+
+define i16 @unordered_i16_generic_cluster(ptr %addr) {
+; CHECK-LABEL: unordered_i16_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_generic_cluster_param_0];
+; CHECK-NEXT: ld.relaxed.cluster.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr %addr syncscope("cluster") unordered, align 2
+ ret i16 %retval
+}
+
+define i16 @unordered_i16_generic_gpu(ptr %addr) {
+; CHECK-LABEL: unordered_i16_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_generic_gpu_param_0];
+; CHECK-NEXT: ld.relaxed.gpu.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr %addr syncscope("device") unordered, align 2
+ ret i16 %retval
+}
+
+define i16 @unordered_i16_generic_sys_volatile(ptr %addr) {
+; CHECK-LABEL: unordered_i16_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr %addr syncscope("") unordered, align 2
+ ret i16 %retval
+}
+
+define i16 @unordered_i16_generic_cta_volatile(ptr %addr) {
+; CHECK-LABEL: unordered_i16_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr %addr syncscope("block") unordered, align 2
+ ret i16 %retval
+}
+
+define i16 @unordered_i16_generic_cluster_volatile(ptr %addr) {
+; CHECK-LABEL: unordered_i16_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr %addr syncscope("cluster") unordered, align 2
+ ret i16 %retval
+}
+
+define i16 @unordered_i16_generic_gpu_volatile(ptr %addr) {
+; CHECK-LABEL: unordered_i16_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr %addr syncscope("device") unordered, align 2
+ ret i16 %retval
+}
+
+define i32 @unordered_i32_generic_sys(ptr %addr) {
+; CHECK-LABEL: unordered_i32_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_generic_sys_param_0];
+; CHECK-NEXT: ld.relaxed.sys.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr %addr syncscope("") unordered, align 4
+ ret i32 %retval
+}
+
+define i32 @unordered_i32_generic_cta(ptr %addr) {
+; CHECK-LABEL: unordered_i32_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_generic_cta_param_0];
+; CHECK-NEXT: ld.relaxed.cta.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr %addr syncscope("block") unordered, align 4
+ ret i32 %retval
+}
+
+define i32 @unordered_i32_generic_cluster(ptr %addr) {
+; CHECK-LABEL: unordered_i32_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_generic_cluster_param_0];
+; CHECK-NEXT: ld.relaxed.cluster.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr %addr syncscope("cluster") unordered, align 4
+ ret i32 %retval
+}
+
+define i32 @unordered_i32_generic_gpu(ptr %addr) {
+; CHECK-LABEL: unordered_i32_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_generic_gpu_param_0];
+; CHECK-NEXT: ld.relaxed.gpu.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr %addr syncscope("device") unordered, align 4
+ ret i32 %retval
+}
+
+define i32 @unordered_i32_generic_sys_volatile(ptr %addr) {
+; CHECK-LABEL: unordered_i32_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr %addr syncscope("") unordered, align 4
+ ret i32 %retval
+}
+
+define i32 @unordered_i32_generic_cta_volatile(ptr %addr) {
+; CHECK-LABEL: unordered_i32_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr %addr syncscope("block") unordered, align 4
+ ret i32 %retval
+}
+
+define i32 @unordered_i32_generic_cluster_volatile(ptr %addr) {
+; CHECK-LABEL: unordered_i32_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr %addr syncscope("cluster") unordered, align 4
+ ret i32 %retval
+}
+
+define i32 @unordered_i32_generic_gpu_volatile(ptr %addr) {
+; CHECK-LABEL: unordered_i32_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr %addr syncscope("device") unordered, align 4
+ ret i32 %retval
+}
+
+define i64 @unordered_i64_generic_sys(ptr %addr) {
+; CHECK-LABEL: unordered_i64_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_generic_sys_param_0];
+; CHECK-NEXT: ld.relaxed.sys.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr %addr syncscope("") unordered, align 8
+ ret i64 %retval
+}
+
+define i64 @unordered_i64_generic_cta(ptr %addr) {
+; CHECK-LABEL: unordered_i64_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_generic_cta_param_0];
+; CHECK-NEXT: ld.relaxed.cta.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr %addr syncscope("block") unordered, align 8
+ ret i64 %retval
+}
+
+define i64 @unordered_i64_generic_cluster(ptr %addr) {
+; CHECK-LABEL: unordered_i64_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_generic_cluster_param_0];
+; CHECK-NEXT: ld.relaxed.cluster.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr %addr syncscope("cluster") unordered, align 8
+ ret i64 %retval
+}
+
+define i64 @unordered_i64_generic_gpu(ptr %addr) {
+; CHECK-LABEL: unordered_i64_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_generic_gpu_param_0];
+; CHECK-NEXT: ld.relaxed.gpu.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr %addr syncscope("device") unordered, align 8
+ ret i64 %retval
+}
+
+define i64 @unordered_i64_generic_sys_volatile(ptr %addr) {
+; CHECK-LABEL: unordered_i64_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr %addr syncscope("") unordered, align 8
+ ret i64 %retval
+}
+
+define i64 @unordered_i64_generic_cta_volatile(ptr %addr) {
+; CHECK-LABEL: unordered_i64_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr %addr syncscope("block") unordered, align 8
+ ret i64 %retval
+}
+
+define i64 @unordered_i64_generic_cluster_volatile(ptr %addr) {
+; CHECK-LABEL: unordered_i64_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr %addr syncscope("cluster") unordered, align 8
+ ret i64 %retval
+}
+
+define i64 @unordered_i64_generic_gpu_volatile(ptr %addr) {
+; CHECK-LABEL: unordered_i64_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr %addr syncscope("device") unordered, align 8
+ ret i64 %retval
+}
+
+define i128 @unordered_i128_generic_sys(ptr %addr) {
+; CHECK-LABEL: unordered_i128_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i128_generic_sys_param_0];
+; CHECK-NEXT: mov.b64 %rd2, 0;
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 cmp, swap, dst;
+; CHECK-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; CHECK-NEXT: mov.b128 swap, {%rd2, %rd2};
+; CHECK-NEXT: atom.relaxed.sys.cas.b128 dst, [%rd1], cmp, swap;
+; CHECK-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; CHECK-NEXT: ret;
+ %retval = load atomic i128, ptr %addr syncscope("") unordered, align 16
+ ret i128 %retval
+}
+
+define i128 @unordered_i128_generic_cta(ptr %addr) {
+; CHECK-LABEL: unordered_i128_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i128_generic_cta_param_0];
+; CHECK-NEXT: mov.b64 %rd2, 0;
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 cmp, swap, dst;
+; CHECK-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; CHECK-NEXT: mov.b128 swap, {%rd2, %rd2};
+; CHECK-NEXT: atom.relaxed.cta.cas.b128 dst, [%rd1], cmp, swap;
+; CHECK-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; CHECK-NEXT: ret;
+ %retval = load atomic i128, ptr %addr syncscope("block") unordered, align 16
+ ret i128 %retval
+}
+
+define i128 @unordered_i128_generic_cluster(ptr %addr) {
+; CHECK-LABEL: unordered_i128_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i128_generic_cluster_param_0];
+; CHECK-NEXT: mov.b64 %rd2, 0;
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 cmp, swap, dst;
+; CHECK-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; CHECK-NEXT: mov.b128 swap, {%rd2, %rd2};
+; CHECK-NEXT: atom.relaxed.cluster.cas.b128 dst, [%rd1], cmp, swap;
+; CHECK-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; CHECK-NEXT: ret;
+ %retval = load atomic i128, ptr %addr syncscope("cluster") unordered, align 16
+ ret i128 %retval
+}
+
+define i128 @unordered_i128_generic_gpu(ptr %addr) {
+; CHECK-LABEL: unordered_i128_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i128_generic_gpu_param_0];
+; CHECK-NEXT: mov.b64 %rd2, 0;
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 cmp, swap, dst;
+; CHECK-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; CHECK-NEXT: mov.b128 swap, {%rd2, %rd2};
+; CHECK-NEXT: atom.relaxed.gpu.cas.b128 dst, [%rd1], cmp, swap;
+; CHECK-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; CHECK-NEXT: ret;
+ %retval = load atomic i128, ptr %addr syncscope("device") unordered, align 16
+ ret i128 %retval
+}
+
+define i128 @unordered_i128_generic_sys_volatile(ptr %addr) {
+; CHECK-LABEL: unordered_i128_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i128_generic_sys_volatile_param_0];
+; CHECK-NEXT: mov.b64 %rd2, 0;
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 cmp, swap, dst;
+; CHECK-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; CHECK-NEXT: mov.b128 swap, {%rd2, %rd2};
+; CHECK-NEXT: atom.relaxed.sys.cas.b128 dst, [%rd1], cmp, swap;
+; CHECK-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i128, ptr %addr syncscope("") unordered, align 16
+ ret i128 %retval
+}
+
+define i128 @unordered_i128_generic_cta_volatile(ptr %addr) {
+; CHECK-LABEL: unordered_i128_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i128_generic_cta_volatile_param_0];
+; CHECK-NEXT: mov.b64 %rd2, 0;
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 cmp, swap, dst;
+; CHECK-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; CHECK-NEXT: mov.b128 swap, {%rd2, %rd2};
+; CHECK-NEXT: atom.relaxed.cta.cas.b128 dst, [%rd1], cmp, swap;
+; CHECK-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i128, ptr %addr syncscope("block") unordered, align 16
+ ret i128 %retval
+}
+
+define i128 @unordered_i128_generic_cluster_volatile(ptr %addr) {
+; CHECK-LABEL: unordered_i128_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i128_generic_cluster_volatile_param_0];
+; CHECK-NEXT: mov.b64 %rd2, 0;
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 cmp, swap, dst;
+; CHECK-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; CHECK-NEXT: mov.b128 swap, {%rd2, %rd2};
+; CHECK-NEXT: atom.relaxed.cluster.cas.b128 dst, [%rd1], cmp, swap;
+; CHECK-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i128, ptr %addr syncscope("cluster") unordered, align 16
+ ret i128 %retval
+}
+
+define i128 @unordered_i128_generic_gpu_volatile(ptr %addr) {
+; CHECK-LABEL: unordered_i128_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i128_generic_gpu_volatile_param_0];
+; CHECK-NEXT: mov.b64 %rd2, 0;
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 cmp, swap, dst;
+; CHECK-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; CHECK-NEXT: mov.b128 swap, {%rd2, %rd2};
+; CHECK-NEXT: atom.relaxed.gpu.cas.b128 dst, [%rd1], cmp, swap;
+; CHECK-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i128, ptr %addr syncscope("device") unordered, align 16
+ ret i128 %retval
+}
+
+define half @unordered_half_generic_sys(ptr %addr) {
+; CHECK-LABEL: unordered_half_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_generic_sys_param_0];
+; CHECK-NEXT: ld.relaxed.sys.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr %addr syncscope("") unordered, align 2
+ ret half %retval
+}
+
+define half @unordered_half_generic_cta(ptr %addr) {
+; CHECK-LABEL: unordered_half_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_generic_cta_param_0];
+; CHECK-NEXT: ld.relaxed.cta.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr %addr syncscope("block") unordered, align 2
+ ret half %retval
+}
+
+define half @unordered_half_generic_cluster(ptr %addr) {
+; CHECK-LABEL: unordered_half_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_generic_cluster_param_0];
+; CHECK-NEXT: ld.relaxed.cluster.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr %addr syncscope("cluster") unordered, align 2
+ ret half %retval
+}
+
+define half @unordered_half_generic_gpu(ptr %addr) {
+; CHECK-LABEL: unordered_half_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_generic_gpu_param_0];
+; CHECK-NEXT: ld.relaxed.gpu.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr %addr syncscope("device") unordered, align 2
+ ret half %retval
+}
+
+define half @unordered_half_generic_sys_volatile(ptr %addr) {
+; CHECK-LABEL: unordered_half_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr %addr syncscope("") unordered, align 2
+ ret half %retval
+}
+
+define half @unordered_half_generic_cta_volatile(ptr %addr) {
+; CHECK-LABEL: unordered_half_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr %addr syncscope("block") unordered, align 2
+ ret half %retval
+}
+
+define half @unordered_half_generic_cluster_volatile(ptr %addr) {
+; CHECK-LABEL: unordered_half_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr %addr syncscope("cluster") unordered, align 2
+ ret half %retval
+}
+
+define half @unordered_half_generic_gpu_volatile(ptr %addr) {
+; CHECK-LABEL: unordered_half_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr %addr syncscope("device") unordered, align 2
+ ret half %retval
+}
+
+define bfloat @unordered_bfloat_generic_sys(ptr %addr) {
+; CHECK-LABEL: unordered_bfloat_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_generic_sys_param_0];
+; CHECK-NEXT: ld.relaxed.sys.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr %addr syncscope("") unordered, align 2
+ ret bfloat %retval
+}
+
+define bfloat @unordered_bfloat_generic_cta(ptr %addr) {
+; CHECK-LABEL: unordered_bfloat_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_generic_cta_param_0];
+; CHECK-NEXT: ld.relaxed.cta.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr %addr syncscope("block") unordered, align 2
+ ret bfloat %retval
+}
+
+define bfloat @unordered_bfloat_generic_cluster(ptr %addr) {
+; CHECK-LABEL: unordered_bfloat_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_generic_cluster_param_0];
+; CHECK-NEXT: ld.relaxed.cluster.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr %addr syncscope("cluster") unordered, align 2
+ ret bfloat %retval
+}
+
+define bfloat @unordered_bfloat_generic_gpu(ptr %addr) {
+; CHECK-LABEL: unordered_bfloat_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_generic_gpu_param_0];
+; CHECK-NEXT: ld.relaxed.gpu.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr %addr syncscope("device") unordered, align 2
+ ret bfloat %retval
+}
+
+define bfloat @unordered_bfloat_generic_sys_volatile(ptr %addr) {
+; CHECK-LABEL: unordered_bfloat_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr %addr syncscope("") unordered, align 2
+ ret bfloat %retval
+}
+
+define bfloat @unordered_bfloat_generic_cta_volatile(ptr %addr) {
+; CHECK-LABEL: unordered_bfloat_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr %addr syncscope("block") unordered, align 2
+ ret bfloat %retval
+}
+
+define bfloat @unordered_bfloat_generic_cluster_volatile(ptr %addr) {
+; CHECK-LABEL: unordered_bfloat_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr %addr syncscope("cluster") unordered, align 2
+ ret bfloat %retval
+}
+
+define bfloat @unordered_bfloat_generic_gpu_volatile(ptr %addr) {
+; CHECK-LABEL: unordered_bfloat_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr %addr syncscope("device") unordered, align 2
+ ret bfloat %retval
+}
+
+define float @unordered_float_generic_sys(ptr %addr) {
+; CHECK-LABEL: unordered_float_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_generic_sys_param_0];
+; CHECK-NEXT: ld.relaxed.sys.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr %addr syncscope("") unordered, align 4
+ ret float %retval
+}
+
+define float @unordered_float_generic_cta(ptr %addr) {
+; CHECK-LABEL: unordered_float_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_generic_cta_param_0];
+; CHECK-NEXT: ld.relaxed.cta.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr %addr syncscope("block") unordered, align 4
+ ret float %retval
+}
+
+define float @unordered_float_generic_cluster(ptr %addr) {
+; CHECK-LABEL: unordered_float_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_generic_cluster_param_0];
+; CHECK-NEXT: ld.relaxed.cluster.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr %addr syncscope("cluster") unordered, align 4
+ ret float %retval
+}
+
+define float @unordered_float_generic_gpu(ptr %addr) {
+; CHECK-LABEL: unordered_float_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_generic_gpu_param_0];
+; CHECK-NEXT: ld.relaxed.gpu.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr %addr syncscope("device") unordered, align 4
+ ret float %retval
+}
+
+define float @unordered_float_generic_sys_volatile(ptr %addr) {
+; CHECK-LABEL: unordered_float_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr %addr syncscope("") unordered, align 4
+ ret float %retval
+}
+
+define float @unordered_float_generic_cta_volatile(ptr %addr) {
+; CHECK-LABEL: unordered_float_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr %addr syncscope("block") unordered, align 4
+ ret float %retval
+}
+
+define float @unordered_float_generic_cluster_volatile(ptr %addr) {
+; CHECK-LABEL: unordered_float_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr %addr syncscope("cluster") unordered, align 4
+ ret float %retval
+}
+
+define float @unordered_float_generic_gpu_volatile(ptr %addr) {
+; CHECK-LABEL: unordered_float_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr %addr syncscope("device") unordered, align 4
+ ret float %retval
+}
+
+define double @unordered_double_generic_sys(ptr %addr) {
+; CHECK-LABEL: unordered_double_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_generic_sys_param_0];
+; CHECK-NEXT: ld.relaxed.sys.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr %addr syncscope("") unordered, align 8
+ ret double %retval
+}
+
+define double @unordered_double_generic_cta(ptr %addr) {
+; CHECK-LABEL: unordered_double_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_generic_cta_param_0];
+; CHECK-NEXT: ld.relaxed.cta.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr %addr syncscope("block") unordered, align 8
+ ret double %retval
+}
+
+define double @unordered_double_generic_cluster(ptr %addr) {
+; CHECK-LABEL: unordered_double_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_generic_cluster_param_0];
+; CHECK-NEXT: ld.relaxed.cluster.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr %addr syncscope("cluster") unordered, align 8
+ ret double %retval
+}
+
+define double @unordered_double_generic_gpu(ptr %addr) {
+; CHECK-LABEL: unordered_double_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_generic_gpu_param_0];
+; CHECK-NEXT: ld.relaxed.gpu.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr %addr syncscope("device") unordered, align 8
+ ret double %retval
+}
+
+define double @unordered_double_generic_sys_volatile(ptr %addr) {
+; CHECK-LABEL: unordered_double_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr %addr syncscope("") unordered, align 8
+ ret double %retval
+}
+
+define double @unordered_double_generic_cta_volatile(ptr %addr) {
+; CHECK-LABEL: unordered_double_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr %addr syncscope("block") unordered, align 8
+ ret double %retval
+}
+
+define double @unordered_double_generic_cluster_volatile(ptr %addr) {
+; CHECK-LABEL: unordered_double_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr %addr syncscope("cluster") unordered, align 8
+ ret double %retval
+}
+
+define double @unordered_double_generic_gpu_volatile(ptr %addr) {
+; CHECK-LABEL: unordered_double_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr %addr syncscope("device") unordered, align 8
+ ret double %retval
+}
+
+define i8 @unordered_i8_global_sys(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i8_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_global_sys_param_0];
+; CHECK-NEXT: ld.relaxed.sys.global.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr addrspace(1) %addr syncscope("") unordered, align 1
+ ret i8 %retval
+}
+
+define i8 @unordered_i8_global_cta(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i8_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_global_cta_param_0];
+; CHECK-NEXT: ld.relaxed.cta.global.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr addrspace(1) %addr syncscope("block") unordered, align 1
+ ret i8 %retval
+}
+
+define i8 @unordered_i8_global_cluster(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i8_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_global_cluster_param_0];
+; CHECK-NEXT: ld.relaxed.cluster.global.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr addrspace(1) %addr syncscope("cluster") unordered, align 1
+ ret i8 %retval
+}
+
+define i8 @unordered_i8_global_gpu(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i8_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_global_gpu_param_0];
+; CHECK-NEXT: ld.relaxed.gpu.global.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr addrspace(1) %addr syncscope("device") unordered, align 1
+ ret i8 %retval
+}
+
+define i8 @unordered_i8_global_sys_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i8_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.mmio.relaxed.sys.global.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr addrspace(1) %addr syncscope("") unordered, align 1
+ ret i8 %retval
+}
+
+define i8 @unordered_i8_global_cta_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i8_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.mmio.relaxed.sys.global.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr addrspace(1) %addr syncscope("block") unordered, align 1
+ ret i8 %retval
+}
+
+define i8 @unordered_i8_global_cluster_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i8_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.mmio.relaxed.sys.global.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr addrspace(1) %addr syncscope("cluster") unordered, align 1
+ ret i8 %retval
+}
+
+define i8 @unordered_i8_global_gpu_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i8_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.mmio.relaxed.sys.global.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr addrspace(1) %addr syncscope("device") unordered, align 1
+ ret i8 %retval
+}
+
+define i16 @unordered_i16_global_sys(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i16_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_global_sys_param_0];
+; CHECK-NEXT: ld.relaxed.sys.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr addrspace(1) %addr syncscope("") unordered, align 2
+ ret i16 %retval
+}
+
+define i16 @unordered_i16_global_cta(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i16_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_global_cta_param_0];
+; CHECK-NEXT: ld.relaxed.cta.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr addrspace(1) %addr syncscope("block") unordered, align 2
+ ret i16 %retval
+}
+
+define i16 @unordered_i16_global_cluster(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i16_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_global_cluster_param_0];
+; CHECK-NEXT: ld.relaxed.cluster.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr addrspace(1) %addr syncscope("cluster") unordered, align 2
+ ret i16 %retval
+}
+
+define i16 @unordered_i16_global_gpu(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i16_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_global_gpu_param_0];
+; CHECK-NEXT: ld.relaxed.gpu.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr addrspace(1) %addr syncscope("device") unordered, align 2
+ ret i16 %retval
+}
+
+define i16 @unordered_i16_global_sys_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i16_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.mmio.relaxed.sys.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr addrspace(1) %addr syncscope("") unordered, align 2
+ ret i16 %retval
+}
+
+define i16 @unordered_i16_global_cta_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i16_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.mmio.relaxed.sys.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr addrspace(1) %addr syncscope("block") unordered, align 2
+ ret i16 %retval
+}
+
+define i16 @unordered_i16_global_cluster_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i16_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.mmio.relaxed.sys.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr addrspace(1) %addr syncscope("cluster") unordered, align 2
+ ret i16 %retval
+}
+
+define i16 @unordered_i16_global_gpu_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i16_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.mmio.relaxed.sys.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr addrspace(1) %addr syncscope("device") unordered, align 2
+ ret i16 %retval
+}
+
+define i32 @unordered_i32_global_sys(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i32_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_global_sys_param_0];
+; CHECK-NEXT: ld.relaxed.sys.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr addrspace(1) %addr syncscope("") unordered, align 4
+ ret i32 %retval
+}
+
+define i32 @unordered_i32_global_cta(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i32_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_global_cta_param_0];
+; CHECK-NEXT: ld.relaxed.cta.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr addrspace(1) %addr syncscope("block") unordered, align 4
+ ret i32 %retval
+}
+
+define i32 @unordered_i32_global_cluster(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i32_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_global_cluster_param_0];
+; CHECK-NEXT: ld.relaxed.cluster.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr addrspace(1) %addr syncscope("cluster") unordered, align 4
+ ret i32 %retval
+}
+
+define i32 @unordered_i32_global_gpu(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i32_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_global_gpu_param_0];
+; CHECK-NEXT: ld.relaxed.gpu.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr addrspace(1) %addr syncscope("device") unordered, align 4
+ ret i32 %retval
+}
+
+define i32 @unordered_i32_global_sys_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i32_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.mmio.relaxed.sys.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr addrspace(1) %addr syncscope("") unordered, align 4
+ ret i32 %retval
+}
+
+define i32 @unordered_i32_global_cta_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i32_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.mmio.relaxed.sys.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr addrspace(1) %addr syncscope("block") unordered, align 4
+ ret i32 %retval
+}
+
+define i32 @unordered_i32_global_cluster_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i32_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.mmio.relaxed.sys.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr addrspace(1) %addr syncscope("cluster") unordered, align 4
+ ret i32 %retval
+}
+
+define i32 @unordered_i32_global_gpu_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i32_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.mmio.relaxed.sys.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr addrspace(1) %addr syncscope("device") unordered, align 4
+ ret i32 %retval
+}
+
+define i64 @unordered_i64_global_sys(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i64_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_global_sys_param_0];
+; CHECK-NEXT: ld.relaxed.sys.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr addrspace(1) %addr syncscope("") unordered, align 8
+ ret i64 %retval
+}
+
+define i64 @unordered_i64_global_cta(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i64_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_global_cta_param_0];
+; CHECK-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr addrspace(1) %addr syncscope("block") unordered, align 8
+ ret i64 %retval
+}
+
+define i64 @unordered_i64_global_cluster(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i64_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_global_cluster_param_0];
+; CHECK-NEXT: ld.relaxed.cluster.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr addrspace(1) %addr syncscope("cluster") unordered, align 8
+ ret i64 %retval
+}
+
+define i64 @unordered_i64_global_gpu(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i64_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_global_gpu_param_0];
+; CHECK-NEXT: ld.relaxed.gpu.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr addrspace(1) %addr syncscope("device") unordered, align 8
+ ret i64 %retval
+}
+
+define i64 @unordered_i64_global_sys_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i64_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.mmio.relaxed.sys.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr addrspace(1) %addr syncscope("") unordered, align 8
+ ret i64 %retval
+}
+
+define i64 @unordered_i64_global_cta_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i64_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.mmio.relaxed.sys.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr addrspace(1) %addr syncscope("block") unordered, align 8
+ ret i64 %retval
+}
+
+define i64 @unordered_i64_global_cluster_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i64_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.mmio.relaxed.sys.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr addrspace(1) %addr syncscope("cluster") unordered, align 8
+ ret i64 %retval
+}
+
+define i64 @unordered_i64_global_gpu_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i64_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.mmio.relaxed.sys.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr addrspace(1) %addr syncscope("device") unordered, align 8
+ ret i64 %retval
+}
+
+define i128 @unordered_i128_global_sys(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i128_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i128_global_sys_param_0];
+; CHECK-NEXT: mov.b64 %rd2, 0;
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 cmp, swap, dst;
+; CHECK-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; CHECK-NEXT: mov.b128 swap, {%rd2, %rd2};
+; CHECK-NEXT: atom.relaxed.sys.global.cas.b128 dst, [%rd1], cmp, swap;
+; CHECK-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; CHECK-NEXT: ret;
+ %retval = load atomic i128, ptr addrspace(1) %addr syncscope("") unordered, align 16
+ ret i128 %retval
+}
+
+define i128 @unordered_i128_global_cta(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i128_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i128_global_cta_param_0];
+; CHECK-NEXT: mov.b64 %rd2, 0;
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 cmp, swap, dst;
+; CHECK-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; CHECK-NEXT: mov.b128 swap, {%rd2, %rd2};
+; CHECK-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; CHECK-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; CHECK-NEXT: ret;
+ %retval = load atomic i128, ptr addrspace(1) %addr syncscope("block") unordered, align 16
+ ret i128 %retval
+}
+
+define i128 @unordered_i128_global_cluster(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i128_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i128_global_cluster_param_0];
+; CHECK-NEXT: mov.b64 %rd2, 0;
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 cmp, swap, dst;
+; CHECK-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; CHECK-NEXT: mov.b128 swap, {%rd2, %rd2};
+; CHECK-NEXT: atom.relaxed.cluster.global.cas.b128 dst, [%rd1], cmp, swap;
+; CHECK-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; CHECK-NEXT: ret;
+ %retval = load atomic i128, ptr addrspace(1) %addr syncscope("cluster") unordered, align 16
+ ret i128 %retval
+}
+
+define i128 @unordered_i128_global_gpu(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i128_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i128_global_gpu_param_0];
+; CHECK-NEXT: mov.b64 %rd2, 0;
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 cmp, swap, dst;
+; CHECK-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; CHECK-NEXT: mov.b128 swap, {%rd2, %rd2};
+; CHECK-NEXT: atom.relaxed.gpu.global.cas.b128 dst, [%rd1], cmp, swap;
+; CHECK-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; CHECK-NEXT: ret;
+ %retval = load atomic i128, ptr addrspace(1) %addr syncscope("device") unordered, align 16
+ ret i128 %retval
+}
+
+define i128 @unordered_i128_global_sys_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i128_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i128_global_sys_volatile_param_0];
+; CHECK-NEXT: mov.b64 %rd2, 0;
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 cmp, swap, dst;
+; CHECK-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; CHECK-NEXT: mov.b128 swap, {%rd2, %rd2};
+; CHECK-NEXT: atom.relaxed.sys.global.cas.b128 dst, [%rd1], cmp, swap;
+; CHECK-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i128, ptr addrspace(1) %addr syncscope("") unordered, align 16
+ ret i128 %retval
+}
+
+define i128 @unordered_i128_global_cta_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i128_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i128_global_cta_volatile_param_0];
+; CHECK-NEXT: mov.b64 %rd2, 0;
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 cmp, swap, dst;
+; CHECK-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; CHECK-NEXT: mov.b128 swap, {%rd2, %rd2};
+; CHECK-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; CHECK-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i128, ptr addrspace(1) %addr syncscope("block") unordered, align 16
+ ret i128 %retval
+}
+
+define i128 @unordered_i128_global_cluster_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i128_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i128_global_cluster_volatile_param_0];
+; CHECK-NEXT: mov.b64 %rd2, 0;
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 cmp, swap, dst;
+; CHECK-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; CHECK-NEXT: mov.b128 swap, {%rd2, %rd2};
+; CHECK-NEXT: atom.relaxed.cluster.global.cas.b128 dst, [%rd1], cmp, swap;
+; CHECK-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i128, ptr addrspace(1) %addr syncscope("cluster") unordered, align 16
+ ret i128 %retval
+}
+
+define i128 @unordered_i128_global_gpu_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i128_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i128_global_gpu_volatile_param_0];
+; CHECK-NEXT: mov.b64 %rd2, 0;
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 cmp, swap, dst;
+; CHECK-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; CHECK-NEXT: mov.b128 swap, {%rd2, %rd2};
+; CHECK-NEXT: atom.relaxed.gpu.global.cas.b128 dst, [%rd1], cmp, swap;
+; CHECK-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i128, ptr addrspace(1) %addr syncscope("device") unordered, align 16
+ ret i128 %retval
+}
+
+define half @unordered_half_global_sys(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_half_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_global_sys_param_0];
+; CHECK-NEXT: ld.relaxed.sys.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr addrspace(1) %addr syncscope("") unordered, align 2
+ ret half %retval
+}
+
+define half @unordered_half_global_cta(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_half_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_global_cta_param_0];
+; CHECK-NEXT: ld.relaxed.cta.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr addrspace(1) %addr syncscope("block") unordered, align 2
+ ret half %retval
+}
+
+define half @unordered_half_global_cluster(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_half_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_global_cluster_param_0];
+; CHECK-NEXT: ld.relaxed.cluster.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr addrspace(1) %addr syncscope("cluster") unordered, align 2
+ ret half %retval
+}
+
+define half @unordered_half_global_gpu(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_half_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_global_gpu_param_0];
+; CHECK-NEXT: ld.relaxed.gpu.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr addrspace(1) %addr syncscope("device") unordered, align 2
+ ret half %retval
+}
+
+define half @unordered_half_global_sys_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_half_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.mmio.relaxed.sys.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr addrspace(1) %addr syncscope("") unordered, align 2
+ ret half %retval
+}
+
+define half @unordered_half_global_cta_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_half_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.mmio.relaxed.sys.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr addrspace(1) %addr syncscope("block") unordered, align 2
+ ret half %retval
+}
+
+define half @unordered_half_global_cluster_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_half_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.mmio.relaxed.sys.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr addrspace(1) %addr syncscope("cluster") unordered, align 2
+ ret half %retval
+}
+
+define half @unordered_half_global_gpu_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_half_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.mmio.relaxed.sys.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr addrspace(1) %addr syncscope("device") unordered, align 2
+ ret half %retval
+}
+
+define bfloat @unordered_bfloat_global_sys(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_bfloat_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_global_sys_param_0];
+; CHECK-NEXT: ld.relaxed.sys.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr addrspace(1) %addr syncscope("") unordered, align 2
+ ret bfloat %retval
+}
+
+define bfloat @unordered_bfloat_global_cta(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_bfloat_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_global_cta_param_0];
+; CHECK-NEXT: ld.relaxed.cta.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr addrspace(1) %addr syncscope("block") unordered, align 2
+ ret bfloat %retval
+}
+
+define bfloat @unordered_bfloat_global_cluster(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_bfloat_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_global_cluster_param_0];
+; CHECK-NEXT: ld.relaxed.cluster.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr addrspace(1) %addr syncscope("cluster") unordered, align 2
+ ret bfloat %retval
+}
+
+define bfloat @unordered_bfloat_global_gpu(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_bfloat_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_global_gpu_param_0];
+; CHECK-NEXT: ld.relaxed.gpu.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr addrspace(1) %addr syncscope("device") unordered, align 2
+ ret bfloat %retval
+}
+
+define bfloat @unordered_bfloat_global_sys_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_bfloat_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.mmio.relaxed.sys.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr addrspace(1) %addr syncscope("") unordered, align 2
+ ret bfloat %retval
+}
+
+define bfloat @unordered_bfloat_global_cta_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_bfloat_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.mmio.relaxed.sys.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr addrspace(1) %addr syncscope("block") unordered, align 2
+ ret bfloat %retval
+}
+
+define bfloat @unordered_bfloat_global_cluster_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_bfloat_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.mmio.relaxed.sys.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr addrspace(1) %addr syncscope("cluster") unordered, align 2
+ ret bfloat %retval
+}
+
+define bfloat @unordered_bfloat_global_gpu_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_bfloat_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.mmio.relaxed.sys.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr addrspace(1) %addr syncscope("device") unordered, align 2
+ ret bfloat %retval
+}
+
+define float @unordered_float_global_sys(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_float_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_global_sys_param_0];
+; CHECK-NEXT: ld.relaxed.sys.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr addrspace(1) %addr syncscope("") unordered, align 4
+ ret float %retval
+}
+
+define float @unordered_float_global_cta(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_float_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_global_cta_param_0];
+; CHECK-NEXT: ld.relaxed.cta.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr addrspace(1) %addr syncscope("block") unordered, align 4
+ ret float %retval
+}
+
+define float @unordered_float_global_cluster(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_float_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_global_cluster_param_0];
+; CHECK-NEXT: ld.relaxed.cluster.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr addrspace(1) %addr syncscope("cluster") unordered, align 4
+ ret float %retval
+}
+
+define float @unordered_float_global_gpu(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_float_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_global_gpu_param_0];
+; CHECK-NEXT: ld.relaxed.gpu.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr addrspace(1) %addr syncscope("device") unordered, align 4
+ ret float %retval
+}
+
+define float @unordered_float_global_sys_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_float_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.mmio.relaxed.sys.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr addrspace(1) %addr syncscope("") unordered, align 4
+ ret float %retval
+}
+
+define float @unordered_float_global_cta_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_float_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.mmio.relaxed.sys.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr addrspace(1) %addr syncscope("block") unordered, align 4
+ ret float %retval
+}
+
+define float @unordered_float_global_cluster_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_float_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.mmio.relaxed.sys.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr addrspace(1) %addr syncscope("cluster") unordered, align 4
+ ret float %retval
+}
+
+define float @unordered_float_global_gpu_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_float_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.mmio.relaxed.sys.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr addrspace(1) %addr syncscope("device") unordered, align 4
+ ret float %retval
+}
+
+define double @unordered_double_global_sys(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_double_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_global_sys_param_0];
+; CHECK-NEXT: ld.relaxed.sys.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr addrspace(1) %addr syncscope("") unordered, align 8
+ ret double %retval
+}
+
+define double @unordered_double_global_cta(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_double_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_global_cta_param_0];
+; CHECK-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr addrspace(1) %addr syncscope("block") unordered, align 8
+ ret double %retval
+}
+
+define double @unordered_double_global_cluster(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_double_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_global_cluster_param_0];
+; CHECK-NEXT: ld.relaxed.cluster.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr addrspace(1) %addr syncscope("cluster") unordered, align 8
+ ret double %retval
+}
+
+define double @unordered_double_global_gpu(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_double_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_global_gpu_param_0];
+; CHECK-NEXT: ld.relaxed.gpu.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr addrspace(1) %addr syncscope("device") unordered, align 8
+ ret double %retval
+}
+
+define double @unordered_double_global_sys_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_double_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.mmio.relaxed.sys.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr addrspace(1) %addr syncscope("") unordered, align 8
+ ret double %retval
+}
+
+define double @unordered_double_global_cta_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_double_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.mmio.relaxed.sys.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr addrspace(1) %addr syncscope("block") unordered, align 8
+ ret double %retval
+}
+
+define double @unordered_double_global_cluster_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_double_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.mmio.relaxed.sys.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr addrspace(1) %addr syncscope("cluster") unordered, align 8
+ ret double %retval
+}
+
+define double @unordered_double_global_gpu_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_double_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.mmio.relaxed.sys.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr addrspace(1) %addr syncscope("device") unordered, align 8
+ ret double %retval
+}
+
+define i8 @unordered_i8_shared_sys(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i8_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_shared_sys_param_0];
+; CHECK-NEXT: ld.relaxed.sys.shared.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr addrspace(3) %addr syncscope("") unordered, align 1
+ ret i8 %retval
+}
+
+define i8 @unordered_i8_shared_cta(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i8_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_shared_cta_param_0];
+; CHECK-NEXT: ld.relaxed.cta.shared.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr addrspace(3) %addr syncscope("block") unordered, align 1
+ ret i8 %retval
+}
+
+define i8 @unordered_i8_shared_cluster(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i8_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_shared_cluster_param_0];
+; CHECK-NEXT: ld.relaxed.cluster.shared.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr addrspace(3) %addr syncscope("cluster") unordered, align 1
+ ret i8 %retval
+}
+
+define i8 @unordered_i8_shared_gpu(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i8_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_shared_gpu_param_0];
+; CHECK-NEXT: ld.relaxed.gpu.shared.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr addrspace(3) %addr syncscope("device") unordered, align 1
+ ret i8 %retval
+}
+
+define i8 @unordered_i8_shared_sys_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i8_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr addrspace(3) %addr syncscope("") unordered, align 1
+ ret i8 %retval
+}
+
+define i8 @unordered_i8_shared_cta_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i8_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr addrspace(3) %addr syncscope("block") unordered, align 1
+ ret i8 %retval
+}
+
+define i8 @unordered_i8_shared_cluster_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i8_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr addrspace(3) %addr syncscope("cluster") unordered, align 1
+ ret i8 %retval
+}
+
+define i8 @unordered_i8_shared_gpu_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i8_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr addrspace(3) %addr syncscope("device") unordered, align 1
+ ret i8 %retval
+}
+
+define i16 @unordered_i16_shared_sys(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i16_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_shared_sys_param_0];
+; CHECK-NEXT: ld.relaxed.sys.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr addrspace(3) %addr syncscope("") unordered, align 2
+ ret i16 %retval
+}
+
+define i16 @unordered_i16_shared_cta(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i16_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_shared_cta_param_0];
+; CHECK-NEXT: ld.relaxed.cta.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr addrspace(3) %addr syncscope("block") unordered, align 2
+ ret i16 %retval
+}
+
+define i16 @unordered_i16_shared_cluster(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i16_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_shared_cluster_param_0];
+; CHECK-NEXT: ld.relaxed.cluster.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr addrspace(3) %addr syncscope("cluster") unordered, align 2
+ ret i16 %retval
+}
+
+define i16 @unordered_i16_shared_gpu(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i16_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_shared_gpu_param_0];
+; CHECK-NEXT: ld.relaxed.gpu.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr addrspace(3) %addr syncscope("device") unordered, align 2
+ ret i16 %retval
+}
+
+define i16 @unordered_i16_shared_sys_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i16_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr addrspace(3) %addr syncscope("") unordered, align 2
+ ret i16 %retval
+}
+
+define i16 @unordered_i16_shared_cta_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i16_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr addrspace(3) %addr syncscope("block") unordered, align 2
+ ret i16 %retval
+}
+
+define i16 @unordered_i16_shared_cluster_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i16_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr addrspace(3) %addr syncscope("cluster") unordered, align 2
+ ret i16 %retval
+}
+
+define i16 @unordered_i16_shared_gpu_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i16_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr addrspace(3) %addr syncscope("device") unordered, align 2
+ ret i16 %retval
+}
+
+define i32 @unordered_i32_shared_sys(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i32_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_shared_sys_param_0];
+; CHECK-NEXT: ld.relaxed.sys.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr addrspace(3) %addr syncscope("") unordered, align 4
+ ret i32 %retval
+}
+
+define i32 @unordered_i32_shared_cta(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i32_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_shared_cta_param_0];
+; CHECK-NEXT: ld.relaxed.cta.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr addrspace(3) %addr syncscope("block") unordered, align 4
+ ret i32 %retval
+}
+
+define i32 @unordered_i32_shared_cluster(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i32_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_shared_cluster_param_0];
+; CHECK-NEXT: ld.relaxed.cluster.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr addrspace(3) %addr syncscope("cluster") unordered, align 4
+ ret i32 %retval
+}
+
+define i32 @unordered_i32_shared_gpu(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i32_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_shared_gpu_param_0];
+; CHECK-NEXT: ld.relaxed.gpu.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr addrspace(3) %addr syncscope("device") unordered, align 4
+ ret i32 %retval
+}
+
+define i32 @unordered_i32_shared_sys_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i32_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr addrspace(3) %addr syncscope("") unordered, align 4
+ ret i32 %retval
+}
+
+define i32 @unordered_i32_shared_cta_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i32_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr addrspace(3) %addr syncscope("block") unordered, align 4
+ ret i32 %retval
+}
+
+define i32 @unordered_i32_shared_cluster_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i32_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr addrspace(3) %addr syncscope("cluster") unordered, align 4
+ ret i32 %retval
+}
+
+define i32 @unordered_i32_shared_gpu_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i32_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr addrspace(3) %addr syncscope("device") unordered, align 4
+ ret i32 %retval
+}
+
+define i64 @unordered_i64_shared_sys(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i64_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_shared_sys_param_0];
+; CHECK-NEXT: ld.relaxed.sys.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr addrspace(3) %addr syncscope("") unordered, align 8
+ ret i64 %retval
+}
+
+define i64 @unordered_i64_shared_cta(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i64_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_shared_cta_param_0];
+; CHECK-NEXT: ld.relaxed.cta.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr addrspace(3) %addr syncscope("block") unordered, align 8
+ ret i64 %retval
+}
+
+define i64 @unordered_i64_shared_cluster(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i64_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_shared_cluster_param_0];
+; CHECK-NEXT: ld.relaxed.cluster.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr addrspace(3) %addr syncscope("cluster") unordered, align 8
+ ret i64 %retval
+}
+
+define i64 @unordered_i64_shared_gpu(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i64_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_shared_gpu_param_0];
+; CHECK-NEXT: ld.relaxed.gpu.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr addrspace(3) %addr syncscope("device") unordered, align 8
+ ret i64 %retval
+}
+
+define i64 @unordered_i64_shared_sys_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i64_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr addrspace(3) %addr syncscope("") unordered, align 8
+ ret i64 %retval
+}
+
+define i64 @unordered_i64_shared_cta_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i64_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr addrspace(3) %addr syncscope("block") unordered, align 8
+ ret i64 %retval
+}
+
+define i64 @unordered_i64_shared_cluster_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i64_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr addrspace(3) %addr syncscope("cluster") unordered, align 8
+ ret i64 %retval
+}
+
+define i64 @unordered_i64_shared_gpu_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i64_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr addrspace(3) %addr syncscope("device") unordered, align 8
+ ret i64 %retval
+}
+
+define i128 @unordered_i128_shared_sys(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i128_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i128_shared_sys_param_0];
+; CHECK-NEXT: mov.b64 %rd2, 0;
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 cmp, swap, dst;
+; CHECK-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; CHECK-NEXT: mov.b128 swap, {%rd2, %rd2};
+; CHECK-NEXT: atom.relaxed.sys.shared.cas.b128 dst, [%rd1], cmp, swap;
+; CHECK-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; CHECK-NEXT: ret;
+ %retval = load atomic i128, ptr addrspace(3) %addr syncscope("") unordered, align 16
+ ret i128 %retval
+}
+
+define i128 @unordered_i128_shared_cta(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i128_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i128_shared_cta_param_0];
+; CHECK-NEXT: mov.b64 %rd2, 0;
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 cmp, swap, dst;
+; CHECK-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; CHECK-NEXT: mov.b128 swap, {%rd2, %rd2};
+; CHECK-NEXT: atom.relaxed.cta.shared.cas.b128 dst, [%rd1], cmp, swap;
+; CHECK-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; CHECK-NEXT: ret;
+ %retval = load atomic i128, ptr addrspace(3) %addr syncscope("block") unordered, align 16
+ ret i128 %retval
+}
+
+define i128 @unordered_i128_shared_cluster(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i128_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i128_shared_cluster_param_0];
+; CHECK-NEXT: mov.b64 %rd2, 0;
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 cmp, swap, dst;
+; CHECK-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; CHECK-NEXT: mov.b128 swap, {%rd2, %rd2};
+; CHECK-NEXT: atom.relaxed.cluster.shared.cas.b128 dst, [%rd1], cmp, swap;
+; CHECK-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; CHECK-NEXT: ret;
+ %retval = load atomic i128, ptr addrspace(3) %addr syncscope("cluster") unordered, align 16
+ ret i128 %retval
+}
+
+define i128 @unordered_i128_shared_gpu(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i128_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i128_shared_gpu_param_0];
+; CHECK-NEXT: mov.b64 %rd2, 0;
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 cmp, swap, dst;
+; CHECK-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; CHECK-NEXT: mov.b128 swap, {%rd2, %rd2};
+; CHECK-NEXT: atom.relaxed.gpu.shared.cas.b128 dst, [%rd1], cmp, swap;
+; CHECK-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; CHECK-NEXT: ret;
+ %retval = load atomic i128, ptr addrspace(3) %addr syncscope("device") unordered, align 16
+ ret i128 %retval
+}
+
+define i128 @unordered_i128_shared_sys_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i128_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i128_shared_sys_volatile_param_0];
+; CHECK-NEXT: mov.b64 %rd2, 0;
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 cmp, swap, dst;
+; CHECK-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; CHECK-NEXT: mov.b128 swap, {%rd2, %rd2};
+; CHECK-NEXT: atom.relaxed.sys.shared.cas.b128 dst, [%rd1], cmp, swap;
+; CHECK-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i128, ptr addrspace(3) %addr syncscope("") unordered, align 16
+ ret i128 %retval
+}
+
+define i128 @unordered_i128_shared_cta_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i128_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i128_shared_cta_volatile_param_0];
+; CHECK-NEXT: mov.b64 %rd2, 0;
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 cmp, swap, dst;
+; CHECK-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; CHECK-NEXT: mov.b128 swap, {%rd2, %rd2};
+; CHECK-NEXT: atom.relaxed.cta.shared.cas.b128 dst, [%rd1], cmp, swap;
+; CHECK-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i128, ptr addrspace(3) %addr syncscope("block") unordered, align 16
+ ret i128 %retval
+}
+
+define i128 @unordered_i128_shared_cluster_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i128_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i128_shared_cluster_volatile_param_0];
+; CHECK-NEXT: mov.b64 %rd2, 0;
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 cmp, swap, dst;
+; CHECK-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; CHECK-NEXT: mov.b128 swap, {%rd2, %rd2};
+; CHECK-NEXT: atom.relaxed.cluster.shared.cas.b128 dst, [%rd1], cmp, swap;
+; CHECK-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i128, ptr addrspace(3) %addr syncscope("cluster") unordered, align 16
+ ret i128 %retval
+}
+
+define i128 @unordered_i128_shared_gpu_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i128_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i128_shared_gpu_volatile_param_0];
+; CHECK-NEXT: mov.b64 %rd2, 0;
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 cmp, swap, dst;
+; CHECK-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; CHECK-NEXT: mov.b128 swap, {%rd2, %rd2};
+; CHECK-NEXT: atom.relaxed.gpu.shared.cas.b128 dst, [%rd1], cmp, swap;
+; CHECK-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i128, ptr addrspace(3) %addr syncscope("device") unordered, align 16
+ ret i128 %retval
+}
+
+define half @unordered_half_shared_sys(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_half_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_shared_sys_param_0];
+; CHECK-NEXT: ld.relaxed.sys.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr addrspace(3) %addr syncscope("") unordered, align 2
+ ret half %retval
+}
+
+define half @unordered_half_shared_cta(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_half_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_shared_cta_param_0];
+; CHECK-NEXT: ld.relaxed.cta.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr addrspace(3) %addr syncscope("block") unordered, align 2
+ ret half %retval
+}
+
+define half @unordered_half_shared_cluster(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_half_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_shared_cluster_param_0];
+; CHECK-NEXT: ld.relaxed.cluster.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr addrspace(3) %addr syncscope("cluster") unordered, align 2
+ ret half %retval
+}
+
+define half @unordered_half_shared_gpu(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_half_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_shared_gpu_param_0];
+; CHECK-NEXT: ld.relaxed.gpu.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr addrspace(3) %addr syncscope("device") unordered, align 2
+ ret half %retval
+}
+
+define half @unordered_half_shared_sys_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_half_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr addrspace(3) %addr syncscope("") unordered, align 2
+ ret half %retval
+}
+
+define half @unordered_half_shared_cta_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_half_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr addrspace(3) %addr syncscope("block") unordered, align 2
+ ret half %retval
+}
+
+define half @unordered_half_shared_cluster_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_half_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr addrspace(3) %addr syncscope("cluster") unordered, align 2
+ ret half %retval
+}
+
+define half @unordered_half_shared_gpu_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_half_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr addrspace(3) %addr syncscope("device") unordered, align 2
+ ret half %retval
+}
+
+define bfloat @unordered_bfloat_shared_sys(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_bfloat_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_shared_sys_param_0];
+; CHECK-NEXT: ld.relaxed.sys.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr addrspace(3) %addr syncscope("") unordered, align 2
+ ret bfloat %retval
+}
+
+define bfloat @unordered_bfloat_shared_cta(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_bfloat_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_shared_cta_param_0];
+; CHECK-NEXT: ld.relaxed.cta.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr addrspace(3) %addr syncscope("block") unordered, align 2
+ ret bfloat %retval
+}
+
+define bfloat @unordered_bfloat_shared_cluster(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_bfloat_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_shared_cluster_param_0];
+; CHECK-NEXT: ld.relaxed.cluster.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr addrspace(3) %addr syncscope("cluster") unordered, align 2
+ ret bfloat %retval
+}
+
+define bfloat @unordered_bfloat_shared_gpu(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_bfloat_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_shared_gpu_param_0];
+; CHECK-NEXT: ld.relaxed.gpu.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr addrspace(3) %addr syncscope("device") unordered, align 2
+ ret bfloat %retval
+}
+
+define bfloat @unordered_bfloat_shared_sys_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_bfloat_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr addrspace(3) %addr syncscope("") unordered, align 2
+ ret bfloat %retval
+}
+
+define bfloat @unordered_bfloat_shared_cta_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_bfloat_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr addrspace(3) %addr syncscope("block") unordered, align 2
+ ret bfloat %retval
+}
+
+define bfloat @unordered_bfloat_shared_cluster_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_bfloat_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr addrspace(3) %addr syncscope("cluster") unordered, align 2
+ ret bfloat %retval
+}
+
+define bfloat @unordered_bfloat_shared_gpu_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_bfloat_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr addrspace(3) %addr syncscope("device") unordered, align 2
+ ret bfloat %retval
+}
+
+define float @unordered_float_shared_sys(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_float_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_shared_sys_param_0];
+; CHECK-NEXT: ld.relaxed.sys.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr addrspace(3) %addr syncscope("") unordered, align 4
+ ret float %retval
+}
+
+define float @unordered_float_shared_cta(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_float_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_shared_cta_param_0];
+; CHECK-NEXT: ld.relaxed.cta.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr addrspace(3) %addr syncscope("block") unordered, align 4
+ ret float %retval
+}
+
+define float @unordered_float_shared_cluster(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_float_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_shared_cluster_param_0];
+; CHECK-NEXT: ld.relaxed.cluster.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr addrspace(3) %addr syncscope("cluster") unordered, align 4
+ ret float %retval
+}
+
+define float @unordered_float_shared_gpu(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_float_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_shared_gpu_param_0];
+; CHECK-NEXT: ld.relaxed.gpu.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr addrspace(3) %addr syncscope("device") unordered, align 4
+ ret float %retval
+}
+
+define float @unordered_float_shared_sys_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_float_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr addrspace(3) %addr syncscope("") unordered, align 4
+ ret float %retval
+}
+
+define float @unordered_float_shared_cta_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_float_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr addrspace(3) %addr syncscope("block") unordered, align 4
+ ret float %retval
+}
+
+define float @unordered_float_shared_cluster_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_float_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr addrspace(3) %addr syncscope("cluster") unordered, align 4
+ ret float %retval
+}
+
+define float @unordered_float_shared_gpu_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_float_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr addrspace(3) %addr syncscope("device") unordered, align 4
+ ret float %retval
+}
+
+define double @unordered_double_shared_sys(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_double_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_shared_sys_param_0];
+; CHECK-NEXT: ld.relaxed.sys.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr addrspace(3) %addr syncscope("") unordered, align 8
+ ret double %retval
+}
+
+define double @unordered_double_shared_cta(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_double_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_shared_cta_param_0];
+; CHECK-NEXT: ld.relaxed.cta.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr addrspace(3) %addr syncscope("block") unordered, align 8
+ ret double %retval
+}
+
+define double @unordered_double_shared_cluster(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_double_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_shared_cluster_param_0];
+; CHECK-NEXT: ld.relaxed.cluster.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr addrspace(3) %addr syncscope("cluster") unordered, align 8
+ ret double %retval
+}
+
+define double @unordered_double_shared_gpu(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_double_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_shared_gpu_param_0];
+; CHECK-NEXT: ld.relaxed.gpu.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr addrspace(3) %addr syncscope("device") unordered, align 8
+ ret double %retval
+}
+
+define double @unordered_double_shared_sys_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_double_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr addrspace(3) %addr syncscope("") unordered, align 8
+ ret double %retval
+}
+
+define double @unordered_double_shared_cta_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_double_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr addrspace(3) %addr syncscope("block") unordered, align 8
+ ret double %retval
+}
+
+define double @unordered_double_shared_cluster_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_double_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr addrspace(3) %addr syncscope("cluster") unordered, align 8
+ ret double %retval
+}
+
+define double @unordered_double_shared_gpu_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_double_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr addrspace(3) %addr syncscope("device") unordered, align 8
+ ret double %retval
+}
+
+define i8 @unordered_i8_local_sys() {
+; CHECK-LABEL: unordered_i8_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot324[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot324;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b8 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i8, ptr addrspace(5) %slot syncscope("") unordered, align 1
+ ret i8 %retval
+}
+
+define i8 @unordered_i8_local_cta() {
+; CHECK-LABEL: unordered_i8_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot325[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot325;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b8 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i8, ptr addrspace(5) %slot syncscope("block") unordered, align 1
+ ret i8 %retval
+}
+
+define i8 @unordered_i8_local_cluster() {
+; CHECK-LABEL: unordered_i8_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot326[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot326;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b8 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i8, ptr addrspace(5) %slot syncscope("cluster") unordered, align 1
+ ret i8 %retval
+}
+
+define i8 @unordered_i8_local_gpu() {
+; CHECK-LABEL: unordered_i8_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot327[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot327;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b8 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i8, ptr addrspace(5) %slot syncscope("device") unordered, align 1
+ ret i8 %retval
+}
+
+define i8 @unordered_i8_local_sys_volatile() {
+; CHECK-LABEL: unordered_i8_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot328[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot328;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b8 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i8, ptr addrspace(5) %slot syncscope("") unordered, align 1
+ ret i8 %retval
+}
+
+define i8 @unordered_i8_local_cta_volatile() {
+; CHECK-LABEL: unordered_i8_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot329[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot329;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b8 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i8, ptr addrspace(5) %slot syncscope("block") unordered, align 1
+ ret i8 %retval
+}
+
+define i8 @unordered_i8_local_cluster_volatile() {
+; CHECK-LABEL: unordered_i8_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot330[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot330;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b8 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i8, ptr addrspace(5) %slot syncscope("cluster") unordered, align 1
+ ret i8 %retval
+}
+
+define i8 @unordered_i8_local_gpu_volatile() {
+; CHECK-LABEL: unordered_i8_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot331[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot331;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b8 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i8, ptr addrspace(5) %slot syncscope("device") unordered, align 1
+ ret i8 %retval
+}
+
+define i16 @unordered_i16_local_sys() {
+; CHECK-LABEL: unordered_i16_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot332[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot332;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i16, ptr addrspace(5) %slot syncscope("") unordered, align 2
+ ret i16 %retval
+}
+
+define i16 @unordered_i16_local_cta() {
+; CHECK-LABEL: unordered_i16_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot333[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot333;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i16, ptr addrspace(5) %slot syncscope("block") unordered, align 2
+ ret i16 %retval
+}
+
+define i16 @unordered_i16_local_cluster() {
+; CHECK-LABEL: unordered_i16_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot334[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot334;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i16, ptr addrspace(5) %slot syncscope("cluster") unordered, align 2
+ ret i16 %retval
+}
+
+define i16 @unordered_i16_local_gpu() {
+; CHECK-LABEL: unordered_i16_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot335[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot335;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i16, ptr addrspace(5) %slot syncscope("device") unordered, align 2
+ ret i16 %retval
+}
+
+define i16 @unordered_i16_local_sys_volatile() {
+; CHECK-LABEL: unordered_i16_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot336[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot336;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i16, ptr addrspace(5) %slot syncscope("") unordered, align 2
+ ret i16 %retval
+}
+
+define i16 @unordered_i16_local_cta_volatile() {
+; CHECK-LABEL: unordered_i16_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot337[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot337;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i16, ptr addrspace(5) %slot syncscope("block") unordered, align 2
+ ret i16 %retval
+}
+
+define i16 @unordered_i16_local_cluster_volatile() {
+; CHECK-LABEL: unordered_i16_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot338[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot338;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i16, ptr addrspace(5) %slot syncscope("cluster") unordered, align 2
+ ret i16 %retval
+}
+
+define i16 @unordered_i16_local_gpu_volatile() {
+; CHECK-LABEL: unordered_i16_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot339[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot339;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i16, ptr addrspace(5) %slot syncscope("device") unordered, align 2
+ ret i16 %retval
+}
+
+define i32 @unordered_i32_local_sys() {
+; CHECK-LABEL: unordered_i32_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot340[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot340;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i32, ptr addrspace(5) %slot syncscope("") unordered, align 4
+ ret i32 %retval
+}
+
+define i32 @unordered_i32_local_cta() {
+; CHECK-LABEL: unordered_i32_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot341[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot341;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i32, ptr addrspace(5) %slot syncscope("block") unordered, align 4
+ ret i32 %retval
+}
+
+define i32 @unordered_i32_local_cluster() {
+; CHECK-LABEL: unordered_i32_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot342[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot342;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i32, ptr addrspace(5) %slot syncscope("cluster") unordered, align 4
+ ret i32 %retval
+}
+
+define i32 @unordered_i32_local_gpu() {
+; CHECK-LABEL: unordered_i32_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot343[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot343;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i32, ptr addrspace(5) %slot syncscope("device") unordered, align 4
+ ret i32 %retval
+}
+
+define i32 @unordered_i32_local_sys_volatile() {
+; CHECK-LABEL: unordered_i32_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot344[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot344;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i32, ptr addrspace(5) %slot syncscope("") unordered, align 4
+ ret i32 %retval
+}
+
+define i32 @unordered_i32_local_cta_volatile() {
+; CHECK-LABEL: unordered_i32_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot345[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot345;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i32, ptr addrspace(5) %slot syncscope("block") unordered, align 4
+ ret i32 %retval
+}
+
+define i32 @unordered_i32_local_cluster_volatile() {
+; CHECK-LABEL: unordered_i32_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot346[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot346;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i32, ptr addrspace(5) %slot syncscope("cluster") unordered, align 4
+ ret i32 %retval
+}
+
+define i32 @unordered_i32_local_gpu_volatile() {
+; CHECK-LABEL: unordered_i32_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot347[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot347;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i32, ptr addrspace(5) %slot syncscope("device") unordered, align 4
+ ret i32 %retval
+}
+
+define i64 @unordered_i64_local_sys() {
+; CHECK-LABEL: unordered_i64_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot348[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot348;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i64, ptr addrspace(5) %slot syncscope("") unordered, align 8
+ ret i64 %retval
+}
+
+define i64 @unordered_i64_local_cta() {
+; CHECK-LABEL: unordered_i64_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot349[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot349;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i64, ptr addrspace(5) %slot syncscope("block") unordered, align 8
+ ret i64 %retval
+}
+
+define i64 @unordered_i64_local_cluster() {
+; CHECK-LABEL: unordered_i64_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot350[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot350;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i64, ptr addrspace(5) %slot syncscope("cluster") unordered, align 8
+ ret i64 %retval
+}
+
+define i64 @unordered_i64_local_gpu() {
+; CHECK-LABEL: unordered_i64_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot351[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot351;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i64, ptr addrspace(5) %slot syncscope("device") unordered, align 8
+ ret i64 %retval
+}
+
+define i64 @unordered_i64_local_sys_volatile() {
+; CHECK-LABEL: unordered_i64_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot352[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot352;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i64, ptr addrspace(5) %slot syncscope("") unordered, align 8
+ ret i64 %retval
+}
+
+define i64 @unordered_i64_local_cta_volatile() {
+; CHECK-LABEL: unordered_i64_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot353[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot353;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i64, ptr addrspace(5) %slot syncscope("block") unordered, align 8
+ ret i64 %retval
+}
+
+define i64 @unordered_i64_local_cluster_volatile() {
+; CHECK-LABEL: unordered_i64_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot354[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot354;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i64, ptr addrspace(5) %slot syncscope("cluster") unordered, align 8
+ ret i64 %retval
+}
+
+define i64 @unordered_i64_local_gpu_volatile() {
+; CHECK-LABEL: unordered_i64_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot355[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot355;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i64, ptr addrspace(5) %slot syncscope("device") unordered, align 8
+ ret i64 %retval
+}
+
+define i128 @unordered_i128_local_sys() {
+; CHECK-LABEL: unordered_i128_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 16 .b8 __local_depot356[16];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot356;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.v2.b64 {%rd2, %rd3}, [%SP];
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd2, %rd3};
+; CHECK-NEXT: ret;
+ %slot = alloca i128, align 16, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i128, ptr addrspace(5) %slot syncscope("") unordered, align 16
+ ret i128 %retval
+}
+
+define i128 @unordered_i128_local_cta() {
+; CHECK-LABEL: unordered_i128_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 16 .b8 __local_depot357[16];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot357;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.v2.b64 {%rd2, %rd3}, [%SP];
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd2, %rd3};
+; CHECK-NEXT: ret;
+ %slot = alloca i128, align 16, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i128, ptr addrspace(5) %slot syncscope("block") unordered, align 16
+ ret i128 %retval
+}
+
+define i128 @unordered_i128_local_cluster() {
+; CHECK-LABEL: unordered_i128_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 16 .b8 __local_depot358[16];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot358;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.v2.b64 {%rd2, %rd3}, [%SP];
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd2, %rd3};
+; CHECK-NEXT: ret;
+ %slot = alloca i128, align 16, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i128, ptr addrspace(5) %slot syncscope("cluster") unordered, align 16
+ ret i128 %retval
+}
+
+define i128 @unordered_i128_local_gpu() {
+; CHECK-LABEL: unordered_i128_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 16 .b8 __local_depot359[16];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot359;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.v2.b64 {%rd2, %rd3}, [%SP];
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd2, %rd3};
+; CHECK-NEXT: ret;
+ %slot = alloca i128, align 16, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i128, ptr addrspace(5) %slot syncscope("device") unordered, align 16
+ ret i128 %retval
+}
+
+define i128 @unordered_i128_local_sys_volatile() {
+; CHECK-LABEL: unordered_i128_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 16 .b8 __local_depot360[16];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot360;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.v2.b64 {%rd2, %rd3}, [%SP];
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd2, %rd3};
+; CHECK-NEXT: ret;
+ %slot = alloca i128, align 16, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i128, ptr addrspace(5) %slot syncscope("") unordered, align 16
+ ret i128 %retval
+}
+
+define i128 @unordered_i128_local_cta_volatile() {
+; CHECK-LABEL: unordered_i128_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 16 .b8 __local_depot361[16];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot361;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.v2.b64 {%rd2, %rd3}, [%SP];
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd2, %rd3};
+; CHECK-NEXT: ret;
+ %slot = alloca i128, align 16, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i128, ptr addrspace(5) %slot syncscope("block") unordered, align 16
+ ret i128 %retval
+}
+
+define i128 @unordered_i128_local_cluster_volatile() {
+; CHECK-LABEL: unordered_i128_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 16 .b8 __local_depot362[16];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot362;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.v2.b64 {%rd2, %rd3}, [%SP];
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd2, %rd3};
+; CHECK-NEXT: ret;
+ %slot = alloca i128, align 16, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i128, ptr addrspace(5) %slot syncscope("cluster") unordered, align 16
+ ret i128 %retval
+}
+
+define i128 @unordered_i128_local_gpu_volatile() {
+; CHECK-LABEL: unordered_i128_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 16 .b8 __local_depot363[16];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot363;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.v2.b64 {%rd2, %rd3}, [%SP];
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd2, %rd3};
+; CHECK-NEXT: ret;
+ %slot = alloca i128, align 16, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i128, ptr addrspace(5) %slot syncscope("device") unordered, align 16
+ ret i128 %retval
+}
+
+define half @unordered_half_local_sys() {
+; CHECK-LABEL: unordered_half_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot364[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot364;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic half, ptr addrspace(5) %slot syncscope("") unordered, align 2
+ ret half %retval
+}
+
+define half @unordered_half_local_cta() {
+; CHECK-LABEL: unordered_half_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot365[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot365;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic half, ptr addrspace(5) %slot syncscope("block") unordered, align 2
+ ret half %retval
+}
+
+define half @unordered_half_local_cluster() {
+; CHECK-LABEL: unordered_half_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot366[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot366;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic half, ptr addrspace(5) %slot syncscope("cluster") unordered, align 2
+ ret half %retval
+}
+
+define half @unordered_half_local_gpu() {
+; CHECK-LABEL: unordered_half_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot367[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot367;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic half, ptr addrspace(5) %slot syncscope("device") unordered, align 2
+ ret half %retval
+}
+
+define half @unordered_half_local_sys_volatile() {
+; CHECK-LABEL: unordered_half_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot368[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot368;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile half, ptr addrspace(5) %slot syncscope("") unordered, align 2
+ ret half %retval
+}
+
+define half @unordered_half_local_cta_volatile() {
+; CHECK-LABEL: unordered_half_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot369[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot369;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile half, ptr addrspace(5) %slot syncscope("block") unordered, align 2
+ ret half %retval
+}
+
+define half @unordered_half_local_cluster_volatile() {
+; CHECK-LABEL: unordered_half_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot370[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot370;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile half, ptr addrspace(5) %slot syncscope("cluster") unordered, align 2
+ ret half %retval
+}
+
+define half @unordered_half_local_gpu_volatile() {
+; CHECK-LABEL: unordered_half_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot371[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot371;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile half, ptr addrspace(5) %slot syncscope("device") unordered, align 2
+ ret half %retval
+}
+
+define bfloat @unordered_bfloat_local_sys() {
+; CHECK-LABEL: unordered_bfloat_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot372[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot372;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic bfloat, ptr addrspace(5) %slot syncscope("") unordered, align 2
+ ret bfloat %retval
+}
+
+define bfloat @unordered_bfloat_local_cta() {
+; CHECK-LABEL: unordered_bfloat_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot373[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot373;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic bfloat, ptr addrspace(5) %slot syncscope("block") unordered, align 2
+ ret bfloat %retval
+}
+
+define bfloat @unordered_bfloat_local_cluster() {
+; CHECK-LABEL: unordered_bfloat_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot374[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot374;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic bfloat, ptr addrspace(5) %slot syncscope("cluster") unordered, align 2
+ ret bfloat %retval
+}
+
+define bfloat @unordered_bfloat_local_gpu() {
+; CHECK-LABEL: unordered_bfloat_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot375[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot375;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic bfloat, ptr addrspace(5) %slot syncscope("device") unordered, align 2
+ ret bfloat %retval
+}
+
+define bfloat @unordered_bfloat_local_sys_volatile() {
+; CHECK-LABEL: unordered_bfloat_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot376[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot376;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile bfloat, ptr addrspace(5) %slot syncscope("") unordered, align 2
+ ret bfloat %retval
+}
+
+define bfloat @unordered_bfloat_local_cta_volatile() {
+; CHECK-LABEL: unordered_bfloat_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot377[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot377;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile bfloat, ptr addrspace(5) %slot syncscope("block") unordered, align 2
+ ret bfloat %retval
+}
+
+define bfloat @unordered_bfloat_local_cluster_volatile() {
+; CHECK-LABEL: unordered_bfloat_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot378[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot378;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile bfloat, ptr addrspace(5) %slot syncscope("cluster") unordered, align 2
+ ret bfloat %retval
+}
+
+define bfloat @unordered_bfloat_local_gpu_volatile() {
+; CHECK-LABEL: unordered_bfloat_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot379[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot379;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile bfloat, ptr addrspace(5) %slot syncscope("device") unordered, align 2
+ ret bfloat %retval
+}
+
+define float @unordered_float_local_sys() {
+; CHECK-LABEL: unordered_float_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot380[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot380;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic float, ptr addrspace(5) %slot syncscope("") unordered, align 4
+ ret float %retval
+}
+
+define float @unordered_float_local_cta() {
+; CHECK-LABEL: unordered_float_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot381[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot381;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic float, ptr addrspace(5) %slot syncscope("block") unordered, align 4
+ ret float %retval
+}
+
+define float @unordered_float_local_cluster() {
+; CHECK-LABEL: unordered_float_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot382[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot382;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic float, ptr addrspace(5) %slot syncscope("cluster") unordered, align 4
+ ret float %retval
+}
+
+define float @unordered_float_local_gpu() {
+; CHECK-LABEL: unordered_float_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot383[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot383;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic float, ptr addrspace(5) %slot syncscope("device") unordered, align 4
+ ret float %retval
+}
+
+define float @unordered_float_local_sys_volatile() {
+; CHECK-LABEL: unordered_float_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot384[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot384;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile float, ptr addrspace(5) %slot syncscope("") unordered, align 4
+ ret float %retval
+}
+
+define float @unordered_float_local_cta_volatile() {
+; CHECK-LABEL: unordered_float_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot385[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot385;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile float, ptr addrspace(5) %slot syncscope("block") unordered, align 4
+ ret float %retval
+}
+
+define float @unordered_float_local_cluster_volatile() {
+; CHECK-LABEL: unordered_float_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot386[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot386;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile float, ptr addrspace(5) %slot syncscope("cluster") unordered, align 4
+ ret float %retval
+}
+
+define float @unordered_float_local_gpu_volatile() {
+; CHECK-LABEL: unordered_float_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot387[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot387;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile float, ptr addrspace(5) %slot syncscope("device") unordered, align 4
+ ret float %retval
+}
+
+define double @unordered_double_local_sys() {
+; CHECK-LABEL: unordered_double_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot388[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot388;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic double, ptr addrspace(5) %slot syncscope("") unordered, align 8
+ ret double %retval
+}
+
+define double @unordered_double_local_cta() {
+; CHECK-LABEL: unordered_double_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot389[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot389;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic double, ptr addrspace(5) %slot syncscope("block") unordered, align 8
+ ret double %retval
+}
+
+define double @unordered_double_local_cluster() {
+; CHECK-LABEL: unordered_double_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot390[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot390;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic double, ptr addrspace(5) %slot syncscope("cluster") unordered, align 8
+ ret double %retval
+}
+
+define double @unordered_double_local_gpu() {
+; CHECK-LABEL: unordered_double_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot391[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot391;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic double, ptr addrspace(5) %slot syncscope("device") unordered, align 8
+ ret double %retval
+}
+
+define double @unordered_double_local_sys_volatile() {
+; CHECK-LABEL: unordered_double_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot392[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot392;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile double, ptr addrspace(5) %slot syncscope("") unordered, align 8
+ ret double %retval
+}
+
+define double @unordered_double_local_cta_volatile() {
+; CHECK-LABEL: unordered_double_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot393[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot393;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile double, ptr addrspace(5) %slot syncscope("block") unordered, align 8
+ ret double %retval
+}
+
+define double @unordered_double_local_cluster_volatile() {
+; CHECK-LABEL: unordered_double_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot394[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot394;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile double, ptr addrspace(5) %slot syncscope("cluster") unordered, align 8
+ ret double %retval
+}
+
+define double @unordered_double_local_gpu_volatile() {
+; CHECK-LABEL: unordered_double_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot395[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot395;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile double, ptr addrspace(5) %slot syncscope("device") unordered, align 8
+ ret double %retval
+}
+
+define i8 @monotonic_i8_generic_sys(ptr %addr) {
+; CHECK-LABEL: monotonic_i8_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_generic_sys_param_0];
+; CHECK-NEXT: ld.relaxed.sys.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr %addr syncscope("") monotonic, align 1
+ ret i8 %retval
+}
+
+define i8 @monotonic_i8_generic_cta(ptr %addr) {
+; CHECK-LABEL: monotonic_i8_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_generic_cta_param_0];
+; CHECK-NEXT: ld.relaxed.cta.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr %addr syncscope("block") monotonic, align 1
+ ret i8 %retval
+}
+
+define i8 @monotonic_i8_generic_cluster(ptr %addr) {
+; CHECK-LABEL: monotonic_i8_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_generic_cluster_param_0];
+; CHECK-NEXT: ld.relaxed.cluster.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr %addr syncscope("cluster") monotonic, align 1
+ ret i8 %retval
+}
+
+define i8 @monotonic_i8_generic_gpu(ptr %addr) {
+; CHECK-LABEL: monotonic_i8_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_generic_gpu_param_0];
+; CHECK-NEXT: ld.relaxed.gpu.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr %addr syncscope("device") monotonic, align 1
+ ret i8 %retval
+}
+
+define i8 @monotonic_i8_generic_sys_volatile(ptr %addr) {
+; CHECK-LABEL: monotonic_i8_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr %addr syncscope("") monotonic, align 1
+ ret i8 %retval
+}
+
+define i8 @monotonic_i8_generic_cta_volatile(ptr %addr) {
+; CHECK-LABEL: monotonic_i8_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr %addr syncscope("block") monotonic, align 1
+ ret i8 %retval
+}
+
+define i8 @monotonic_i8_generic_cluster_volatile(ptr %addr) {
+; CHECK-LABEL: monotonic_i8_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr %addr syncscope("cluster") monotonic, align 1
+ ret i8 %retval
+}
+
+define i8 @monotonic_i8_generic_gpu_volatile(ptr %addr) {
+; CHECK-LABEL: monotonic_i8_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr %addr syncscope("device") monotonic, align 1
+ ret i8 %retval
+}
+
+define i16 @monotonic_i16_generic_sys(ptr %addr) {
+; CHECK-LABEL: monotonic_i16_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_generic_sys_param_0];
+; CHECK-NEXT: ld.relaxed.sys.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr %addr syncscope("") monotonic, align 2
+ ret i16 %retval
+}
+
+define i16 @monotonic_i16_generic_cta(ptr %addr) {
+; CHECK-LABEL: monotonic_i16_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_generic_cta_param_0];
+; CHECK-NEXT: ld.relaxed.cta.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr %addr syncscope("block") monotonic, align 2
+ ret i16 %retval
+}
+
+define i16 @monotonic_i16_generic_cluster(ptr %addr) {
+; CHECK-LABEL: monotonic_i16_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_generic_cluster_param_0];
+; CHECK-NEXT: ld.relaxed.cluster.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr %addr syncscope("cluster") monotonic, align 2
+ ret i16 %retval
+}
+
+define i16 @monotonic_i16_generic_gpu(ptr %addr) {
+; CHECK-LABEL: monotonic_i16_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_generic_gpu_param_0];
+; CHECK-NEXT: ld.relaxed.gpu.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr %addr syncscope("device") monotonic, align 2
+ ret i16 %retval
+}
+
+define i16 @monotonic_i16_generic_sys_volatile(ptr %addr) {
+; CHECK-LABEL: monotonic_i16_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr %addr syncscope("") monotonic, align 2
+ ret i16 %retval
+}
+
+define i16 @monotonic_i16_generic_cta_volatile(ptr %addr) {
+; CHECK-LABEL: monotonic_i16_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr %addr syncscope("block") monotonic, align 2
+ ret i16 %retval
+}
+
+define i16 @monotonic_i16_generic_cluster_volatile(ptr %addr) {
+; CHECK-LABEL: monotonic_i16_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr %addr syncscope("cluster") monotonic, align 2
+ ret i16 %retval
+}
+
+define i16 @monotonic_i16_generic_gpu_volatile(ptr %addr) {
+; CHECK-LABEL: monotonic_i16_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr %addr syncscope("device") monotonic, align 2
+ ret i16 %retval
+}
+
+define i32 @monotonic_i32_generic_sys(ptr %addr) {
+; CHECK-LABEL: monotonic_i32_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_generic_sys_param_0];
+; CHECK-NEXT: ld.relaxed.sys.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr %addr syncscope("") monotonic, align 4
+ ret i32 %retval
+}
+
+define i32 @monotonic_i32_generic_cta(ptr %addr) {
+; CHECK-LABEL: monotonic_i32_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_generic_cta_param_0];
+; CHECK-NEXT: ld.relaxed.cta.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr %addr syncscope("block") monotonic, align 4
+ ret i32 %retval
+}
+
+define i32 @monotonic_i32_generic_cluster(ptr %addr) {
+; CHECK-LABEL: monotonic_i32_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_generic_cluster_param_0];
+; CHECK-NEXT: ld.relaxed.cluster.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr %addr syncscope("cluster") monotonic, align 4
+ ret i32 %retval
+}
+
+define i32 @monotonic_i32_generic_gpu(ptr %addr) {
+; CHECK-LABEL: monotonic_i32_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_generic_gpu_param_0];
+; CHECK-NEXT: ld.relaxed.gpu.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr %addr syncscope("device") monotonic, align 4
+ ret i32 %retval
+}
+
+define i32 @monotonic_i32_generic_sys_volatile(ptr %addr) {
+; CHECK-LABEL: monotonic_i32_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr %addr syncscope("") monotonic, align 4
+ ret i32 %retval
+}
+
+define i32 @monotonic_i32_generic_cta_volatile(ptr %addr) {
+; CHECK-LABEL: monotonic_i32_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr %addr syncscope("block") monotonic, align 4
+ ret i32 %retval
+}
+
+define i32 @monotonic_i32_generic_cluster_volatile(ptr %addr) {
+; CHECK-LABEL: monotonic_i32_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr %addr syncscope("cluster") monotonic, align 4
+ ret i32 %retval
+}
+
+define i32 @monotonic_i32_generic_gpu_volatile(ptr %addr) {
+; CHECK-LABEL: monotonic_i32_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr %addr syncscope("device") monotonic, align 4
+ ret i32 %retval
+}
+
+define i64 @monotonic_i64_generic_sys(ptr %addr) {
+; CHECK-LABEL: monotonic_i64_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_generic_sys_param_0];
+; CHECK-NEXT: ld.relaxed.sys.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr %addr syncscope("") monotonic, align 8
+ ret i64 %retval
+}
+
+define i64 @monotonic_i64_generic_cta(ptr %addr) {
+; CHECK-LABEL: monotonic_i64_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_generic_cta_param_0];
+; CHECK-NEXT: ld.relaxed.cta.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr %addr syncscope("block") monotonic, align 8
+ ret i64 %retval
+}
+
+define i64 @monotonic_i64_generic_cluster(ptr %addr) {
+; CHECK-LABEL: monotonic_i64_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_generic_cluster_param_0];
+; CHECK-NEXT: ld.relaxed.cluster.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr %addr syncscope("cluster") monotonic, align 8
+ ret i64 %retval
+}
+
+define i64 @monotonic_i64_generic_gpu(ptr %addr) {
+; CHECK-LABEL: monotonic_i64_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_generic_gpu_param_0];
+; CHECK-NEXT: ld.relaxed.gpu.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr %addr syncscope("device") monotonic, align 8
+ ret i64 %retval
+}
+
+define i64 @monotonic_i64_generic_sys_volatile(ptr %addr) {
+; CHECK-LABEL: monotonic_i64_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr %addr syncscope("") monotonic, align 8
+ ret i64 %retval
+}
+
+define i64 @monotonic_i64_generic_cta_volatile(ptr %addr) {
+; CHECK-LABEL: monotonic_i64_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr %addr syncscope("block") monotonic, align 8
+ ret i64 %retval
+}
+
+define i64 @monotonic_i64_generic_cluster_volatile(ptr %addr) {
+; CHECK-LABEL: monotonic_i64_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr %addr syncscope("cluster") monotonic, align 8
+ ret i64 %retval
+}
+
+define i64 @monotonic_i64_generic_gpu_volatile(ptr %addr) {
+; CHECK-LABEL: monotonic_i64_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr %addr syncscope("device") monotonic, align 8
+ ret i64 %retval
+}
+
+define i128 @monotonic_i128_generic_sys(ptr %addr) {
+; CHECK-LABEL: monotonic_i128_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i128_generic_sys_param_0];
+; CHECK-NEXT: mov.b64 %rd2, 0;
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 cmp, swap, dst;
+; CHECK-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; CHECK-NEXT: mov.b128 swap, {%rd2, %rd2};
+; CHECK-NEXT: atom.relaxed.sys.cas.b128 dst, [%rd1], cmp, swap;
+; CHECK-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; CHECK-NEXT: ret;
+ %retval = load atomic i128, ptr %addr syncscope("") monotonic, align 16
+ ret i128 %retval
+}
+
+define i128 @monotonic_i128_generic_cta(ptr %addr) {
+; CHECK-LABEL: monotonic_i128_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i128_generic_cta_param_0];
+; CHECK-NEXT: mov.b64 %rd2, 0;
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 cmp, swap, dst;
+; CHECK-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; CHECK-NEXT: mov.b128 swap, {%rd2, %rd2};
+; CHECK-NEXT: atom.relaxed.cta.cas.b128 dst, [%rd1], cmp, swap;
+; CHECK-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; CHECK-NEXT: ret;
+ %retval = load atomic i128, ptr %addr syncscope("block") monotonic, align 16
+ ret i128 %retval
+}
+
+define i128 @monotonic_i128_generic_cluster(ptr %addr) {
+; CHECK-LABEL: monotonic_i128_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i128_generic_cluster_param_0];
+; CHECK-NEXT: mov.b64 %rd2, 0;
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 cmp, swap, dst;
+; CHECK-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; CHECK-NEXT: mov.b128 swap, {%rd2, %rd2};
+; CHECK-NEXT: atom.relaxed.cluster.cas.b128 dst, [%rd1], cmp, swap;
+; CHECK-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; CHECK-NEXT: ret;
+ %retval = load atomic i128, ptr %addr syncscope("cluster") monotonic, align 16
+ ret i128 %retval
+}
+
+define i128 @monotonic_i128_generic_gpu(ptr %addr) {
+; CHECK-LABEL: monotonic_i128_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i128_generic_gpu_param_0];
+; CHECK-NEXT: mov.b64 %rd2, 0;
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 cmp, swap, dst;
+; CHECK-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; CHECK-NEXT: mov.b128 swap, {%rd2, %rd2};
+; CHECK-NEXT: atom.relaxed.gpu.cas.b128 dst, [%rd1], cmp, swap;
+; CHECK-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; CHECK-NEXT: ret;
+ %retval = load atomic i128, ptr %addr syncscope("device") monotonic, align 16
+ ret i128 %retval
+}
+
+define i128 @monotonic_i128_generic_sys_volatile(ptr %addr) {
+; CHECK-LABEL: monotonic_i128_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i128_generic_sys_volatile_param_0];
+; CHECK-NEXT: mov.b64 %rd2, 0;
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 cmp, swap, dst;
+; CHECK-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; CHECK-NEXT: mov.b128 swap, {%rd2, %rd2};
+; CHECK-NEXT: atom.relaxed.sys.cas.b128 dst, [%rd1], cmp, swap;
+; CHECK-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i128, ptr %addr syncscope("") monotonic, align 16
+ ret i128 %retval
+}
+
+define i128 @monotonic_i128_generic_cta_volatile(ptr %addr) {
+; CHECK-LABEL: monotonic_i128_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i128_generic_cta_volatile_param_0];
+; CHECK-NEXT: mov.b64 %rd2, 0;
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 cmp, swap, dst;
+; CHECK-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; CHECK-NEXT: mov.b128 swap, {%rd2, %rd2};
+; CHECK-NEXT: atom.relaxed.cta.cas.b128 dst, [%rd1], cmp, swap;
+; CHECK-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i128, ptr %addr syncscope("block") monotonic, align 16
+ ret i128 %retval
+}
+
+define i128 @monotonic_i128_generic_cluster_volatile(ptr %addr) {
+; CHECK-LABEL: monotonic_i128_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i128_generic_cluster_volatile_param_0];
+; CHECK-NEXT: mov.b64 %rd2, 0;
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 cmp, swap, dst;
+; CHECK-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; CHECK-NEXT: mov.b128 swap, {%rd2, %rd2};
+; CHECK-NEXT: atom.relaxed.cluster.cas.b128 dst, [%rd1], cmp, swap;
+; CHECK-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i128, ptr %addr syncscope("cluster") monotonic, align 16
+ ret i128 %retval
+}
+
+define i128 @monotonic_i128_generic_gpu_volatile(ptr %addr) {
+; CHECK-LABEL: monotonic_i128_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i128_generic_gpu_volatile_param_0];
+; CHECK-NEXT: mov.b64 %rd2, 0;
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 cmp, swap, dst;
+; CHECK-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; CHECK-NEXT: mov.b128 swap, {%rd2, %rd2};
+; CHECK-NEXT: atom.relaxed.gpu.cas.b128 dst, [%rd1], cmp, swap;
+; CHECK-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i128, ptr %addr syncscope("device") monotonic, align 16
+ ret i128 %retval
+}
+
+define half @monotonic_half_generic_sys(ptr %addr) {
+; CHECK-LABEL: monotonic_half_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_generic_sys_param_0];
+; CHECK-NEXT: ld.relaxed.sys.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr %addr syncscope("") monotonic, align 2
+ ret half %retval
+}
+
+define half @monotonic_half_generic_cta(ptr %addr) {
+; CHECK-LABEL: monotonic_half_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_generic_cta_param_0];
+; CHECK-NEXT: ld.relaxed.cta.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr %addr syncscope("block") monotonic, align 2
+ ret half %retval
+}
+
+define half @monotonic_half_generic_cluster(ptr %addr) {
+; CHECK-LABEL: monotonic_half_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_generic_cluster_param_0];
+; CHECK-NEXT: ld.relaxed.cluster.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr %addr syncscope("cluster") monotonic, align 2
+ ret half %retval
+}
+
+define half @monotonic_half_generic_gpu(ptr %addr) {
+; CHECK-LABEL: monotonic_half_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_generic_gpu_param_0];
+; CHECK-NEXT: ld.relaxed.gpu.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr %addr syncscope("device") monotonic, align 2
+ ret half %retval
+}
+
+define half @monotonic_half_generic_sys_volatile(ptr %addr) {
+; CHECK-LABEL: monotonic_half_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr %addr syncscope("") monotonic, align 2
+ ret half %retval
+}
+
+define half @monotonic_half_generic_cta_volatile(ptr %addr) {
+; CHECK-LABEL: monotonic_half_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr %addr syncscope("block") monotonic, align 2
+ ret half %retval
+}
+
+define half @monotonic_half_generic_cluster_volatile(ptr %addr) {
+; CHECK-LABEL: monotonic_half_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr %addr syncscope("cluster") monotonic, align 2
+ ret half %retval
+}
+
+define half @monotonic_half_generic_gpu_volatile(ptr %addr) {
+; CHECK-LABEL: monotonic_half_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr %addr syncscope("device") monotonic, align 2
+ ret half %retval
+}
+
+define bfloat @monotonic_bfloat_generic_sys(ptr %addr) {
+; CHECK-LABEL: monotonic_bfloat_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_generic_sys_param_0];
+; CHECK-NEXT: ld.relaxed.sys.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr %addr syncscope("") monotonic, align 2
+ ret bfloat %retval
+}
+
+define bfloat @monotonic_bfloat_generic_cta(ptr %addr) {
+; CHECK-LABEL: monotonic_bfloat_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_generic_cta_param_0];
+; CHECK-NEXT: ld.relaxed.cta.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr %addr syncscope("block") monotonic, align 2
+ ret bfloat %retval
+}
+
+define bfloat @monotonic_bfloat_generic_cluster(ptr %addr) {
+; CHECK-LABEL: monotonic_bfloat_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_generic_cluster_param_0];
+; CHECK-NEXT: ld.relaxed.cluster.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr %addr syncscope("cluster") monotonic, align 2
+ ret bfloat %retval
+}
+
+define bfloat @monotonic_bfloat_generic_gpu(ptr %addr) {
+; CHECK-LABEL: monotonic_bfloat_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_generic_gpu_param_0];
+; CHECK-NEXT: ld.relaxed.gpu.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr %addr syncscope("device") monotonic, align 2
+ ret bfloat %retval
+}
+
+define bfloat @monotonic_bfloat_generic_sys_volatile(ptr %addr) {
+; CHECK-LABEL: monotonic_bfloat_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr %addr syncscope("") monotonic, align 2
+ ret bfloat %retval
+}
+
+define bfloat @monotonic_bfloat_generic_cta_volatile(ptr %addr) {
+; CHECK-LABEL: monotonic_bfloat_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr %addr syncscope("block") monotonic, align 2
+ ret bfloat %retval
+}
+
+define bfloat @monotonic_bfloat_generic_cluster_volatile(ptr %addr) {
+; CHECK-LABEL: monotonic_bfloat_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr %addr syncscope("cluster") monotonic, align 2
+ ret bfloat %retval
+}
+
+define bfloat @monotonic_bfloat_generic_gpu_volatile(ptr %addr) {
+; CHECK-LABEL: monotonic_bfloat_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr %addr syncscope("device") monotonic, align 2
+ ret bfloat %retval
+}
+
+define float @monotonic_float_generic_sys(ptr %addr) {
+; CHECK-LABEL: monotonic_float_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_generic_sys_param_0];
+; CHECK-NEXT: ld.relaxed.sys.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr %addr syncscope("") monotonic, align 4
+ ret float %retval
+}
+
+define float @monotonic_float_generic_cta(ptr %addr) {
+; CHECK-LABEL: monotonic_float_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_generic_cta_param_0];
+; CHECK-NEXT: ld.relaxed.cta.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr %addr syncscope("block") monotonic, align 4
+ ret float %retval
+}
+
+define float @monotonic_float_generic_cluster(ptr %addr) {
+; CHECK-LABEL: monotonic_float_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_generic_cluster_param_0];
+; CHECK-NEXT: ld.relaxed.cluster.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr %addr syncscope("cluster") monotonic, align 4
+ ret float %retval
+}
+
+define float @monotonic_float_generic_gpu(ptr %addr) {
+; CHECK-LABEL: monotonic_float_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_generic_gpu_param_0];
+; CHECK-NEXT: ld.relaxed.gpu.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr %addr syncscope("device") monotonic, align 4
+ ret float %retval
+}
+
+define float @monotonic_float_generic_sys_volatile(ptr %addr) {
+; CHECK-LABEL: monotonic_float_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr %addr syncscope("") monotonic, align 4
+ ret float %retval
+}
+
+define float @monotonic_float_generic_cta_volatile(ptr %addr) {
+; CHECK-LABEL: monotonic_float_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr %addr syncscope("block") monotonic, align 4
+ ret float %retval
+}
+
+define float @monotonic_float_generic_cluster_volatile(ptr %addr) {
+; CHECK-LABEL: monotonic_float_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr %addr syncscope("cluster") monotonic, align 4
+ ret float %retval
+}
+
+define float @monotonic_float_generic_gpu_volatile(ptr %addr) {
+; CHECK-LABEL: monotonic_float_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr %addr syncscope("device") monotonic, align 4
+ ret float %retval
+}
+
+define double @monotonic_double_generic_sys(ptr %addr) {
+; CHECK-LABEL: monotonic_double_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_generic_sys_param_0];
+; CHECK-NEXT: ld.relaxed.sys.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr %addr syncscope("") monotonic, align 8
+ ret double %retval
+}
+
+define double @monotonic_double_generic_cta(ptr %addr) {
+; CHECK-LABEL: monotonic_double_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_generic_cta_param_0];
+; CHECK-NEXT: ld.relaxed.cta.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr %addr syncscope("block") monotonic, align 8
+ ret double %retval
+}
+
+define double @monotonic_double_generic_cluster(ptr %addr) {
+; CHECK-LABEL: monotonic_double_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_generic_cluster_param_0];
+; CHECK-NEXT: ld.relaxed.cluster.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr %addr syncscope("cluster") monotonic, align 8
+ ret double %retval
+}
+
+define double @monotonic_double_generic_gpu(ptr %addr) {
+; CHECK-LABEL: monotonic_double_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_generic_gpu_param_0];
+; CHECK-NEXT: ld.relaxed.gpu.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr %addr syncscope("device") monotonic, align 8
+ ret double %retval
+}
+
+define double @monotonic_double_generic_sys_volatile(ptr %addr) {
+; CHECK-LABEL: monotonic_double_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr %addr syncscope("") monotonic, align 8
+ ret double %retval
+}
+
+define double @monotonic_double_generic_cta_volatile(ptr %addr) {
+; CHECK-LABEL: monotonic_double_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr %addr syncscope("block") monotonic, align 8
+ ret double %retval
+}
+
+define double @monotonic_double_generic_cluster_volatile(ptr %addr) {
+; CHECK-LABEL: monotonic_double_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr %addr syncscope("cluster") monotonic, align 8
+ ret double %retval
+}
+
+define double @monotonic_double_generic_gpu_volatile(ptr %addr) {
+; CHECK-LABEL: monotonic_double_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr %addr syncscope("device") monotonic, align 8
+ ret double %retval
+}
+
+define i8 @monotonic_i8_global_sys(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i8_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_global_sys_param_0];
+; CHECK-NEXT: ld.relaxed.sys.global.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr addrspace(1) %addr syncscope("") monotonic, align 1
+ ret i8 %retval
+}
+
+define i8 @monotonic_i8_global_cta(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i8_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_global_cta_param_0];
+; CHECK-NEXT: ld.relaxed.cta.global.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr addrspace(1) %addr syncscope("block") monotonic, align 1
+ ret i8 %retval
+}
+
+define i8 @monotonic_i8_global_cluster(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i8_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_global_cluster_param_0];
+; CHECK-NEXT: ld.relaxed.cluster.global.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr addrspace(1) %addr syncscope("cluster") monotonic, align 1
+ ret i8 %retval
+}
+
+define i8 @monotonic_i8_global_gpu(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i8_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_global_gpu_param_0];
+; CHECK-NEXT: ld.relaxed.gpu.global.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr addrspace(1) %addr syncscope("device") monotonic, align 1
+ ret i8 %retval
+}
+
+define i8 @monotonic_i8_global_sys_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i8_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.mmio.relaxed.sys.global.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr addrspace(1) %addr syncscope("") monotonic, align 1
+ ret i8 %retval
+}
+
+define i8 @monotonic_i8_global_cta_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i8_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.mmio.relaxed.sys.global.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr addrspace(1) %addr syncscope("block") monotonic, align 1
+ ret i8 %retval
+}
+
+define i8 @monotonic_i8_global_cluster_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i8_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.mmio.relaxed.sys.global.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr addrspace(1) %addr syncscope("cluster") monotonic, align 1
+ ret i8 %retval
+}
+
+define i8 @monotonic_i8_global_gpu_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i8_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.mmio.relaxed.sys.global.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr addrspace(1) %addr syncscope("device") monotonic, align 1
+ ret i8 %retval
+}
+
+define i16 @monotonic_i16_global_sys(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i16_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_global_sys_param_0];
+; CHECK-NEXT: ld.relaxed.sys.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr addrspace(1) %addr syncscope("") monotonic, align 2
+ ret i16 %retval
+}
+
+define i16 @monotonic_i16_global_cta(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i16_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_global_cta_param_0];
+; CHECK-NEXT: ld.relaxed.cta.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr addrspace(1) %addr syncscope("block") monotonic, align 2
+ ret i16 %retval
+}
+
+define i16 @monotonic_i16_global_cluster(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i16_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_global_cluster_param_0];
+; CHECK-NEXT: ld.relaxed.cluster.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr addrspace(1) %addr syncscope("cluster") monotonic, align 2
+ ret i16 %retval
+}
+
+define i16 @monotonic_i16_global_gpu(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i16_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_global_gpu_param_0];
+; CHECK-NEXT: ld.relaxed.gpu.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr addrspace(1) %addr syncscope("device") monotonic, align 2
+ ret i16 %retval
+}
+
+define i16 @monotonic_i16_global_sys_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i16_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.mmio.relaxed.sys.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr addrspace(1) %addr syncscope("") monotonic, align 2
+ ret i16 %retval
+}
+
+define i16 @monotonic_i16_global_cta_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i16_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.mmio.relaxed.sys.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr addrspace(1) %addr syncscope("block") monotonic, align 2
+ ret i16 %retval
+}
+
+define i16 @monotonic_i16_global_cluster_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i16_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.mmio.relaxed.sys.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr addrspace(1) %addr syncscope("cluster") monotonic, align 2
+ ret i16 %retval
+}
+
+define i16 @monotonic_i16_global_gpu_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i16_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.mmio.relaxed.sys.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr addrspace(1) %addr syncscope("device") monotonic, align 2
+ ret i16 %retval
+}
+
+define i32 @monotonic_i32_global_sys(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i32_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_global_sys_param_0];
+; CHECK-NEXT: ld.relaxed.sys.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr addrspace(1) %addr syncscope("") monotonic, align 4
+ ret i32 %retval
+}
+
+define i32 @monotonic_i32_global_cta(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i32_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_global_cta_param_0];
+; CHECK-NEXT: ld.relaxed.cta.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr addrspace(1) %addr syncscope("block") monotonic, align 4
+ ret i32 %retval
+}
+
+define i32 @monotonic_i32_global_cluster(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i32_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_global_cluster_param_0];
+; CHECK-NEXT: ld.relaxed.cluster.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr addrspace(1) %addr syncscope("cluster") monotonic, align 4
+ ret i32 %retval
+}
+
+define i32 @monotonic_i32_global_gpu(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i32_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_global_gpu_param_0];
+; CHECK-NEXT: ld.relaxed.gpu.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr addrspace(1) %addr syncscope("device") monotonic, align 4
+ ret i32 %retval
+}
+
+define i32 @monotonic_i32_global_sys_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i32_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.mmio.relaxed.sys.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr addrspace(1) %addr syncscope("") monotonic, align 4
+ ret i32 %retval
+}
+
+define i32 @monotonic_i32_global_cta_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i32_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.mmio.relaxed.sys.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr addrspace(1) %addr syncscope("block") monotonic, align 4
+ ret i32 %retval
+}
+
+define i32 @monotonic_i32_global_cluster_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i32_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.mmio.relaxed.sys.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr addrspace(1) %addr syncscope("cluster") monotonic, align 4
+ ret i32 %retval
+}
+
+define i32 @monotonic_i32_global_gpu_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i32_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.mmio.relaxed.sys.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr addrspace(1) %addr syncscope("device") monotonic, align 4
+ ret i32 %retval
+}
+
+define i64 @monotonic_i64_global_sys(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i64_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_global_sys_param_0];
+; CHECK-NEXT: ld.relaxed.sys.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr addrspace(1) %addr syncscope("") monotonic, align 8
+ ret i64 %retval
+}
+
+define i64 @monotonic_i64_global_cta(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i64_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_global_cta_param_0];
+; CHECK-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr addrspace(1) %addr syncscope("block") monotonic, align 8
+ ret i64 %retval
+}
+
+define i64 @monotonic_i64_global_cluster(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i64_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_global_cluster_param_0];
+; CHECK-NEXT: ld.relaxed.cluster.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr addrspace(1) %addr syncscope("cluster") monotonic, align 8
+ ret i64 %retval
+}
+
+define i64 @monotonic_i64_global_gpu(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i64_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_global_gpu_param_0];
+; CHECK-NEXT: ld.relaxed.gpu.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr addrspace(1) %addr syncscope("device") monotonic, align 8
+ ret i64 %retval
+}
+
+define i64 @monotonic_i64_global_sys_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i64_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.mmio.relaxed.sys.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr addrspace(1) %addr syncscope("") monotonic, align 8
+ ret i64 %retval
+}
+
+define i64 @monotonic_i64_global_cta_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i64_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.mmio.relaxed.sys.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr addrspace(1) %addr syncscope("block") monotonic, align 8
+ ret i64 %retval
+}
+
+define i64 @monotonic_i64_global_cluster_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i64_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.mmio.relaxed.sys.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr addrspace(1) %addr syncscope("cluster") monotonic, align 8
+ ret i64 %retval
+}
+
+define i64 @monotonic_i64_global_gpu_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i64_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.mmio.relaxed.sys.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr addrspace(1) %addr syncscope("device") monotonic, align 8
+ ret i64 %retval
+}
+
+define i128 @monotonic_i128_global_sys(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i128_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i128_global_sys_param_0];
+; CHECK-NEXT: mov.b64 %rd2, 0;
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 cmp, swap, dst;
+; CHECK-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; CHECK-NEXT: mov.b128 swap, {%rd2, %rd2};
+; CHECK-NEXT: atom.relaxed.sys.global.cas.b128 dst, [%rd1], cmp, swap;
+; CHECK-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; CHECK-NEXT: ret;
+ %retval = load atomic i128, ptr addrspace(1) %addr syncscope("") monotonic, align 16
+ ret i128 %retval
+}
+
+define i128 @monotonic_i128_global_cta(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i128_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i128_global_cta_param_0];
+; CHECK-NEXT: mov.b64 %rd2, 0;
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 cmp, swap, dst;
+; CHECK-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; CHECK-NEXT: mov.b128 swap, {%rd2, %rd2};
+; CHECK-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; CHECK-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; CHECK-NEXT: ret;
+ %retval = load atomic i128, ptr addrspace(1) %addr syncscope("block") monotonic, align 16
+ ret i128 %retval
+}
+
+define i128 @monotonic_i128_global_cluster(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i128_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i128_global_cluster_param_0];
+; CHECK-NEXT: mov.b64 %rd2, 0;
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 cmp, swap, dst;
+; CHECK-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; CHECK-NEXT: mov.b128 swap, {%rd2, %rd2};
+; CHECK-NEXT: atom.relaxed.cluster.global.cas.b128 dst, [%rd1], cmp, swap;
+; CHECK-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; CHECK-NEXT: ret;
+ %retval = load atomic i128, ptr addrspace(1) %addr syncscope("cluster") monotonic, align 16
+ ret i128 %retval
+}
+
+define i128 @monotonic_i128_global_gpu(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i128_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i128_global_gpu_param_0];
+; CHECK-NEXT: mov.b64 %rd2, 0;
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 cmp, swap, dst;
+; CHECK-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; CHECK-NEXT: mov.b128 swap, {%rd2, %rd2};
+; CHECK-NEXT: atom.relaxed.gpu.global.cas.b128 dst, [%rd1], cmp, swap;
+; CHECK-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; CHECK-NEXT: ret;
+ %retval = load atomic i128, ptr addrspace(1) %addr syncscope("device") monotonic, align 16
+ ret i128 %retval
+}
+
+define i128 @monotonic_i128_global_sys_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i128_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i128_global_sys_volatile_param_0];
+; CHECK-NEXT: mov.b64 %rd2, 0;
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 cmp, swap, dst;
+; CHECK-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; CHECK-NEXT: mov.b128 swap, {%rd2, %rd2};
+; CHECK-NEXT: atom.relaxed.sys.global.cas.b128 dst, [%rd1], cmp, swap;
+; CHECK-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i128, ptr addrspace(1) %addr syncscope("") monotonic, align 16
+ ret i128 %retval
+}
+
+define i128 @monotonic_i128_global_cta_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i128_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i128_global_cta_volatile_param_0];
+; CHECK-NEXT: mov.b64 %rd2, 0;
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 cmp, swap, dst;
+; CHECK-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; CHECK-NEXT: mov.b128 swap, {%rd2, %rd2};
+; CHECK-NEXT: atom.relaxed.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; CHECK-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i128, ptr addrspace(1) %addr syncscope("block") monotonic, align 16
+ ret i128 %retval
+}
+
+define i128 @monotonic_i128_global_cluster_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i128_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i128_global_cluster_volatile_param_0];
+; CHECK-NEXT: mov.b64 %rd2, 0;
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 cmp, swap, dst;
+; CHECK-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; CHECK-NEXT: mov.b128 swap, {%rd2, %rd2};
+; CHECK-NEXT: atom.relaxed.cluster.global.cas.b128 dst, [%rd1], cmp, swap;
+; CHECK-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i128, ptr addrspace(1) %addr syncscope("cluster") monotonic, align 16
+ ret i128 %retval
+}
+
+define i128 @monotonic_i128_global_gpu_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i128_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i128_global_gpu_volatile_param_0];
+; CHECK-NEXT: mov.b64 %rd2, 0;
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 cmp, swap, dst;
+; CHECK-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; CHECK-NEXT: mov.b128 swap, {%rd2, %rd2};
+; CHECK-NEXT: atom.relaxed.gpu.global.cas.b128 dst, [%rd1], cmp, swap;
+; CHECK-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i128, ptr addrspace(1) %addr syncscope("device") monotonic, align 16
+ ret i128 %retval
+}
+
+define half @monotonic_half_global_sys(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_half_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_global_sys_param_0];
+; CHECK-NEXT: ld.relaxed.sys.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr addrspace(1) %addr syncscope("") monotonic, align 2
+ ret half %retval
+}
+
+define half @monotonic_half_global_cta(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_half_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_global_cta_param_0];
+; CHECK-NEXT: ld.relaxed.cta.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr addrspace(1) %addr syncscope("block") monotonic, align 2
+ ret half %retval
+}
+
+define half @monotonic_half_global_cluster(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_half_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_global_cluster_param_0];
+; CHECK-NEXT: ld.relaxed.cluster.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr addrspace(1) %addr syncscope("cluster") monotonic, align 2
+ ret half %retval
+}
+
+define half @monotonic_half_global_gpu(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_half_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_global_gpu_param_0];
+; CHECK-NEXT: ld.relaxed.gpu.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr addrspace(1) %addr syncscope("device") monotonic, align 2
+ ret half %retval
+}
+
+define half @monotonic_half_global_sys_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_half_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.mmio.relaxed.sys.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr addrspace(1) %addr syncscope("") monotonic, align 2
+ ret half %retval
+}
+
+define half @monotonic_half_global_cta_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_half_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.mmio.relaxed.sys.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr addrspace(1) %addr syncscope("block") monotonic, align 2
+ ret half %retval
+}
+
+define half @monotonic_half_global_cluster_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_half_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.mmio.relaxed.sys.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr addrspace(1) %addr syncscope("cluster") monotonic, align 2
+ ret half %retval
+}
+
+define half @monotonic_half_global_gpu_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_half_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.mmio.relaxed.sys.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr addrspace(1) %addr syncscope("device") monotonic, align 2
+ ret half %retval
+}
+
+define bfloat @monotonic_bfloat_global_sys(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_bfloat_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_global_sys_param_0];
+; CHECK-NEXT: ld.relaxed.sys.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr addrspace(1) %addr syncscope("") monotonic, align 2
+ ret bfloat %retval
+}
+
+define bfloat @monotonic_bfloat_global_cta(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_bfloat_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_global_cta_param_0];
+; CHECK-NEXT: ld.relaxed.cta.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr addrspace(1) %addr syncscope("block") monotonic, align 2
+ ret bfloat %retval
+}
+
+define bfloat @monotonic_bfloat_global_cluster(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_bfloat_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_global_cluster_param_0];
+; CHECK-NEXT: ld.relaxed.cluster.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr addrspace(1) %addr syncscope("cluster") monotonic, align 2
+ ret bfloat %retval
+}
+
+define bfloat @monotonic_bfloat_global_gpu(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_bfloat_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_global_gpu_param_0];
+; CHECK-NEXT: ld.relaxed.gpu.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr addrspace(1) %addr syncscope("device") monotonic, align 2
+ ret bfloat %retval
+}
+
+define bfloat @monotonic_bfloat_global_sys_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_bfloat_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.mmio.relaxed.sys.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr addrspace(1) %addr syncscope("") monotonic, align 2
+ ret bfloat %retval
+}
+
+define bfloat @monotonic_bfloat_global_cta_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_bfloat_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.mmio.relaxed.sys.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr addrspace(1) %addr syncscope("block") monotonic, align 2
+ ret bfloat %retval
+}
+
+define bfloat @monotonic_bfloat_global_cluster_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_bfloat_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.mmio.relaxed.sys.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr addrspace(1) %addr syncscope("cluster") monotonic, align 2
+ ret bfloat %retval
+}
+
+define bfloat @monotonic_bfloat_global_gpu_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_bfloat_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.mmio.relaxed.sys.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr addrspace(1) %addr syncscope("device") monotonic, align 2
+ ret bfloat %retval
+}
+
+define float @monotonic_float_global_sys(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_float_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_global_sys_param_0];
+; CHECK-NEXT: ld.relaxed.sys.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr addrspace(1) %addr syncscope("") monotonic, align 4
+ ret float %retval
+}
+
+define float @monotonic_float_global_cta(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_float_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_global_cta_param_0];
+; CHECK-NEXT: ld.relaxed.cta.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr addrspace(1) %addr syncscope("block") monotonic, align 4
+ ret float %retval
+}
+
+define float @monotonic_float_global_cluster(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_float_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_global_cluster_param_0];
+; CHECK-NEXT: ld.relaxed.cluster.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr addrspace(1) %addr syncscope("cluster") monotonic, align 4
+ ret float %retval
+}
+
+define float @monotonic_float_global_gpu(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_float_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_global_gpu_param_0];
+; CHECK-NEXT: ld.relaxed.gpu.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr addrspace(1) %addr syncscope("device") monotonic, align 4
+ ret float %retval
+}
+
+define float @monotonic_float_global_sys_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_float_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.mmio.relaxed.sys.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr addrspace(1) %addr syncscope("") monotonic, align 4
+ ret float %retval
+}
+
+define float @monotonic_float_global_cta_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_float_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.mmio.relaxed.sys.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr addrspace(1) %addr syncscope("block") monotonic, align 4
+ ret float %retval
+}
+
+define float @monotonic_float_global_cluster_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_float_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.mmio.relaxed.sys.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr addrspace(1) %addr syncscope("cluster") monotonic, align 4
+ ret float %retval
+}
+
+define float @monotonic_float_global_gpu_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_float_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.mmio.relaxed.sys.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr addrspace(1) %addr syncscope("device") monotonic, align 4
+ ret float %retval
+}
+
+define double @monotonic_double_global_sys(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_double_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_global_sys_param_0];
+; CHECK-NEXT: ld.relaxed.sys.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr addrspace(1) %addr syncscope("") monotonic, align 8
+ ret double %retval
+}
+
+define double @monotonic_double_global_cta(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_double_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_global_cta_param_0];
+; CHECK-NEXT: ld.relaxed.cta.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr addrspace(1) %addr syncscope("block") monotonic, align 8
+ ret double %retval
+}
+
+define double @monotonic_double_global_cluster(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_double_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_global_cluster_param_0];
+; CHECK-NEXT: ld.relaxed.cluster.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr addrspace(1) %addr syncscope("cluster") monotonic, align 8
+ ret double %retval
+}
+
+define double @monotonic_double_global_gpu(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_double_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_global_gpu_param_0];
+; CHECK-NEXT: ld.relaxed.gpu.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr addrspace(1) %addr syncscope("device") monotonic, align 8
+ ret double %retval
+}
+
+define double @monotonic_double_global_sys_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_double_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.mmio.relaxed.sys.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr addrspace(1) %addr syncscope("") monotonic, align 8
+ ret double %retval
+}
+
+define double @monotonic_double_global_cta_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_double_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.mmio.relaxed.sys.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr addrspace(1) %addr syncscope("block") monotonic, align 8
+ ret double %retval
+}
+
+define double @monotonic_double_global_cluster_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_double_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.mmio.relaxed.sys.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr addrspace(1) %addr syncscope("cluster") monotonic, align 8
+ ret double %retval
+}
+
+define double @monotonic_double_global_gpu_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_double_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.mmio.relaxed.sys.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr addrspace(1) %addr syncscope("device") monotonic, align 8
+ ret double %retval
+}
+
+define i8 @monotonic_i8_shared_sys(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i8_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_shared_sys_param_0];
+; CHECK-NEXT: ld.relaxed.sys.shared.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr addrspace(3) %addr syncscope("") monotonic, align 1
+ ret i8 %retval
+}
+
+define i8 @monotonic_i8_shared_cta(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i8_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_shared_cta_param_0];
+; CHECK-NEXT: ld.relaxed.cta.shared.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr addrspace(3) %addr syncscope("block") monotonic, align 1
+ ret i8 %retval
+}
+
+define i8 @monotonic_i8_shared_cluster(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i8_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_shared_cluster_param_0];
+; CHECK-NEXT: ld.relaxed.cluster.shared.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr addrspace(3) %addr syncscope("cluster") monotonic, align 1
+ ret i8 %retval
+}
+
+define i8 @monotonic_i8_shared_gpu(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i8_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_shared_gpu_param_0];
+; CHECK-NEXT: ld.relaxed.gpu.shared.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr addrspace(3) %addr syncscope("device") monotonic, align 1
+ ret i8 %retval
+}
+
+define i8 @monotonic_i8_shared_sys_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i8_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr addrspace(3) %addr syncscope("") monotonic, align 1
+ ret i8 %retval
+}
+
+define i8 @monotonic_i8_shared_cta_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i8_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr addrspace(3) %addr syncscope("block") monotonic, align 1
+ ret i8 %retval
+}
+
+define i8 @monotonic_i8_shared_cluster_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i8_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr addrspace(3) %addr syncscope("cluster") monotonic, align 1
+ ret i8 %retval
+}
+
+define i8 @monotonic_i8_shared_gpu_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i8_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr addrspace(3) %addr syncscope("device") monotonic, align 1
+ ret i8 %retval
+}
+
+define i16 @monotonic_i16_shared_sys(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i16_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_shared_sys_param_0];
+; CHECK-NEXT: ld.relaxed.sys.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr addrspace(3) %addr syncscope("") monotonic, align 2
+ ret i16 %retval
+}
+
+define i16 @monotonic_i16_shared_cta(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i16_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_shared_cta_param_0];
+; CHECK-NEXT: ld.relaxed.cta.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr addrspace(3) %addr syncscope("block") monotonic, align 2
+ ret i16 %retval
+}
+
+define i16 @monotonic_i16_shared_cluster(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i16_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_shared_cluster_param_0];
+; CHECK-NEXT: ld.relaxed.cluster.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr addrspace(3) %addr syncscope("cluster") monotonic, align 2
+ ret i16 %retval
+}
+
+define i16 @monotonic_i16_shared_gpu(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i16_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_shared_gpu_param_0];
+; CHECK-NEXT: ld.relaxed.gpu.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr addrspace(3) %addr syncscope("device") monotonic, align 2
+ ret i16 %retval
+}
+
+define i16 @monotonic_i16_shared_sys_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i16_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr addrspace(3) %addr syncscope("") monotonic, align 2
+ ret i16 %retval
+}
+
+define i16 @monotonic_i16_shared_cta_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i16_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr addrspace(3) %addr syncscope("block") monotonic, align 2
+ ret i16 %retval
+}
+
+define i16 @monotonic_i16_shared_cluster_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i16_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr addrspace(3) %addr syncscope("cluster") monotonic, align 2
+ ret i16 %retval
+}
+
+define i16 @monotonic_i16_shared_gpu_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i16_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr addrspace(3) %addr syncscope("device") monotonic, align 2
+ ret i16 %retval
+}
+
+define i32 @monotonic_i32_shared_sys(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i32_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_shared_sys_param_0];
+; CHECK-NEXT: ld.relaxed.sys.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr addrspace(3) %addr syncscope("") monotonic, align 4
+ ret i32 %retval
+}
+
+define i32 @monotonic_i32_shared_cta(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i32_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_shared_cta_param_0];
+; CHECK-NEXT: ld.relaxed.cta.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr addrspace(3) %addr syncscope("block") monotonic, align 4
+ ret i32 %retval
+}
+
+define i32 @monotonic_i32_shared_cluster(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i32_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_shared_cluster_param_0];
+; CHECK-NEXT: ld.relaxed.cluster.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr addrspace(3) %addr syncscope("cluster") monotonic, align 4
+ ret i32 %retval
+}
+
+define i32 @monotonic_i32_shared_gpu(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i32_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_shared_gpu_param_0];
+; CHECK-NEXT: ld.relaxed.gpu.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr addrspace(3) %addr syncscope("device") monotonic, align 4
+ ret i32 %retval
+}
+
+define i32 @monotonic_i32_shared_sys_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i32_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr addrspace(3) %addr syncscope("") monotonic, align 4
+ ret i32 %retval
+}
+
+define i32 @monotonic_i32_shared_cta_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i32_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr addrspace(3) %addr syncscope("block") monotonic, align 4
+ ret i32 %retval
+}
+
+define i32 @monotonic_i32_shared_cluster_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i32_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr addrspace(3) %addr syncscope("cluster") monotonic, align 4
+ ret i32 %retval
+}
+
+define i32 @monotonic_i32_shared_gpu_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i32_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr addrspace(3) %addr syncscope("device") monotonic, align 4
+ ret i32 %retval
+}
+
+define i64 @monotonic_i64_shared_sys(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i64_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_shared_sys_param_0];
+; CHECK-NEXT: ld.relaxed.sys.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr addrspace(3) %addr syncscope("") monotonic, align 8
+ ret i64 %retval
+}
+
+define i64 @monotonic_i64_shared_cta(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i64_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_shared_cta_param_0];
+; CHECK-NEXT: ld.relaxed.cta.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr addrspace(3) %addr syncscope("block") monotonic, align 8
+ ret i64 %retval
+}
+
+define i64 @monotonic_i64_shared_cluster(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i64_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_shared_cluster_param_0];
+; CHECK-NEXT: ld.relaxed.cluster.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr addrspace(3) %addr syncscope("cluster") monotonic, align 8
+ ret i64 %retval
+}
+
+define i64 @monotonic_i64_shared_gpu(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i64_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_shared_gpu_param_0];
+; CHECK-NEXT: ld.relaxed.gpu.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr addrspace(3) %addr syncscope("device") monotonic, align 8
+ ret i64 %retval
+}
+
+define i64 @monotonic_i64_shared_sys_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i64_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr addrspace(3) %addr syncscope("") monotonic, align 8
+ ret i64 %retval
+}
+
+define i64 @monotonic_i64_shared_cta_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i64_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr addrspace(3) %addr syncscope("block") monotonic, align 8
+ ret i64 %retval
+}
+
+define i64 @monotonic_i64_shared_cluster_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i64_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr addrspace(3) %addr syncscope("cluster") monotonic, align 8
+ ret i64 %retval
+}
+
+define i64 @monotonic_i64_shared_gpu_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i64_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr addrspace(3) %addr syncscope("device") monotonic, align 8
+ ret i64 %retval
+}
+
+define i128 @monotonic_i128_shared_sys(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i128_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i128_shared_sys_param_0];
+; CHECK-NEXT: mov.b64 %rd2, 0;
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 cmp, swap, dst;
+; CHECK-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; CHECK-NEXT: mov.b128 swap, {%rd2, %rd2};
+; CHECK-NEXT: atom.relaxed.sys.shared.cas.b128 dst, [%rd1], cmp, swap;
+; CHECK-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; CHECK-NEXT: ret;
+ %retval = load atomic i128, ptr addrspace(3) %addr syncscope("") monotonic, align 16
+ ret i128 %retval
+}
+
+define i128 @monotonic_i128_shared_cta(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i128_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i128_shared_cta_param_0];
+; CHECK-NEXT: mov.b64 %rd2, 0;
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 cmp, swap, dst;
+; CHECK-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; CHECK-NEXT: mov.b128 swap, {%rd2, %rd2};
+; CHECK-NEXT: atom.relaxed.cta.shared.cas.b128 dst, [%rd1], cmp, swap;
+; CHECK-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; CHECK-NEXT: ret;
+ %retval = load atomic i128, ptr addrspace(3) %addr syncscope("block") monotonic, align 16
+ ret i128 %retval
+}
+
+define i128 @monotonic_i128_shared_cluster(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i128_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i128_shared_cluster_param_0];
+; CHECK-NEXT: mov.b64 %rd2, 0;
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 cmp, swap, dst;
+; CHECK-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; CHECK-NEXT: mov.b128 swap, {%rd2, %rd2};
+; CHECK-NEXT: atom.relaxed.cluster.shared.cas.b128 dst, [%rd1], cmp, swap;
+; CHECK-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; CHECK-NEXT: ret;
+ %retval = load atomic i128, ptr addrspace(3) %addr syncscope("cluster") monotonic, align 16
+ ret i128 %retval
+}
+
+define i128 @monotonic_i128_shared_gpu(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i128_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i128_shared_gpu_param_0];
+; CHECK-NEXT: mov.b64 %rd2, 0;
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 cmp, swap, dst;
+; CHECK-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; CHECK-NEXT: mov.b128 swap, {%rd2, %rd2};
+; CHECK-NEXT: atom.relaxed.gpu.shared.cas.b128 dst, [%rd1], cmp, swap;
+; CHECK-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; CHECK-NEXT: ret;
+ %retval = load atomic i128, ptr addrspace(3) %addr syncscope("device") monotonic, align 16
+ ret i128 %retval
+}
+
+define i128 @monotonic_i128_shared_sys_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i128_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i128_shared_sys_volatile_param_0];
+; CHECK-NEXT: mov.b64 %rd2, 0;
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 cmp, swap, dst;
+; CHECK-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; CHECK-NEXT: mov.b128 swap, {%rd2, %rd2};
+; CHECK-NEXT: atom.relaxed.sys.shared.cas.b128 dst, [%rd1], cmp, swap;
+; CHECK-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i128, ptr addrspace(3) %addr syncscope("") monotonic, align 16
+ ret i128 %retval
+}
+
+define i128 @monotonic_i128_shared_cta_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i128_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i128_shared_cta_volatile_param_0];
+; CHECK-NEXT: mov.b64 %rd2, 0;
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 cmp, swap, dst;
+; CHECK-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; CHECK-NEXT: mov.b128 swap, {%rd2, %rd2};
+; CHECK-NEXT: atom.relaxed.cta.shared.cas.b128 dst, [%rd1], cmp, swap;
+; CHECK-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i128, ptr addrspace(3) %addr syncscope("block") monotonic, align 16
+ ret i128 %retval
+}
+
+define i128 @monotonic_i128_shared_cluster_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i128_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i128_shared_cluster_volatile_param_0];
+; CHECK-NEXT: mov.b64 %rd2, 0;
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 cmp, swap, dst;
+; CHECK-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; CHECK-NEXT: mov.b128 swap, {%rd2, %rd2};
+; CHECK-NEXT: atom.relaxed.cluster.shared.cas.b128 dst, [%rd1], cmp, swap;
+; CHECK-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i128, ptr addrspace(3) %addr syncscope("cluster") monotonic, align 16
+ ret i128 %retval
+}
+
+define i128 @monotonic_i128_shared_gpu_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i128_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i128_shared_gpu_volatile_param_0];
+; CHECK-NEXT: mov.b64 %rd2, 0;
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 cmp, swap, dst;
+; CHECK-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; CHECK-NEXT: mov.b128 swap, {%rd2, %rd2};
+; CHECK-NEXT: atom.relaxed.gpu.shared.cas.b128 dst, [%rd1], cmp, swap;
+; CHECK-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i128, ptr addrspace(3) %addr syncscope("device") monotonic, align 16
+ ret i128 %retval
+}
+
+define half @monotonic_half_shared_sys(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_half_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_shared_sys_param_0];
+; CHECK-NEXT: ld.relaxed.sys.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr addrspace(3) %addr syncscope("") monotonic, align 2
+ ret half %retval
+}
+
+define half @monotonic_half_shared_cta(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_half_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_shared_cta_param_0];
+; CHECK-NEXT: ld.relaxed.cta.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr addrspace(3) %addr syncscope("block") monotonic, align 2
+ ret half %retval
+}
+
+define half @monotonic_half_shared_cluster(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_half_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_shared_cluster_param_0];
+; CHECK-NEXT: ld.relaxed.cluster.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr addrspace(3) %addr syncscope("cluster") monotonic, align 2
+ ret half %retval
+}
+
+define half @monotonic_half_shared_gpu(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_half_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_shared_gpu_param_0];
+; CHECK-NEXT: ld.relaxed.gpu.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr addrspace(3) %addr syncscope("device") monotonic, align 2
+ ret half %retval
+}
+
+define half @monotonic_half_shared_sys_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_half_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr addrspace(3) %addr syncscope("") monotonic, align 2
+ ret half %retval
+}
+
+define half @monotonic_half_shared_cta_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_half_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr addrspace(3) %addr syncscope("block") monotonic, align 2
+ ret half %retval
+}
+
+define half @monotonic_half_shared_cluster_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_half_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr addrspace(3) %addr syncscope("cluster") monotonic, align 2
+ ret half %retval
+}
+
+define half @monotonic_half_shared_gpu_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_half_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr addrspace(3) %addr syncscope("device") monotonic, align 2
+ ret half %retval
+}
+
+define bfloat @monotonic_bfloat_shared_sys(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_bfloat_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_shared_sys_param_0];
+; CHECK-NEXT: ld.relaxed.sys.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr addrspace(3) %addr syncscope("") monotonic, align 2
+ ret bfloat %retval
+}
+
+define bfloat @monotonic_bfloat_shared_cta(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_bfloat_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_shared_cta_param_0];
+; CHECK-NEXT: ld.relaxed.cta.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr addrspace(3) %addr syncscope("block") monotonic, align 2
+ ret bfloat %retval
+}
+
+define bfloat @monotonic_bfloat_shared_cluster(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_bfloat_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_shared_cluster_param_0];
+; CHECK-NEXT: ld.relaxed.cluster.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr addrspace(3) %addr syncscope("cluster") monotonic, align 2
+ ret bfloat %retval
+}
+
+define bfloat @monotonic_bfloat_shared_gpu(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_bfloat_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_shared_gpu_param_0];
+; CHECK-NEXT: ld.relaxed.gpu.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr addrspace(3) %addr syncscope("device") monotonic, align 2
+ ret bfloat %retval
+}
+
+define bfloat @monotonic_bfloat_shared_sys_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_bfloat_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr addrspace(3) %addr syncscope("") monotonic, align 2
+ ret bfloat %retval
+}
+
+define bfloat @monotonic_bfloat_shared_cta_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_bfloat_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr addrspace(3) %addr syncscope("block") monotonic, align 2
+ ret bfloat %retval
+}
+
+define bfloat @monotonic_bfloat_shared_cluster_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_bfloat_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr addrspace(3) %addr syncscope("cluster") monotonic, align 2
+ ret bfloat %retval
+}
+
+define bfloat @monotonic_bfloat_shared_gpu_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_bfloat_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr addrspace(3) %addr syncscope("device") monotonic, align 2
+ ret bfloat %retval
+}
+
+define float @monotonic_float_shared_sys(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_float_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_shared_sys_param_0];
+; CHECK-NEXT: ld.relaxed.sys.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr addrspace(3) %addr syncscope("") monotonic, align 4
+ ret float %retval
+}
+
+define float @monotonic_float_shared_cta(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_float_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_shared_cta_param_0];
+; CHECK-NEXT: ld.relaxed.cta.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr addrspace(3) %addr syncscope("block") monotonic, align 4
+ ret float %retval
+}
+
+define float @monotonic_float_shared_cluster(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_float_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_shared_cluster_param_0];
+; CHECK-NEXT: ld.relaxed.cluster.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr addrspace(3) %addr syncscope("cluster") monotonic, align 4
+ ret float %retval
+}
+
+define float @monotonic_float_shared_gpu(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_float_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_shared_gpu_param_0];
+; CHECK-NEXT: ld.relaxed.gpu.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr addrspace(3) %addr syncscope("device") monotonic, align 4
+ ret float %retval
+}
+
+define float @monotonic_float_shared_sys_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_float_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr addrspace(3) %addr syncscope("") monotonic, align 4
+ ret float %retval
+}
+
+define float @monotonic_float_shared_cta_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_float_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr addrspace(3) %addr syncscope("block") monotonic, align 4
+ ret float %retval
+}
+
+define float @monotonic_float_shared_cluster_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_float_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr addrspace(3) %addr syncscope("cluster") monotonic, align 4
+ ret float %retval
+}
+
+define float @monotonic_float_shared_gpu_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_float_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr addrspace(3) %addr syncscope("device") monotonic, align 4
+ ret float %retval
+}
+
+define double @monotonic_double_shared_sys(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_double_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_shared_sys_param_0];
+; CHECK-NEXT: ld.relaxed.sys.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr addrspace(3) %addr syncscope("") monotonic, align 8
+ ret double %retval
+}
+
+define double @monotonic_double_shared_cta(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_double_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_shared_cta_param_0];
+; CHECK-NEXT: ld.relaxed.cta.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr addrspace(3) %addr syncscope("block") monotonic, align 8
+ ret double %retval
+}
+
+define double @monotonic_double_shared_cluster(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_double_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_shared_cluster_param_0];
+; CHECK-NEXT: ld.relaxed.cluster.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr addrspace(3) %addr syncscope("cluster") monotonic, align 8
+ ret double %retval
+}
+
+define double @monotonic_double_shared_gpu(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_double_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_shared_gpu_param_0];
+; CHECK-NEXT: ld.relaxed.gpu.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr addrspace(3) %addr syncscope("device") monotonic, align 8
+ ret double %retval
+}
+
+define double @monotonic_double_shared_sys_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_double_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr addrspace(3) %addr syncscope("") monotonic, align 8
+ ret double %retval
+}
+
+define double @monotonic_double_shared_cta_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_double_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr addrspace(3) %addr syncscope("block") monotonic, align 8
+ ret double %retval
+}
+
+define double @monotonic_double_shared_cluster_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_double_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr addrspace(3) %addr syncscope("cluster") monotonic, align 8
+ ret double %retval
+}
+
+define double @monotonic_double_shared_gpu_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_double_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.volatile.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr addrspace(3) %addr syncscope("device") monotonic, align 8
+ ret double %retval
+}
+
+define i8 @monotonic_i8_local_sys() {
+; CHECK-LABEL: monotonic_i8_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot612[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot612;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b8 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i8, ptr addrspace(5) %slot syncscope("") monotonic, align 1
+ ret i8 %retval
+}
+
+define i8 @monotonic_i8_local_cta() {
+; CHECK-LABEL: monotonic_i8_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot613[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot613;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b8 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i8, ptr addrspace(5) %slot syncscope("block") monotonic, align 1
+ ret i8 %retval
+}
+
+define i8 @monotonic_i8_local_cluster() {
+; CHECK-LABEL: monotonic_i8_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot614[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot614;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b8 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i8, ptr addrspace(5) %slot syncscope("cluster") monotonic, align 1
+ ret i8 %retval
+}
+
+define i8 @monotonic_i8_local_gpu() {
+; CHECK-LABEL: monotonic_i8_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot615[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot615;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b8 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i8, ptr addrspace(5) %slot syncscope("device") monotonic, align 1
+ ret i8 %retval
+}
+
+define i8 @monotonic_i8_local_sys_volatile() {
+; CHECK-LABEL: monotonic_i8_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot616[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot616;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b8 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i8, ptr addrspace(5) %slot syncscope("") monotonic, align 1
+ ret i8 %retval
+}
+
+define i8 @monotonic_i8_local_cta_volatile() {
+; CHECK-LABEL: monotonic_i8_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot617[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot617;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b8 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i8, ptr addrspace(5) %slot syncscope("block") monotonic, align 1
+ ret i8 %retval
+}
+
+define i8 @monotonic_i8_local_cluster_volatile() {
+; CHECK-LABEL: monotonic_i8_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot618[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot618;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b8 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i8, ptr addrspace(5) %slot syncscope("cluster") monotonic, align 1
+ ret i8 %retval
+}
+
+define i8 @monotonic_i8_local_gpu_volatile() {
+; CHECK-LABEL: monotonic_i8_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot619[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot619;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b8 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i8, ptr addrspace(5) %slot syncscope("device") monotonic, align 1
+ ret i8 %retval
+}
+
+define i16 @monotonic_i16_local_sys() {
+; CHECK-LABEL: monotonic_i16_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot620[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot620;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i16, ptr addrspace(5) %slot syncscope("") monotonic, align 2
+ ret i16 %retval
+}
+
+define i16 @monotonic_i16_local_cta() {
+; CHECK-LABEL: monotonic_i16_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot621[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot621;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i16, ptr addrspace(5) %slot syncscope("block") monotonic, align 2
+ ret i16 %retval
+}
+
+define i16 @monotonic_i16_local_cluster() {
+; CHECK-LABEL: monotonic_i16_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot622[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot622;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i16, ptr addrspace(5) %slot syncscope("cluster") monotonic, align 2
+ ret i16 %retval
+}
+
+define i16 @monotonic_i16_local_gpu() {
+; CHECK-LABEL: monotonic_i16_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot623[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot623;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i16, ptr addrspace(5) %slot syncscope("device") monotonic, align 2
+ ret i16 %retval
+}
+
+define i16 @monotonic_i16_local_sys_volatile() {
+; CHECK-LABEL: monotonic_i16_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot624[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot624;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i16, ptr addrspace(5) %slot syncscope("") monotonic, align 2
+ ret i16 %retval
+}
+
+define i16 @monotonic_i16_local_cta_volatile() {
+; CHECK-LABEL: monotonic_i16_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot625[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot625;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i16, ptr addrspace(5) %slot syncscope("block") monotonic, align 2
+ ret i16 %retval
+}
+
+define i16 @monotonic_i16_local_cluster_volatile() {
+; CHECK-LABEL: monotonic_i16_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot626[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot626;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i16, ptr addrspace(5) %slot syncscope("cluster") monotonic, align 2
+ ret i16 %retval
+}
+
+define i16 @monotonic_i16_local_gpu_volatile() {
+; CHECK-LABEL: monotonic_i16_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot627[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot627;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i16, ptr addrspace(5) %slot syncscope("device") monotonic, align 2
+ ret i16 %retval
+}
+
+define i32 @monotonic_i32_local_sys() {
+; CHECK-LABEL: monotonic_i32_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot628[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot628;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i32, ptr addrspace(5) %slot syncscope("") monotonic, align 4
+ ret i32 %retval
+}
+
+define i32 @monotonic_i32_local_cta() {
+; CHECK-LABEL: monotonic_i32_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot629[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot629;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i32, ptr addrspace(5) %slot syncscope("block") monotonic, align 4
+ ret i32 %retval
+}
+
+define i32 @monotonic_i32_local_cluster() {
+; CHECK-LABEL: monotonic_i32_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot630[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot630;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i32, ptr addrspace(5) %slot syncscope("cluster") monotonic, align 4
+ ret i32 %retval
+}
+
+define i32 @monotonic_i32_local_gpu() {
+; CHECK-LABEL: monotonic_i32_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot631[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot631;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i32, ptr addrspace(5) %slot syncscope("device") monotonic, align 4
+ ret i32 %retval
+}
+
+define i32 @monotonic_i32_local_sys_volatile() {
+; CHECK-LABEL: monotonic_i32_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot632[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot632;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i32, ptr addrspace(5) %slot syncscope("") monotonic, align 4
+ ret i32 %retval
+}
+
+define i32 @monotonic_i32_local_cta_volatile() {
+; CHECK-LABEL: monotonic_i32_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot633[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot633;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i32, ptr addrspace(5) %slot syncscope("block") monotonic, align 4
+ ret i32 %retval
+}
+
+define i32 @monotonic_i32_local_cluster_volatile() {
+; CHECK-LABEL: monotonic_i32_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot634[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot634;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i32, ptr addrspace(5) %slot syncscope("cluster") monotonic, align 4
+ ret i32 %retval
+}
+
+define i32 @monotonic_i32_local_gpu_volatile() {
+; CHECK-LABEL: monotonic_i32_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot635[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot635;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i32, ptr addrspace(5) %slot syncscope("device") monotonic, align 4
+ ret i32 %retval
+}
+
+define i64 @monotonic_i64_local_sys() {
+; CHECK-LABEL: monotonic_i64_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot636[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot636;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i64, ptr addrspace(5) %slot syncscope("") monotonic, align 8
+ ret i64 %retval
+}
+
+define i64 @monotonic_i64_local_cta() {
+; CHECK-LABEL: monotonic_i64_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot637[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot637;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i64, ptr addrspace(5) %slot syncscope("block") monotonic, align 8
+ ret i64 %retval
+}
+
+define i64 @monotonic_i64_local_cluster() {
+; CHECK-LABEL: monotonic_i64_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot638[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot638;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i64, ptr addrspace(5) %slot syncscope("cluster") monotonic, align 8
+ ret i64 %retval
+}
+
+define i64 @monotonic_i64_local_gpu() {
+; CHECK-LABEL: monotonic_i64_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot639[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot639;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i64, ptr addrspace(5) %slot syncscope("device") monotonic, align 8
+ ret i64 %retval
+}
+
+define i64 @monotonic_i64_local_sys_volatile() {
+; CHECK-LABEL: monotonic_i64_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot640[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot640;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i64, ptr addrspace(5) %slot syncscope("") monotonic, align 8
+ ret i64 %retval
+}
+
+define i64 @monotonic_i64_local_cta_volatile() {
+; CHECK-LABEL: monotonic_i64_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot641[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot641;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i64, ptr addrspace(5) %slot syncscope("block") monotonic, align 8
+ ret i64 %retval
+}
+
+define i64 @monotonic_i64_local_cluster_volatile() {
+; CHECK-LABEL: monotonic_i64_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot642[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot642;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i64, ptr addrspace(5) %slot syncscope("cluster") monotonic, align 8
+ ret i64 %retval
+}
+
+define i64 @monotonic_i64_local_gpu_volatile() {
+; CHECK-LABEL: monotonic_i64_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot643[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot643;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i64, ptr addrspace(5) %slot syncscope("device") monotonic, align 8
+ ret i64 %retval
+}
+
+define i128 @monotonic_i128_local_sys() {
+; CHECK-LABEL: monotonic_i128_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 16 .b8 __local_depot644[16];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot644;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.v2.b64 {%rd2, %rd3}, [%SP];
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd2, %rd3};
+; CHECK-NEXT: ret;
+ %slot = alloca i128, align 16, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i128, ptr addrspace(5) %slot syncscope("") monotonic, align 16
+ ret i128 %retval
+}
+
+define i128 @monotonic_i128_local_cta() {
+; CHECK-LABEL: monotonic_i128_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 16 .b8 __local_depot645[16];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot645;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.v2.b64 {%rd2, %rd3}, [%SP];
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd2, %rd3};
+; CHECK-NEXT: ret;
+ %slot = alloca i128, align 16, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i128, ptr addrspace(5) %slot syncscope("block") monotonic, align 16
+ ret i128 %retval
+}
+
+define i128 @monotonic_i128_local_cluster() {
+; CHECK-LABEL: monotonic_i128_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 16 .b8 __local_depot646[16];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot646;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.v2.b64 {%rd2, %rd3}, [%SP];
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd2, %rd3};
+; CHECK-NEXT: ret;
+ %slot = alloca i128, align 16, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i128, ptr addrspace(5) %slot syncscope("cluster") monotonic, align 16
+ ret i128 %retval
+}
+
+define i128 @monotonic_i128_local_gpu() {
+; CHECK-LABEL: monotonic_i128_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 16 .b8 __local_depot647[16];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot647;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.v2.b64 {%rd2, %rd3}, [%SP];
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd2, %rd3};
+; CHECK-NEXT: ret;
+ %slot = alloca i128, align 16, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i128, ptr addrspace(5) %slot syncscope("device") monotonic, align 16
+ ret i128 %retval
+}
+
+define i128 @monotonic_i128_local_sys_volatile() {
+; CHECK-LABEL: monotonic_i128_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 16 .b8 __local_depot648[16];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot648;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.v2.b64 {%rd2, %rd3}, [%SP];
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd2, %rd3};
+; CHECK-NEXT: ret;
+ %slot = alloca i128, align 16, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i128, ptr addrspace(5) %slot syncscope("") monotonic, align 16
+ ret i128 %retval
+}
+
+define i128 @monotonic_i128_local_cta_volatile() {
+; CHECK-LABEL: monotonic_i128_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 16 .b8 __local_depot649[16];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot649;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.v2.b64 {%rd2, %rd3}, [%SP];
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd2, %rd3};
+; CHECK-NEXT: ret;
+ %slot = alloca i128, align 16, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i128, ptr addrspace(5) %slot syncscope("block") monotonic, align 16
+ ret i128 %retval
+}
+
+define i128 @monotonic_i128_local_cluster_volatile() {
+; CHECK-LABEL: monotonic_i128_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 16 .b8 __local_depot650[16];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot650;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.v2.b64 {%rd2, %rd3}, [%SP];
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd2, %rd3};
+; CHECK-NEXT: ret;
+ %slot = alloca i128, align 16, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i128, ptr addrspace(5) %slot syncscope("cluster") monotonic, align 16
+ ret i128 %retval
+}
+
+define i128 @monotonic_i128_local_gpu_volatile() {
+; CHECK-LABEL: monotonic_i128_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 16 .b8 __local_depot651[16];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot651;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.v2.b64 {%rd2, %rd3}, [%SP];
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd2, %rd3};
+; CHECK-NEXT: ret;
+ %slot = alloca i128, align 16, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i128, ptr addrspace(5) %slot syncscope("device") monotonic, align 16
+ ret i128 %retval
+}
+
+define half @monotonic_half_local_sys() {
+; CHECK-LABEL: monotonic_half_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot652[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot652;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic half, ptr addrspace(5) %slot syncscope("") monotonic, align 2
+ ret half %retval
+}
+
+define half @monotonic_half_local_cta() {
+; CHECK-LABEL: monotonic_half_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot653[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot653;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic half, ptr addrspace(5) %slot syncscope("block") monotonic, align 2
+ ret half %retval
+}
+
+define half @monotonic_half_local_cluster() {
+; CHECK-LABEL: monotonic_half_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot654[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot654;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic half, ptr addrspace(5) %slot syncscope("cluster") monotonic, align 2
+ ret half %retval
+}
+
+define half @monotonic_half_local_gpu() {
+; CHECK-LABEL: monotonic_half_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot655[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot655;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic half, ptr addrspace(5) %slot syncscope("device") monotonic, align 2
+ ret half %retval
+}
+
+define half @monotonic_half_local_sys_volatile() {
+; CHECK-LABEL: monotonic_half_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot656[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot656;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile half, ptr addrspace(5) %slot syncscope("") monotonic, align 2
+ ret half %retval
+}
+
+define half @monotonic_half_local_cta_volatile() {
+; CHECK-LABEL: monotonic_half_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot657[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot657;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile half, ptr addrspace(5) %slot syncscope("block") monotonic, align 2
+ ret half %retval
+}
+
+define half @monotonic_half_local_cluster_volatile() {
+; CHECK-LABEL: monotonic_half_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot658[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot658;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile half, ptr addrspace(5) %slot syncscope("cluster") monotonic, align 2
+ ret half %retval
+}
+
+define half @monotonic_half_local_gpu_volatile() {
+; CHECK-LABEL: monotonic_half_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot659[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot659;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile half, ptr addrspace(5) %slot syncscope("device") monotonic, align 2
+ ret half %retval
+}
+
+define bfloat @monotonic_bfloat_local_sys() {
+; CHECK-LABEL: monotonic_bfloat_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot660[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot660;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic bfloat, ptr addrspace(5) %slot syncscope("") monotonic, align 2
+ ret bfloat %retval
+}
+
+define bfloat @monotonic_bfloat_local_cta() {
+; CHECK-LABEL: monotonic_bfloat_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot661[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot661;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic bfloat, ptr addrspace(5) %slot syncscope("block") monotonic, align 2
+ ret bfloat %retval
+}
+
+define bfloat @monotonic_bfloat_local_cluster() {
+; CHECK-LABEL: monotonic_bfloat_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot662[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot662;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic bfloat, ptr addrspace(5) %slot syncscope("cluster") monotonic, align 2
+ ret bfloat %retval
+}
+
+define bfloat @monotonic_bfloat_local_gpu() {
+; CHECK-LABEL: monotonic_bfloat_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot663[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot663;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic bfloat, ptr addrspace(5) %slot syncscope("device") monotonic, align 2
+ ret bfloat %retval
+}
+
+define bfloat @monotonic_bfloat_local_sys_volatile() {
+; CHECK-LABEL: monotonic_bfloat_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot664[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot664;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile bfloat, ptr addrspace(5) %slot syncscope("") monotonic, align 2
+ ret bfloat %retval
+}
+
+define bfloat @monotonic_bfloat_local_cta_volatile() {
+; CHECK-LABEL: monotonic_bfloat_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot665[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot665;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile bfloat, ptr addrspace(5) %slot syncscope("block") monotonic, align 2
+ ret bfloat %retval
+}
+
+define bfloat @monotonic_bfloat_local_cluster_volatile() {
+; CHECK-LABEL: monotonic_bfloat_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot666[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot666;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile bfloat, ptr addrspace(5) %slot syncscope("cluster") monotonic, align 2
+ ret bfloat %retval
+}
+
+define bfloat @monotonic_bfloat_local_gpu_volatile() {
+; CHECK-LABEL: monotonic_bfloat_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot667[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot667;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile bfloat, ptr addrspace(5) %slot syncscope("device") monotonic, align 2
+ ret bfloat %retval
+}
+
+define float @monotonic_float_local_sys() {
+; CHECK-LABEL: monotonic_float_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot668[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot668;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic float, ptr addrspace(5) %slot syncscope("") monotonic, align 4
+ ret float %retval
+}
+
+define float @monotonic_float_local_cta() {
+; CHECK-LABEL: monotonic_float_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot669[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot669;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic float, ptr addrspace(5) %slot syncscope("block") monotonic, align 4
+ ret float %retval
+}
+
+define float @monotonic_float_local_cluster() {
+; CHECK-LABEL: monotonic_float_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot670[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot670;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic float, ptr addrspace(5) %slot syncscope("cluster") monotonic, align 4
+ ret float %retval
+}
+
+define float @monotonic_float_local_gpu() {
+; CHECK-LABEL: monotonic_float_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot671[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot671;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic float, ptr addrspace(5) %slot syncscope("device") monotonic, align 4
+ ret float %retval
+}
+
+define float @monotonic_float_local_sys_volatile() {
+; CHECK-LABEL: monotonic_float_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot672[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot672;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile float, ptr addrspace(5) %slot syncscope("") monotonic, align 4
+ ret float %retval
+}
+
+define float @monotonic_float_local_cta_volatile() {
+; CHECK-LABEL: monotonic_float_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot673[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot673;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile float, ptr addrspace(5) %slot syncscope("block") monotonic, align 4
+ ret float %retval
+}
+
+define float @monotonic_float_local_cluster_volatile() {
+; CHECK-LABEL: monotonic_float_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot674[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot674;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile float, ptr addrspace(5) %slot syncscope("cluster") monotonic, align 4
+ ret float %retval
+}
+
+define float @monotonic_float_local_gpu_volatile() {
+; CHECK-LABEL: monotonic_float_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot675[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot675;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile float, ptr addrspace(5) %slot syncscope("device") monotonic, align 4
+ ret float %retval
+}
+
+define double @monotonic_double_local_sys() {
+; CHECK-LABEL: monotonic_double_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot676[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot676;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic double, ptr addrspace(5) %slot syncscope("") monotonic, align 8
+ ret double %retval
+}
+
+define double @monotonic_double_local_cta() {
+; CHECK-LABEL: monotonic_double_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot677[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot677;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic double, ptr addrspace(5) %slot syncscope("block") monotonic, align 8
+ ret double %retval
+}
+
+define double @monotonic_double_local_cluster() {
+; CHECK-LABEL: monotonic_double_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot678[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot678;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic double, ptr addrspace(5) %slot syncscope("cluster") monotonic, align 8
+ ret double %retval
+}
+
+define double @monotonic_double_local_gpu() {
+; CHECK-LABEL: monotonic_double_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot679[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot679;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic double, ptr addrspace(5) %slot syncscope("device") monotonic, align 8
+ ret double %retval
+}
+
+define double @monotonic_double_local_sys_volatile() {
+; CHECK-LABEL: monotonic_double_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot680[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot680;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile double, ptr addrspace(5) %slot syncscope("") monotonic, align 8
+ ret double %retval
+}
+
+define double @monotonic_double_local_cta_volatile() {
+; CHECK-LABEL: monotonic_double_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot681[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot681;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile double, ptr addrspace(5) %slot syncscope("block") monotonic, align 8
+ ret double %retval
+}
+
+define double @monotonic_double_local_cluster_volatile() {
+; CHECK-LABEL: monotonic_double_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot682[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot682;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile double, ptr addrspace(5) %slot syncscope("cluster") monotonic, align 8
+ ret double %retval
+}
+
+define double @monotonic_double_local_gpu_volatile() {
+; CHECK-LABEL: monotonic_double_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot683[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot683;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile double, ptr addrspace(5) %slot syncscope("device") monotonic, align 8
+ ret double %retval
+}
+
+define i8 @acquire_i8_generic_sys(ptr %addr) {
+; CHECK-LABEL: acquire_i8_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i8_generic_sys_param_0];
+; CHECK-NEXT: ld.acquire.sys.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr %addr syncscope("") acquire, align 1
+ ret i8 %retval
+}
+
+define i8 @acquire_i8_generic_cta(ptr %addr) {
+; CHECK-LABEL: acquire_i8_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i8_generic_cta_param_0];
+; CHECK-NEXT: ld.acquire.cta.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr %addr syncscope("block") acquire, align 1
+ ret i8 %retval
+}
+
+define i8 @acquire_i8_generic_cluster(ptr %addr) {
+; CHECK-LABEL: acquire_i8_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i8_generic_cluster_param_0];
+; CHECK-NEXT: ld.acquire.cluster.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr %addr syncscope("cluster") acquire, align 1
+ ret i8 %retval
+}
+
+define i8 @acquire_i8_generic_gpu(ptr %addr) {
+; CHECK-LABEL: acquire_i8_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i8_generic_gpu_param_0];
+; CHECK-NEXT: ld.acquire.gpu.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr %addr syncscope("device") acquire, align 1
+ ret i8 %retval
+}
+
+define i8 @acquire_i8_generic_sys_volatile(ptr %addr) {
+; CHECK-LABEL: acquire_i8_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i8_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr %addr syncscope("") acquire, align 1
+ ret i8 %retval
+}
+
+define i8 @acquire_i8_generic_cta_volatile(ptr %addr) {
+; CHECK-LABEL: acquire_i8_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i8_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr %addr syncscope("block") acquire, align 1
+ ret i8 %retval
+}
+
+define i8 @acquire_i8_generic_cluster_volatile(ptr %addr) {
+; CHECK-LABEL: acquire_i8_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i8_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr %addr syncscope("cluster") acquire, align 1
+ ret i8 %retval
+}
+
+define i8 @acquire_i8_generic_gpu_volatile(ptr %addr) {
+; CHECK-LABEL: acquire_i8_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i8_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr %addr syncscope("device") acquire, align 1
+ ret i8 %retval
+}
+
+define i16 @acquire_i16_generic_sys(ptr %addr) {
+; CHECK-LABEL: acquire_i16_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i16_generic_sys_param_0];
+; CHECK-NEXT: ld.acquire.sys.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr %addr syncscope("") acquire, align 2
+ ret i16 %retval
+}
+
+define i16 @acquire_i16_generic_cta(ptr %addr) {
+; CHECK-LABEL: acquire_i16_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i16_generic_cta_param_0];
+; CHECK-NEXT: ld.acquire.cta.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr %addr syncscope("block") acquire, align 2
+ ret i16 %retval
+}
+
+define i16 @acquire_i16_generic_cluster(ptr %addr) {
+; CHECK-LABEL: acquire_i16_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i16_generic_cluster_param_0];
+; CHECK-NEXT: ld.acquire.cluster.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr %addr syncscope("cluster") acquire, align 2
+ ret i16 %retval
+}
+
+define i16 @acquire_i16_generic_gpu(ptr %addr) {
+; CHECK-LABEL: acquire_i16_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i16_generic_gpu_param_0];
+; CHECK-NEXT: ld.acquire.gpu.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr %addr syncscope("device") acquire, align 2
+ ret i16 %retval
+}
+
+define i16 @acquire_i16_generic_sys_volatile(ptr %addr) {
+; CHECK-LABEL: acquire_i16_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i16_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr %addr syncscope("") acquire, align 2
+ ret i16 %retval
+}
+
+define i16 @acquire_i16_generic_cta_volatile(ptr %addr) {
+; CHECK-LABEL: acquire_i16_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i16_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr %addr syncscope("block") acquire, align 2
+ ret i16 %retval
+}
+
+define i16 @acquire_i16_generic_cluster_volatile(ptr %addr) {
+; CHECK-LABEL: acquire_i16_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i16_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr %addr syncscope("cluster") acquire, align 2
+ ret i16 %retval
+}
+
+define i16 @acquire_i16_generic_gpu_volatile(ptr %addr) {
+; CHECK-LABEL: acquire_i16_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i16_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr %addr syncscope("device") acquire, align 2
+ ret i16 %retval
+}
+
+define i32 @acquire_i32_generic_sys(ptr %addr) {
+; CHECK-LABEL: acquire_i32_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i32_generic_sys_param_0];
+; CHECK-NEXT: ld.acquire.sys.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr %addr syncscope("") acquire, align 4
+ ret i32 %retval
+}
+
+define i32 @acquire_i32_generic_cta(ptr %addr) {
+; CHECK-LABEL: acquire_i32_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i32_generic_cta_param_0];
+; CHECK-NEXT: ld.acquire.cta.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr %addr syncscope("block") acquire, align 4
+ ret i32 %retval
+}
+
+define i32 @acquire_i32_generic_cluster(ptr %addr) {
+; CHECK-LABEL: acquire_i32_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i32_generic_cluster_param_0];
+; CHECK-NEXT: ld.acquire.cluster.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr %addr syncscope("cluster") acquire, align 4
+ ret i32 %retval
+}
+
+define i32 @acquire_i32_generic_gpu(ptr %addr) {
+; CHECK-LABEL: acquire_i32_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i32_generic_gpu_param_0];
+; CHECK-NEXT: ld.acquire.gpu.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr %addr syncscope("device") acquire, align 4
+ ret i32 %retval
+}
+
+define i32 @acquire_i32_generic_sys_volatile(ptr %addr) {
+; CHECK-LABEL: acquire_i32_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i32_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr %addr syncscope("") acquire, align 4
+ ret i32 %retval
+}
+
+define i32 @acquire_i32_generic_cta_volatile(ptr %addr) {
+; CHECK-LABEL: acquire_i32_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i32_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr %addr syncscope("block") acquire, align 4
+ ret i32 %retval
+}
+
+define i32 @acquire_i32_generic_cluster_volatile(ptr %addr) {
+; CHECK-LABEL: acquire_i32_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i32_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr %addr syncscope("cluster") acquire, align 4
+ ret i32 %retval
+}
+
+define i32 @acquire_i32_generic_gpu_volatile(ptr %addr) {
+; CHECK-LABEL: acquire_i32_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i32_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr %addr syncscope("device") acquire, align 4
+ ret i32 %retval
+}
+
+define i64 @acquire_i64_generic_sys(ptr %addr) {
+; CHECK-LABEL: acquire_i64_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i64_generic_sys_param_0];
+; CHECK-NEXT: ld.acquire.sys.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr %addr syncscope("") acquire, align 8
+ ret i64 %retval
+}
+
+define i64 @acquire_i64_generic_cta(ptr %addr) {
+; CHECK-LABEL: acquire_i64_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i64_generic_cta_param_0];
+; CHECK-NEXT: ld.acquire.cta.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr %addr syncscope("block") acquire, align 8
+ ret i64 %retval
+}
+
+define i64 @acquire_i64_generic_cluster(ptr %addr) {
+; CHECK-LABEL: acquire_i64_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i64_generic_cluster_param_0];
+; CHECK-NEXT: ld.acquire.cluster.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr %addr syncscope("cluster") acquire, align 8
+ ret i64 %retval
+}
+
+define i64 @acquire_i64_generic_gpu(ptr %addr) {
+; CHECK-LABEL: acquire_i64_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i64_generic_gpu_param_0];
+; CHECK-NEXT: ld.acquire.gpu.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr %addr syncscope("device") acquire, align 8
+ ret i64 %retval
+}
+
+define i64 @acquire_i64_generic_sys_volatile(ptr %addr) {
+; CHECK-LABEL: acquire_i64_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i64_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr %addr syncscope("") acquire, align 8
+ ret i64 %retval
+}
+
+define i64 @acquire_i64_generic_cta_volatile(ptr %addr) {
+; CHECK-LABEL: acquire_i64_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i64_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr %addr syncscope("block") acquire, align 8
+ ret i64 %retval
+}
+
+define i64 @acquire_i64_generic_cluster_volatile(ptr %addr) {
+; CHECK-LABEL: acquire_i64_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i64_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr %addr syncscope("cluster") acquire, align 8
+ ret i64 %retval
+}
+
+define i64 @acquire_i64_generic_gpu_volatile(ptr %addr) {
+; CHECK-LABEL: acquire_i64_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i64_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr %addr syncscope("device") acquire, align 8
+ ret i64 %retval
+}
+
+define i128 @acquire_i128_generic_sys(ptr %addr) {
+; CHECK-LABEL: acquire_i128_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i128_generic_sys_param_0];
+; CHECK-NEXT: mov.b64 %rd2, 0;
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 cmp, swap, dst;
+; CHECK-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; CHECK-NEXT: mov.b128 swap, {%rd2, %rd2};
+; CHECK-NEXT: atom.acquire.sys.cas.b128 dst, [%rd1], cmp, swap;
+; CHECK-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; CHECK-NEXT: ret;
+ %retval = load atomic i128, ptr %addr syncscope("") acquire, align 16
+ ret i128 %retval
+}
+
+define i128 @acquire_i128_generic_cta(ptr %addr) {
+; CHECK-LABEL: acquire_i128_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i128_generic_cta_param_0];
+; CHECK-NEXT: mov.b64 %rd2, 0;
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 cmp, swap, dst;
+; CHECK-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; CHECK-NEXT: mov.b128 swap, {%rd2, %rd2};
+; CHECK-NEXT: atom.acquire.cta.cas.b128 dst, [%rd1], cmp, swap;
+; CHECK-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; CHECK-NEXT: ret;
+ %retval = load atomic i128, ptr %addr syncscope("block") acquire, align 16
+ ret i128 %retval
+}
+
+define i128 @acquire_i128_generic_cluster(ptr %addr) {
+; CHECK-LABEL: acquire_i128_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i128_generic_cluster_param_0];
+; CHECK-NEXT: mov.b64 %rd2, 0;
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 cmp, swap, dst;
+; CHECK-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; CHECK-NEXT: mov.b128 swap, {%rd2, %rd2};
+; CHECK-NEXT: atom.acquire.cluster.cas.b128 dst, [%rd1], cmp, swap;
+; CHECK-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; CHECK-NEXT: ret;
+ %retval = load atomic i128, ptr %addr syncscope("cluster") acquire, align 16
+ ret i128 %retval
+}
+
+define i128 @acquire_i128_generic_gpu(ptr %addr) {
+; CHECK-LABEL: acquire_i128_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i128_generic_gpu_param_0];
+; CHECK-NEXT: mov.b64 %rd2, 0;
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 cmp, swap, dst;
+; CHECK-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; CHECK-NEXT: mov.b128 swap, {%rd2, %rd2};
+; CHECK-NEXT: atom.acquire.gpu.cas.b128 dst, [%rd1], cmp, swap;
+; CHECK-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; CHECK-NEXT: ret;
+ %retval = load atomic i128, ptr %addr syncscope("device") acquire, align 16
+ ret i128 %retval
+}
+
+define i128 @acquire_i128_generic_sys_volatile(ptr %addr) {
+; CHECK-LABEL: acquire_i128_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i128_generic_sys_volatile_param_0];
+; CHECK-NEXT: mov.b64 %rd2, 0;
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 cmp, swap, dst;
+; CHECK-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; CHECK-NEXT: mov.b128 swap, {%rd2, %rd2};
+; CHECK-NEXT: atom.acquire.sys.cas.b128 dst, [%rd1], cmp, swap;
+; CHECK-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i128, ptr %addr syncscope("") acquire, align 16
+ ret i128 %retval
+}
+
+define i128 @acquire_i128_generic_cta_volatile(ptr %addr) {
+; CHECK-LABEL: acquire_i128_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i128_generic_cta_volatile_param_0];
+; CHECK-NEXT: mov.b64 %rd2, 0;
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 cmp, swap, dst;
+; CHECK-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; CHECK-NEXT: mov.b128 swap, {%rd2, %rd2};
+; CHECK-NEXT: atom.acquire.cta.cas.b128 dst, [%rd1], cmp, swap;
+; CHECK-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i128, ptr %addr syncscope("block") acquire, align 16
+ ret i128 %retval
+}
+
+define i128 @acquire_i128_generic_cluster_volatile(ptr %addr) {
+; CHECK-LABEL: acquire_i128_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i128_generic_cluster_volatile_param_0];
+; CHECK-NEXT: mov.b64 %rd2, 0;
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 cmp, swap, dst;
+; CHECK-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; CHECK-NEXT: mov.b128 swap, {%rd2, %rd2};
+; CHECK-NEXT: atom.acquire.cluster.cas.b128 dst, [%rd1], cmp, swap;
+; CHECK-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i128, ptr %addr syncscope("cluster") acquire, align 16
+ ret i128 %retval
+}
+
+define i128 @acquire_i128_generic_gpu_volatile(ptr %addr) {
+; CHECK-LABEL: acquire_i128_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i128_generic_gpu_volatile_param_0];
+; CHECK-NEXT: mov.b64 %rd2, 0;
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 cmp, swap, dst;
+; CHECK-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; CHECK-NEXT: mov.b128 swap, {%rd2, %rd2};
+; CHECK-NEXT: atom.acquire.gpu.cas.b128 dst, [%rd1], cmp, swap;
+; CHECK-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i128, ptr %addr syncscope("device") acquire, align 16
+ ret i128 %retval
+}
+
+define half @acquire_half_generic_sys(ptr %addr) {
+; CHECK-LABEL: acquire_half_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_half_generic_sys_param_0];
+; CHECK-NEXT: ld.acquire.sys.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr %addr syncscope("") acquire, align 2
+ ret half %retval
+}
+
+define half @acquire_half_generic_cta(ptr %addr) {
+; CHECK-LABEL: acquire_half_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_half_generic_cta_param_0];
+; CHECK-NEXT: ld.acquire.cta.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr %addr syncscope("block") acquire, align 2
+ ret half %retval
+}
+
+define half @acquire_half_generic_cluster(ptr %addr) {
+; CHECK-LABEL: acquire_half_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_half_generic_cluster_param_0];
+; CHECK-NEXT: ld.acquire.cluster.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr %addr syncscope("cluster") acquire, align 2
+ ret half %retval
+}
+
+define half @acquire_half_generic_gpu(ptr %addr) {
+; CHECK-LABEL: acquire_half_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_half_generic_gpu_param_0];
+; CHECK-NEXT: ld.acquire.gpu.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr %addr syncscope("device") acquire, align 2
+ ret half %retval
+}
+
+define half @acquire_half_generic_sys_volatile(ptr %addr) {
+; CHECK-LABEL: acquire_half_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_half_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr %addr syncscope("") acquire, align 2
+ ret half %retval
+}
+
+define half @acquire_half_generic_cta_volatile(ptr %addr) {
+; CHECK-LABEL: acquire_half_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_half_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr %addr syncscope("block") acquire, align 2
+ ret half %retval
+}
+
+define half @acquire_half_generic_cluster_volatile(ptr %addr) {
+; CHECK-LABEL: acquire_half_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_half_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr %addr syncscope("cluster") acquire, align 2
+ ret half %retval
+}
+
+define half @acquire_half_generic_gpu_volatile(ptr %addr) {
+; CHECK-LABEL: acquire_half_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_half_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr %addr syncscope("device") acquire, align 2
+ ret half %retval
+}
+
+define bfloat @acquire_bfloat_generic_sys(ptr %addr) {
+; CHECK-LABEL: acquire_bfloat_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_bfloat_generic_sys_param_0];
+; CHECK-NEXT: ld.acquire.sys.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr %addr syncscope("") acquire, align 2
+ ret bfloat %retval
+}
+
+define bfloat @acquire_bfloat_generic_cta(ptr %addr) {
+; CHECK-LABEL: acquire_bfloat_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_bfloat_generic_cta_param_0];
+; CHECK-NEXT: ld.acquire.cta.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr %addr syncscope("block") acquire, align 2
+ ret bfloat %retval
+}
+
+define bfloat @acquire_bfloat_generic_cluster(ptr %addr) {
+; CHECK-LABEL: acquire_bfloat_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_bfloat_generic_cluster_param_0];
+; CHECK-NEXT: ld.acquire.cluster.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr %addr syncscope("cluster") acquire, align 2
+ ret bfloat %retval
+}
+
+define bfloat @acquire_bfloat_generic_gpu(ptr %addr) {
+; CHECK-LABEL: acquire_bfloat_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_bfloat_generic_gpu_param_0];
+; CHECK-NEXT: ld.acquire.gpu.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr %addr syncscope("device") acquire, align 2
+ ret bfloat %retval
+}
+
+define bfloat @acquire_bfloat_generic_sys_volatile(ptr %addr) {
+; CHECK-LABEL: acquire_bfloat_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_bfloat_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr %addr syncscope("") acquire, align 2
+ ret bfloat %retval
+}
+
+define bfloat @acquire_bfloat_generic_cta_volatile(ptr %addr) {
+; CHECK-LABEL: acquire_bfloat_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_bfloat_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr %addr syncscope("block") acquire, align 2
+ ret bfloat %retval
+}
+
+define bfloat @acquire_bfloat_generic_cluster_volatile(ptr %addr) {
+; CHECK-LABEL: acquire_bfloat_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_bfloat_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr %addr syncscope("cluster") acquire, align 2
+ ret bfloat %retval
+}
+
+define bfloat @acquire_bfloat_generic_gpu_volatile(ptr %addr) {
+; CHECK-LABEL: acquire_bfloat_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_bfloat_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr %addr syncscope("device") acquire, align 2
+ ret bfloat %retval
+}
+
+define float @acquire_float_generic_sys(ptr %addr) {
+; CHECK-LABEL: acquire_float_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_float_generic_sys_param_0];
+; CHECK-NEXT: ld.acquire.sys.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr %addr syncscope("") acquire, align 4
+ ret float %retval
+}
+
+define float @acquire_float_generic_cta(ptr %addr) {
+; CHECK-LABEL: acquire_float_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_float_generic_cta_param_0];
+; CHECK-NEXT: ld.acquire.cta.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr %addr syncscope("block") acquire, align 4
+ ret float %retval
+}
+
+define float @acquire_float_generic_cluster(ptr %addr) {
+; CHECK-LABEL: acquire_float_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_float_generic_cluster_param_0];
+; CHECK-NEXT: ld.acquire.cluster.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr %addr syncscope("cluster") acquire, align 4
+ ret float %retval
+}
+
+define float @acquire_float_generic_gpu(ptr %addr) {
+; CHECK-LABEL: acquire_float_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_float_generic_gpu_param_0];
+; CHECK-NEXT: ld.acquire.gpu.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr %addr syncscope("device") acquire, align 4
+ ret float %retval
+}
+
+define float @acquire_float_generic_sys_volatile(ptr %addr) {
+; CHECK-LABEL: acquire_float_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_float_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr %addr syncscope("") acquire, align 4
+ ret float %retval
+}
+
+define float @acquire_float_generic_cta_volatile(ptr %addr) {
+; CHECK-LABEL: acquire_float_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_float_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr %addr syncscope("block") acquire, align 4
+ ret float %retval
+}
+
+define float @acquire_float_generic_cluster_volatile(ptr %addr) {
+; CHECK-LABEL: acquire_float_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_float_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr %addr syncscope("cluster") acquire, align 4
+ ret float %retval
+}
+
+define float @acquire_float_generic_gpu_volatile(ptr %addr) {
+; CHECK-LABEL: acquire_float_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_float_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr %addr syncscope("device") acquire, align 4
+ ret float %retval
+}
+
+define double @acquire_double_generic_sys(ptr %addr) {
+; CHECK-LABEL: acquire_double_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_double_generic_sys_param_0];
+; CHECK-NEXT: ld.acquire.sys.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr %addr syncscope("") acquire, align 8
+ ret double %retval
+}
+
+define double @acquire_double_generic_cta(ptr %addr) {
+; CHECK-LABEL: acquire_double_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_double_generic_cta_param_0];
+; CHECK-NEXT: ld.acquire.cta.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr %addr syncscope("block") acquire, align 8
+ ret double %retval
+}
+
+define double @acquire_double_generic_cluster(ptr %addr) {
+; CHECK-LABEL: acquire_double_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_double_generic_cluster_param_0];
+; CHECK-NEXT: ld.acquire.cluster.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr %addr syncscope("cluster") acquire, align 8
+ ret double %retval
+}
+
+define double @acquire_double_generic_gpu(ptr %addr) {
+; CHECK-LABEL: acquire_double_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_double_generic_gpu_param_0];
+; CHECK-NEXT: ld.acquire.gpu.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr %addr syncscope("device") acquire, align 8
+ ret double %retval
+}
+
+define double @acquire_double_generic_sys_volatile(ptr %addr) {
+; CHECK-LABEL: acquire_double_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_double_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr %addr syncscope("") acquire, align 8
+ ret double %retval
+}
+
+define double @acquire_double_generic_cta_volatile(ptr %addr) {
+; CHECK-LABEL: acquire_double_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_double_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr %addr syncscope("block") acquire, align 8
+ ret double %retval
+}
+
+define double @acquire_double_generic_cluster_volatile(ptr %addr) {
+; CHECK-LABEL: acquire_double_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_double_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr %addr syncscope("cluster") acquire, align 8
+ ret double %retval
+}
+
+define double @acquire_double_generic_gpu_volatile(ptr %addr) {
+; CHECK-LABEL: acquire_double_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_double_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr %addr syncscope("device") acquire, align 8
+ ret double %retval
+}
+
+define i8 @acquire_i8_global_sys(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_i8_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i8_global_sys_param_0];
+; CHECK-NEXT: ld.acquire.sys.global.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr addrspace(1) %addr syncscope("") acquire, align 1
+ ret i8 %retval
+}
+
+define i8 @acquire_i8_global_cta(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_i8_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i8_global_cta_param_0];
+; CHECK-NEXT: ld.acquire.cta.global.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr addrspace(1) %addr syncscope("block") acquire, align 1
+ ret i8 %retval
+}
+
+define i8 @acquire_i8_global_cluster(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_i8_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i8_global_cluster_param_0];
+; CHECK-NEXT: ld.acquire.cluster.global.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr addrspace(1) %addr syncscope("cluster") acquire, align 1
+ ret i8 %retval
+}
+
+define i8 @acquire_i8_global_gpu(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_i8_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i8_global_gpu_param_0];
+; CHECK-NEXT: ld.acquire.gpu.global.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr addrspace(1) %addr syncscope("device") acquire, align 1
+ ret i8 %retval
+}
+
+define i8 @acquire_i8_global_sys_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_i8_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i8_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.global.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr addrspace(1) %addr syncscope("") acquire, align 1
+ ret i8 %retval
+}
+
+define i8 @acquire_i8_global_cta_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_i8_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i8_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.global.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr addrspace(1) %addr syncscope("block") acquire, align 1
+ ret i8 %retval
+}
+
+define i8 @acquire_i8_global_cluster_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_i8_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i8_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.global.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr addrspace(1) %addr syncscope("cluster") acquire, align 1
+ ret i8 %retval
+}
+
+define i8 @acquire_i8_global_gpu_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_i8_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i8_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.global.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr addrspace(1) %addr syncscope("device") acquire, align 1
+ ret i8 %retval
+}
+
+define i16 @acquire_i16_global_sys(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_i16_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i16_global_sys_param_0];
+; CHECK-NEXT: ld.acquire.sys.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr addrspace(1) %addr syncscope("") acquire, align 2
+ ret i16 %retval
+}
+
+define i16 @acquire_i16_global_cta(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_i16_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i16_global_cta_param_0];
+; CHECK-NEXT: ld.acquire.cta.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr addrspace(1) %addr syncscope("block") acquire, align 2
+ ret i16 %retval
+}
+
+define i16 @acquire_i16_global_cluster(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_i16_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i16_global_cluster_param_0];
+; CHECK-NEXT: ld.acquire.cluster.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr addrspace(1) %addr syncscope("cluster") acquire, align 2
+ ret i16 %retval
+}
+
+define i16 @acquire_i16_global_gpu(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_i16_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i16_global_gpu_param_0];
+; CHECK-NEXT: ld.acquire.gpu.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr addrspace(1) %addr syncscope("device") acquire, align 2
+ ret i16 %retval
+}
+
+define i16 @acquire_i16_global_sys_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_i16_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i16_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr addrspace(1) %addr syncscope("") acquire, align 2
+ ret i16 %retval
+}
+
+define i16 @acquire_i16_global_cta_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_i16_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i16_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr addrspace(1) %addr syncscope("block") acquire, align 2
+ ret i16 %retval
+}
+
+define i16 @acquire_i16_global_cluster_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_i16_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i16_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr addrspace(1) %addr syncscope("cluster") acquire, align 2
+ ret i16 %retval
+}
+
+define i16 @acquire_i16_global_gpu_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_i16_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i16_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr addrspace(1) %addr syncscope("device") acquire, align 2
+ ret i16 %retval
+}
+
+define i32 @acquire_i32_global_sys(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_i32_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i32_global_sys_param_0];
+; CHECK-NEXT: ld.acquire.sys.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr addrspace(1) %addr syncscope("") acquire, align 4
+ ret i32 %retval
+}
+
+define i32 @acquire_i32_global_cta(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_i32_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i32_global_cta_param_0];
+; CHECK-NEXT: ld.acquire.cta.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr addrspace(1) %addr syncscope("block") acquire, align 4
+ ret i32 %retval
+}
+
+define i32 @acquire_i32_global_cluster(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_i32_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i32_global_cluster_param_0];
+; CHECK-NEXT: ld.acquire.cluster.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr addrspace(1) %addr syncscope("cluster") acquire, align 4
+ ret i32 %retval
+}
+
+define i32 @acquire_i32_global_gpu(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_i32_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i32_global_gpu_param_0];
+; CHECK-NEXT: ld.acquire.gpu.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr addrspace(1) %addr syncscope("device") acquire, align 4
+ ret i32 %retval
+}
+
+define i32 @acquire_i32_global_sys_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_i32_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i32_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr addrspace(1) %addr syncscope("") acquire, align 4
+ ret i32 %retval
+}
+
+define i32 @acquire_i32_global_cta_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_i32_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i32_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr addrspace(1) %addr syncscope("block") acquire, align 4
+ ret i32 %retval
+}
+
+define i32 @acquire_i32_global_cluster_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_i32_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i32_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr addrspace(1) %addr syncscope("cluster") acquire, align 4
+ ret i32 %retval
+}
+
+define i32 @acquire_i32_global_gpu_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_i32_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i32_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr addrspace(1) %addr syncscope("device") acquire, align 4
+ ret i32 %retval
+}
+
+define i64 @acquire_i64_global_sys(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_i64_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i64_global_sys_param_0];
+; CHECK-NEXT: ld.acquire.sys.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr addrspace(1) %addr syncscope("") acquire, align 8
+ ret i64 %retval
+}
+
+define i64 @acquire_i64_global_cta(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_i64_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i64_global_cta_param_0];
+; CHECK-NEXT: ld.acquire.cta.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr addrspace(1) %addr syncscope("block") acquire, align 8
+ ret i64 %retval
+}
+
+define i64 @acquire_i64_global_cluster(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_i64_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i64_global_cluster_param_0];
+; CHECK-NEXT: ld.acquire.cluster.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr addrspace(1) %addr syncscope("cluster") acquire, align 8
+ ret i64 %retval
+}
+
+define i64 @acquire_i64_global_gpu(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_i64_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i64_global_gpu_param_0];
+; CHECK-NEXT: ld.acquire.gpu.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr addrspace(1) %addr syncscope("device") acquire, align 8
+ ret i64 %retval
+}
+
+define i64 @acquire_i64_global_sys_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_i64_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i64_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr addrspace(1) %addr syncscope("") acquire, align 8
+ ret i64 %retval
+}
+
+define i64 @acquire_i64_global_cta_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_i64_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i64_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr addrspace(1) %addr syncscope("block") acquire, align 8
+ ret i64 %retval
+}
+
+define i64 @acquire_i64_global_cluster_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_i64_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i64_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr addrspace(1) %addr syncscope("cluster") acquire, align 8
+ ret i64 %retval
+}
+
+define i64 @acquire_i64_global_gpu_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_i64_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i64_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr addrspace(1) %addr syncscope("device") acquire, align 8
+ ret i64 %retval
+}
+
+define i128 @acquire_i128_global_sys(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_i128_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i128_global_sys_param_0];
+; CHECK-NEXT: mov.b64 %rd2, 0;
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 cmp, swap, dst;
+; CHECK-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; CHECK-NEXT: mov.b128 swap, {%rd2, %rd2};
+; CHECK-NEXT: atom.acquire.sys.global.cas.b128 dst, [%rd1], cmp, swap;
+; CHECK-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; CHECK-NEXT: ret;
+ %retval = load atomic i128, ptr addrspace(1) %addr syncscope("") acquire, align 16
+ ret i128 %retval
+}
+
+define i128 @acquire_i128_global_cta(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_i128_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i128_global_cta_param_0];
+; CHECK-NEXT: mov.b64 %rd2, 0;
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 cmp, swap, dst;
+; CHECK-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; CHECK-NEXT: mov.b128 swap, {%rd2, %rd2};
+; CHECK-NEXT: atom.acquire.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; CHECK-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; CHECK-NEXT: ret;
+ %retval = load atomic i128, ptr addrspace(1) %addr syncscope("block") acquire, align 16
+ ret i128 %retval
+}
+
+define i128 @acquire_i128_global_cluster(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_i128_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i128_global_cluster_param_0];
+; CHECK-NEXT: mov.b64 %rd2, 0;
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 cmp, swap, dst;
+; CHECK-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; CHECK-NEXT: mov.b128 swap, {%rd2, %rd2};
+; CHECK-NEXT: atom.acquire.cluster.global.cas.b128 dst, [%rd1], cmp, swap;
+; CHECK-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; CHECK-NEXT: ret;
+ %retval = load atomic i128, ptr addrspace(1) %addr syncscope("cluster") acquire, align 16
+ ret i128 %retval
+}
+
+define i128 @acquire_i128_global_gpu(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_i128_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i128_global_gpu_param_0];
+; CHECK-NEXT: mov.b64 %rd2, 0;
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 cmp, swap, dst;
+; CHECK-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; CHECK-NEXT: mov.b128 swap, {%rd2, %rd2};
+; CHECK-NEXT: atom.acquire.gpu.global.cas.b128 dst, [%rd1], cmp, swap;
+; CHECK-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; CHECK-NEXT: ret;
+ %retval = load atomic i128, ptr addrspace(1) %addr syncscope("device") acquire, align 16
+ ret i128 %retval
+}
+
+define i128 @acquire_i128_global_sys_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_i128_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i128_global_sys_volatile_param_0];
+; CHECK-NEXT: mov.b64 %rd2, 0;
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 cmp, swap, dst;
+; CHECK-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; CHECK-NEXT: mov.b128 swap, {%rd2, %rd2};
+; CHECK-NEXT: atom.acquire.sys.global.cas.b128 dst, [%rd1], cmp, swap;
+; CHECK-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i128, ptr addrspace(1) %addr syncscope("") acquire, align 16
+ ret i128 %retval
+}
+
+define i128 @acquire_i128_global_cta_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_i128_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i128_global_cta_volatile_param_0];
+; CHECK-NEXT: mov.b64 %rd2, 0;
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 cmp, swap, dst;
+; CHECK-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; CHECK-NEXT: mov.b128 swap, {%rd2, %rd2};
+; CHECK-NEXT: atom.acquire.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; CHECK-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i128, ptr addrspace(1) %addr syncscope("block") acquire, align 16
+ ret i128 %retval
+}
+
+define i128 @acquire_i128_global_cluster_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_i128_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i128_global_cluster_volatile_param_0];
+; CHECK-NEXT: mov.b64 %rd2, 0;
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 cmp, swap, dst;
+; CHECK-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; CHECK-NEXT: mov.b128 swap, {%rd2, %rd2};
+; CHECK-NEXT: atom.acquire.cluster.global.cas.b128 dst, [%rd1], cmp, swap;
+; CHECK-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i128, ptr addrspace(1) %addr syncscope("cluster") acquire, align 16
+ ret i128 %retval
+}
+
+define i128 @acquire_i128_global_gpu_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_i128_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i128_global_gpu_volatile_param_0];
+; CHECK-NEXT: mov.b64 %rd2, 0;
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 cmp, swap, dst;
+; CHECK-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; CHECK-NEXT: mov.b128 swap, {%rd2, %rd2};
+; CHECK-NEXT: atom.acquire.gpu.global.cas.b128 dst, [%rd1], cmp, swap;
+; CHECK-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i128, ptr addrspace(1) %addr syncscope("device") acquire, align 16
+ ret i128 %retval
+}
+
+define half @acquire_half_global_sys(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_half_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_half_global_sys_param_0];
+; CHECK-NEXT: ld.acquire.sys.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr addrspace(1) %addr syncscope("") acquire, align 2
+ ret half %retval
+}
+
+define half @acquire_half_global_cta(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_half_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_half_global_cta_param_0];
+; CHECK-NEXT: ld.acquire.cta.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr addrspace(1) %addr syncscope("block") acquire, align 2
+ ret half %retval
+}
+
+define half @acquire_half_global_cluster(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_half_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_half_global_cluster_param_0];
+; CHECK-NEXT: ld.acquire.cluster.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr addrspace(1) %addr syncscope("cluster") acquire, align 2
+ ret half %retval
+}
+
+define half @acquire_half_global_gpu(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_half_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_half_global_gpu_param_0];
+; CHECK-NEXT: ld.acquire.gpu.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr addrspace(1) %addr syncscope("device") acquire, align 2
+ ret half %retval
+}
+
+define half @acquire_half_global_sys_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_half_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_half_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr addrspace(1) %addr syncscope("") acquire, align 2
+ ret half %retval
+}
+
+define half @acquire_half_global_cta_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_half_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_half_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr addrspace(1) %addr syncscope("block") acquire, align 2
+ ret half %retval
+}
+
+define half @acquire_half_global_cluster_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_half_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_half_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr addrspace(1) %addr syncscope("cluster") acquire, align 2
+ ret half %retval
+}
+
+define half @acquire_half_global_gpu_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_half_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_half_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr addrspace(1) %addr syncscope("device") acquire, align 2
+ ret half %retval
+}
+
+define bfloat @acquire_bfloat_global_sys(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_bfloat_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_bfloat_global_sys_param_0];
+; CHECK-NEXT: ld.acquire.sys.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr addrspace(1) %addr syncscope("") acquire, align 2
+ ret bfloat %retval
+}
+
+define bfloat @acquire_bfloat_global_cta(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_bfloat_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_bfloat_global_cta_param_0];
+; CHECK-NEXT: ld.acquire.cta.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr addrspace(1) %addr syncscope("block") acquire, align 2
+ ret bfloat %retval
+}
+
+define bfloat @acquire_bfloat_global_cluster(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_bfloat_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_bfloat_global_cluster_param_0];
+; CHECK-NEXT: ld.acquire.cluster.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr addrspace(1) %addr syncscope("cluster") acquire, align 2
+ ret bfloat %retval
+}
+
+define bfloat @acquire_bfloat_global_gpu(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_bfloat_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_bfloat_global_gpu_param_0];
+; CHECK-NEXT: ld.acquire.gpu.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr addrspace(1) %addr syncscope("device") acquire, align 2
+ ret bfloat %retval
+}
+
+define bfloat @acquire_bfloat_global_sys_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_bfloat_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_bfloat_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr addrspace(1) %addr syncscope("") acquire, align 2
+ ret bfloat %retval
+}
+
+define bfloat @acquire_bfloat_global_cta_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_bfloat_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_bfloat_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr addrspace(1) %addr syncscope("block") acquire, align 2
+ ret bfloat %retval
+}
+
+define bfloat @acquire_bfloat_global_cluster_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_bfloat_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_bfloat_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr addrspace(1) %addr syncscope("cluster") acquire, align 2
+ ret bfloat %retval
+}
+
+define bfloat @acquire_bfloat_global_gpu_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_bfloat_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_bfloat_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr addrspace(1) %addr syncscope("device") acquire, align 2
+ ret bfloat %retval
+}
+
+define float @acquire_float_global_sys(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_float_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_float_global_sys_param_0];
+; CHECK-NEXT: ld.acquire.sys.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr addrspace(1) %addr syncscope("") acquire, align 4
+ ret float %retval
+}
+
+define float @acquire_float_global_cta(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_float_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_float_global_cta_param_0];
+; CHECK-NEXT: ld.acquire.cta.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr addrspace(1) %addr syncscope("block") acquire, align 4
+ ret float %retval
+}
+
+define float @acquire_float_global_cluster(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_float_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_float_global_cluster_param_0];
+; CHECK-NEXT: ld.acquire.cluster.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr addrspace(1) %addr syncscope("cluster") acquire, align 4
+ ret float %retval
+}
+
+define float @acquire_float_global_gpu(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_float_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_float_global_gpu_param_0];
+; CHECK-NEXT: ld.acquire.gpu.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr addrspace(1) %addr syncscope("device") acquire, align 4
+ ret float %retval
+}
+
+define float @acquire_float_global_sys_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_float_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_float_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr addrspace(1) %addr syncscope("") acquire, align 4
+ ret float %retval
+}
+
+define float @acquire_float_global_cta_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_float_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_float_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr addrspace(1) %addr syncscope("block") acquire, align 4
+ ret float %retval
+}
+
+define float @acquire_float_global_cluster_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_float_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_float_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr addrspace(1) %addr syncscope("cluster") acquire, align 4
+ ret float %retval
+}
+
+define float @acquire_float_global_gpu_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_float_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_float_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr addrspace(1) %addr syncscope("device") acquire, align 4
+ ret float %retval
+}
+
+define double @acquire_double_global_sys(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_double_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_double_global_sys_param_0];
+; CHECK-NEXT: ld.acquire.sys.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr addrspace(1) %addr syncscope("") acquire, align 8
+ ret double %retval
+}
+
+define double @acquire_double_global_cta(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_double_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_double_global_cta_param_0];
+; CHECK-NEXT: ld.acquire.cta.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr addrspace(1) %addr syncscope("block") acquire, align 8
+ ret double %retval
+}
+
+define double @acquire_double_global_cluster(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_double_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_double_global_cluster_param_0];
+; CHECK-NEXT: ld.acquire.cluster.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr addrspace(1) %addr syncscope("cluster") acquire, align 8
+ ret double %retval
+}
+
+define double @acquire_double_global_gpu(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_double_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_double_global_gpu_param_0];
+; CHECK-NEXT: ld.acquire.gpu.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr addrspace(1) %addr syncscope("device") acquire, align 8
+ ret double %retval
+}
+
+define double @acquire_double_global_sys_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_double_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_double_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr addrspace(1) %addr syncscope("") acquire, align 8
+ ret double %retval
+}
+
+define double @acquire_double_global_cta_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_double_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_double_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr addrspace(1) %addr syncscope("block") acquire, align 8
+ ret double %retval
+}
+
+define double @acquire_double_global_cluster_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_double_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_double_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr addrspace(1) %addr syncscope("cluster") acquire, align 8
+ ret double %retval
+}
+
+define double @acquire_double_global_gpu_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: acquire_double_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_double_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr addrspace(1) %addr syncscope("device") acquire, align 8
+ ret double %retval
+}
+
+define i8 @acquire_i8_shared_sys(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_i8_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i8_shared_sys_param_0];
+; CHECK-NEXT: ld.acquire.sys.shared.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr addrspace(3) %addr syncscope("") acquire, align 1
+ ret i8 %retval
+}
+
+define i8 @acquire_i8_shared_cta(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_i8_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i8_shared_cta_param_0];
+; CHECK-NEXT: ld.acquire.cta.shared.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr addrspace(3) %addr syncscope("block") acquire, align 1
+ ret i8 %retval
+}
+
+define i8 @acquire_i8_shared_cluster(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_i8_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i8_shared_cluster_param_0];
+; CHECK-NEXT: ld.acquire.cluster.shared.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr addrspace(3) %addr syncscope("cluster") acquire, align 1
+ ret i8 %retval
+}
+
+define i8 @acquire_i8_shared_gpu(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_i8_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i8_shared_gpu_param_0];
+; CHECK-NEXT: ld.acquire.gpu.shared.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr addrspace(3) %addr syncscope("device") acquire, align 1
+ ret i8 %retval
+}
+
+define i8 @acquire_i8_shared_sys_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_i8_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i8_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.shared.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr addrspace(3) %addr syncscope("") acquire, align 1
+ ret i8 %retval
+}
+
+define i8 @acquire_i8_shared_cta_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_i8_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i8_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.shared.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr addrspace(3) %addr syncscope("block") acquire, align 1
+ ret i8 %retval
+}
+
+define i8 @acquire_i8_shared_cluster_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_i8_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i8_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.shared.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr addrspace(3) %addr syncscope("cluster") acquire, align 1
+ ret i8 %retval
+}
+
+define i8 @acquire_i8_shared_gpu_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_i8_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i8_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.shared.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr addrspace(3) %addr syncscope("device") acquire, align 1
+ ret i8 %retval
+}
+
+define i16 @acquire_i16_shared_sys(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_i16_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i16_shared_sys_param_0];
+; CHECK-NEXT: ld.acquire.sys.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr addrspace(3) %addr syncscope("") acquire, align 2
+ ret i16 %retval
+}
+
+define i16 @acquire_i16_shared_cta(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_i16_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i16_shared_cta_param_0];
+; CHECK-NEXT: ld.acquire.cta.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr addrspace(3) %addr syncscope("block") acquire, align 2
+ ret i16 %retval
+}
+
+define i16 @acquire_i16_shared_cluster(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_i16_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i16_shared_cluster_param_0];
+; CHECK-NEXT: ld.acquire.cluster.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr addrspace(3) %addr syncscope("cluster") acquire, align 2
+ ret i16 %retval
+}
+
+define i16 @acquire_i16_shared_gpu(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_i16_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i16_shared_gpu_param_0];
+; CHECK-NEXT: ld.acquire.gpu.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr addrspace(3) %addr syncscope("device") acquire, align 2
+ ret i16 %retval
+}
+
+define i16 @acquire_i16_shared_sys_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_i16_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i16_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr addrspace(3) %addr syncscope("") acquire, align 2
+ ret i16 %retval
+}
+
+define i16 @acquire_i16_shared_cta_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_i16_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i16_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr addrspace(3) %addr syncscope("block") acquire, align 2
+ ret i16 %retval
+}
+
+define i16 @acquire_i16_shared_cluster_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_i16_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i16_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr addrspace(3) %addr syncscope("cluster") acquire, align 2
+ ret i16 %retval
+}
+
+define i16 @acquire_i16_shared_gpu_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_i16_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i16_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr addrspace(3) %addr syncscope("device") acquire, align 2
+ ret i16 %retval
+}
+
+define i32 @acquire_i32_shared_sys(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_i32_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i32_shared_sys_param_0];
+; CHECK-NEXT: ld.acquire.sys.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr addrspace(3) %addr syncscope("") acquire, align 4
+ ret i32 %retval
+}
+
+define i32 @acquire_i32_shared_cta(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_i32_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i32_shared_cta_param_0];
+; CHECK-NEXT: ld.acquire.cta.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr addrspace(3) %addr syncscope("block") acquire, align 4
+ ret i32 %retval
+}
+
+define i32 @acquire_i32_shared_cluster(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_i32_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i32_shared_cluster_param_0];
+; CHECK-NEXT: ld.acquire.cluster.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr addrspace(3) %addr syncscope("cluster") acquire, align 4
+ ret i32 %retval
+}
+
+define i32 @acquire_i32_shared_gpu(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_i32_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i32_shared_gpu_param_0];
+; CHECK-NEXT: ld.acquire.gpu.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr addrspace(3) %addr syncscope("device") acquire, align 4
+ ret i32 %retval
+}
+
+define i32 @acquire_i32_shared_sys_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_i32_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i32_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr addrspace(3) %addr syncscope("") acquire, align 4
+ ret i32 %retval
+}
+
+define i32 @acquire_i32_shared_cta_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_i32_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i32_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr addrspace(3) %addr syncscope("block") acquire, align 4
+ ret i32 %retval
+}
+
+define i32 @acquire_i32_shared_cluster_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_i32_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i32_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr addrspace(3) %addr syncscope("cluster") acquire, align 4
+ ret i32 %retval
+}
+
+define i32 @acquire_i32_shared_gpu_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_i32_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i32_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr addrspace(3) %addr syncscope("device") acquire, align 4
+ ret i32 %retval
+}
+
+define i64 @acquire_i64_shared_sys(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_i64_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i64_shared_sys_param_0];
+; CHECK-NEXT: ld.acquire.sys.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr addrspace(3) %addr syncscope("") acquire, align 8
+ ret i64 %retval
+}
+
+define i64 @acquire_i64_shared_cta(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_i64_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i64_shared_cta_param_0];
+; CHECK-NEXT: ld.acquire.cta.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr addrspace(3) %addr syncscope("block") acquire, align 8
+ ret i64 %retval
+}
+
+define i64 @acquire_i64_shared_cluster(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_i64_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i64_shared_cluster_param_0];
+; CHECK-NEXT: ld.acquire.cluster.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr addrspace(3) %addr syncscope("cluster") acquire, align 8
+ ret i64 %retval
+}
+
+define i64 @acquire_i64_shared_gpu(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_i64_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i64_shared_gpu_param_0];
+; CHECK-NEXT: ld.acquire.gpu.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr addrspace(3) %addr syncscope("device") acquire, align 8
+ ret i64 %retval
+}
+
+define i64 @acquire_i64_shared_sys_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_i64_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i64_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr addrspace(3) %addr syncscope("") acquire, align 8
+ ret i64 %retval
+}
+
+define i64 @acquire_i64_shared_cta_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_i64_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i64_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr addrspace(3) %addr syncscope("block") acquire, align 8
+ ret i64 %retval
+}
+
+define i64 @acquire_i64_shared_cluster_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_i64_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i64_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr addrspace(3) %addr syncscope("cluster") acquire, align 8
+ ret i64 %retval
+}
+
+define i64 @acquire_i64_shared_gpu_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_i64_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i64_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr addrspace(3) %addr syncscope("device") acquire, align 8
+ ret i64 %retval
+}
+
+define i128 @acquire_i128_shared_sys(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_i128_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i128_shared_sys_param_0];
+; CHECK-NEXT: mov.b64 %rd2, 0;
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 cmp, swap, dst;
+; CHECK-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; CHECK-NEXT: mov.b128 swap, {%rd2, %rd2};
+; CHECK-NEXT: atom.acquire.sys.shared.cas.b128 dst, [%rd1], cmp, swap;
+; CHECK-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; CHECK-NEXT: ret;
+ %retval = load atomic i128, ptr addrspace(3) %addr syncscope("") acquire, align 16
+ ret i128 %retval
+}
+
+define i128 @acquire_i128_shared_cta(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_i128_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i128_shared_cta_param_0];
+; CHECK-NEXT: mov.b64 %rd2, 0;
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 cmp, swap, dst;
+; CHECK-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; CHECK-NEXT: mov.b128 swap, {%rd2, %rd2};
+; CHECK-NEXT: atom.acquire.cta.shared.cas.b128 dst, [%rd1], cmp, swap;
+; CHECK-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; CHECK-NEXT: ret;
+ %retval = load atomic i128, ptr addrspace(3) %addr syncscope("block") acquire, align 16
+ ret i128 %retval
+}
+
+define i128 @acquire_i128_shared_cluster(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_i128_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i128_shared_cluster_param_0];
+; CHECK-NEXT: mov.b64 %rd2, 0;
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 cmp, swap, dst;
+; CHECK-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; CHECK-NEXT: mov.b128 swap, {%rd2, %rd2};
+; CHECK-NEXT: atom.acquire.cluster.shared.cas.b128 dst, [%rd1], cmp, swap;
+; CHECK-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; CHECK-NEXT: ret;
+ %retval = load atomic i128, ptr addrspace(3) %addr syncscope("cluster") acquire, align 16
+ ret i128 %retval
+}
+
+define i128 @acquire_i128_shared_gpu(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_i128_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i128_shared_gpu_param_0];
+; CHECK-NEXT: mov.b64 %rd2, 0;
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 cmp, swap, dst;
+; CHECK-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; CHECK-NEXT: mov.b128 swap, {%rd2, %rd2};
+; CHECK-NEXT: atom.acquire.gpu.shared.cas.b128 dst, [%rd1], cmp, swap;
+; CHECK-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; CHECK-NEXT: ret;
+ %retval = load atomic i128, ptr addrspace(3) %addr syncscope("device") acquire, align 16
+ ret i128 %retval
+}
+
+define i128 @acquire_i128_shared_sys_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_i128_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i128_shared_sys_volatile_param_0];
+; CHECK-NEXT: mov.b64 %rd2, 0;
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 cmp, swap, dst;
+; CHECK-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; CHECK-NEXT: mov.b128 swap, {%rd2, %rd2};
+; CHECK-NEXT: atom.acquire.sys.shared.cas.b128 dst, [%rd1], cmp, swap;
+; CHECK-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i128, ptr addrspace(3) %addr syncscope("") acquire, align 16
+ ret i128 %retval
+}
+
+define i128 @acquire_i128_shared_cta_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_i128_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i128_shared_cta_volatile_param_0];
+; CHECK-NEXT: mov.b64 %rd2, 0;
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 cmp, swap, dst;
+; CHECK-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; CHECK-NEXT: mov.b128 swap, {%rd2, %rd2};
+; CHECK-NEXT: atom.acquire.cta.shared.cas.b128 dst, [%rd1], cmp, swap;
+; CHECK-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i128, ptr addrspace(3) %addr syncscope("block") acquire, align 16
+ ret i128 %retval
+}
+
+define i128 @acquire_i128_shared_cluster_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_i128_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i128_shared_cluster_volatile_param_0];
+; CHECK-NEXT: mov.b64 %rd2, 0;
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 cmp, swap, dst;
+; CHECK-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; CHECK-NEXT: mov.b128 swap, {%rd2, %rd2};
+; CHECK-NEXT: atom.acquire.cluster.shared.cas.b128 dst, [%rd1], cmp, swap;
+; CHECK-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i128, ptr addrspace(3) %addr syncscope("cluster") acquire, align 16
+ ret i128 %retval
+}
+
+define i128 @acquire_i128_shared_gpu_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_i128_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_i128_shared_gpu_volatile_param_0];
+; CHECK-NEXT: mov.b64 %rd2, 0;
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 cmp, swap, dst;
+; CHECK-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; CHECK-NEXT: mov.b128 swap, {%rd2, %rd2};
+; CHECK-NEXT: atom.acquire.gpu.shared.cas.b128 dst, [%rd1], cmp, swap;
+; CHECK-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i128, ptr addrspace(3) %addr syncscope("device") acquire, align 16
+ ret i128 %retval
+}
+
+define half @acquire_half_shared_sys(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_half_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_half_shared_sys_param_0];
+; CHECK-NEXT: ld.acquire.sys.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr addrspace(3) %addr syncscope("") acquire, align 2
+ ret half %retval
+}
+
+define half @acquire_half_shared_cta(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_half_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_half_shared_cta_param_0];
+; CHECK-NEXT: ld.acquire.cta.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr addrspace(3) %addr syncscope("block") acquire, align 2
+ ret half %retval
+}
+
+define half @acquire_half_shared_cluster(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_half_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_half_shared_cluster_param_0];
+; CHECK-NEXT: ld.acquire.cluster.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr addrspace(3) %addr syncscope("cluster") acquire, align 2
+ ret half %retval
+}
+
+define half @acquire_half_shared_gpu(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_half_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_half_shared_gpu_param_0];
+; CHECK-NEXT: ld.acquire.gpu.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr addrspace(3) %addr syncscope("device") acquire, align 2
+ ret half %retval
+}
+
+define half @acquire_half_shared_sys_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_half_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_half_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr addrspace(3) %addr syncscope("") acquire, align 2
+ ret half %retval
+}
+
+define half @acquire_half_shared_cta_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_half_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_half_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr addrspace(3) %addr syncscope("block") acquire, align 2
+ ret half %retval
+}
+
+define half @acquire_half_shared_cluster_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_half_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_half_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr addrspace(3) %addr syncscope("cluster") acquire, align 2
+ ret half %retval
+}
+
+define half @acquire_half_shared_gpu_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_half_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_half_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr addrspace(3) %addr syncscope("device") acquire, align 2
+ ret half %retval
+}
+
+define bfloat @acquire_bfloat_shared_sys(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_bfloat_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_bfloat_shared_sys_param_0];
+; CHECK-NEXT: ld.acquire.sys.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr addrspace(3) %addr syncscope("") acquire, align 2
+ ret bfloat %retval
+}
+
+define bfloat @acquire_bfloat_shared_cta(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_bfloat_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_bfloat_shared_cta_param_0];
+; CHECK-NEXT: ld.acquire.cta.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr addrspace(3) %addr syncscope("block") acquire, align 2
+ ret bfloat %retval
+}
+
+define bfloat @acquire_bfloat_shared_cluster(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_bfloat_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_bfloat_shared_cluster_param_0];
+; CHECK-NEXT: ld.acquire.cluster.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr addrspace(3) %addr syncscope("cluster") acquire, align 2
+ ret bfloat %retval
+}
+
+define bfloat @acquire_bfloat_shared_gpu(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_bfloat_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_bfloat_shared_gpu_param_0];
+; CHECK-NEXT: ld.acquire.gpu.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr addrspace(3) %addr syncscope("device") acquire, align 2
+ ret bfloat %retval
+}
+
+define bfloat @acquire_bfloat_shared_sys_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_bfloat_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_bfloat_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr addrspace(3) %addr syncscope("") acquire, align 2
+ ret bfloat %retval
+}
+
+define bfloat @acquire_bfloat_shared_cta_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_bfloat_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_bfloat_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr addrspace(3) %addr syncscope("block") acquire, align 2
+ ret bfloat %retval
+}
+
+define bfloat @acquire_bfloat_shared_cluster_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_bfloat_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_bfloat_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr addrspace(3) %addr syncscope("cluster") acquire, align 2
+ ret bfloat %retval
+}
+
+define bfloat @acquire_bfloat_shared_gpu_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_bfloat_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_bfloat_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr addrspace(3) %addr syncscope("device") acquire, align 2
+ ret bfloat %retval
+}
+
+define float @acquire_float_shared_sys(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_float_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_float_shared_sys_param_0];
+; CHECK-NEXT: ld.acquire.sys.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr addrspace(3) %addr syncscope("") acquire, align 4
+ ret float %retval
+}
+
+define float @acquire_float_shared_cta(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_float_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_float_shared_cta_param_0];
+; CHECK-NEXT: ld.acquire.cta.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr addrspace(3) %addr syncscope("block") acquire, align 4
+ ret float %retval
+}
+
+define float @acquire_float_shared_cluster(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_float_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_float_shared_cluster_param_0];
+; CHECK-NEXT: ld.acquire.cluster.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr addrspace(3) %addr syncscope("cluster") acquire, align 4
+ ret float %retval
+}
+
+define float @acquire_float_shared_gpu(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_float_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_float_shared_gpu_param_0];
+; CHECK-NEXT: ld.acquire.gpu.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr addrspace(3) %addr syncscope("device") acquire, align 4
+ ret float %retval
+}
+
+define float @acquire_float_shared_sys_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_float_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_float_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr addrspace(3) %addr syncscope("") acquire, align 4
+ ret float %retval
+}
+
+define float @acquire_float_shared_cta_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_float_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_float_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr addrspace(3) %addr syncscope("block") acquire, align 4
+ ret float %retval
+}
+
+define float @acquire_float_shared_cluster_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_float_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_float_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr addrspace(3) %addr syncscope("cluster") acquire, align 4
+ ret float %retval
+}
+
+define float @acquire_float_shared_gpu_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_float_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_float_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr addrspace(3) %addr syncscope("device") acquire, align 4
+ ret float %retval
+}
+
+define double @acquire_double_shared_sys(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_double_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_double_shared_sys_param_0];
+; CHECK-NEXT: ld.acquire.sys.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr addrspace(3) %addr syncscope("") acquire, align 8
+ ret double %retval
+}
+
+define double @acquire_double_shared_cta(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_double_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_double_shared_cta_param_0];
+; CHECK-NEXT: ld.acquire.cta.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr addrspace(3) %addr syncscope("block") acquire, align 8
+ ret double %retval
+}
+
+define double @acquire_double_shared_cluster(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_double_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_double_shared_cluster_param_0];
+; CHECK-NEXT: ld.acquire.cluster.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr addrspace(3) %addr syncscope("cluster") acquire, align 8
+ ret double %retval
+}
+
+define double @acquire_double_shared_gpu(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_double_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_double_shared_gpu_param_0];
+; CHECK-NEXT: ld.acquire.gpu.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr addrspace(3) %addr syncscope("device") acquire, align 8
+ ret double %retval
+}
+
+define double @acquire_double_shared_sys_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_double_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_double_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr addrspace(3) %addr syncscope("") acquire, align 8
+ ret double %retval
+}
+
+define double @acquire_double_shared_cta_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_double_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_double_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr addrspace(3) %addr syncscope("block") acquire, align 8
+ ret double %retval
+}
+
+define double @acquire_double_shared_cluster_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_double_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_double_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr addrspace(3) %addr syncscope("cluster") acquire, align 8
+ ret double %retval
+}
+
+define double @acquire_double_shared_gpu_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: acquire_double_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [acquire_double_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.acquire.sys.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr addrspace(3) %addr syncscope("device") acquire, align 8
+ ret double %retval
+}
+
+define i8 @acquire_i8_local_sys() {
+; CHECK-LABEL: acquire_i8_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot900[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot900;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b8 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i8, ptr addrspace(5) %slot syncscope("") acquire, align 1
+ ret i8 %retval
+}
+
+define i8 @acquire_i8_local_cta() {
+; CHECK-LABEL: acquire_i8_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot901[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot901;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b8 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i8, ptr addrspace(5) %slot syncscope("block") acquire, align 1
+ ret i8 %retval
+}
+
+define i8 @acquire_i8_local_cluster() {
+; CHECK-LABEL: acquire_i8_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot902[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot902;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b8 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i8, ptr addrspace(5) %slot syncscope("cluster") acquire, align 1
+ ret i8 %retval
+}
+
+define i8 @acquire_i8_local_gpu() {
+; CHECK-LABEL: acquire_i8_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot903[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot903;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b8 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i8, ptr addrspace(5) %slot syncscope("device") acquire, align 1
+ ret i8 %retval
+}
+
+define i8 @acquire_i8_local_sys_volatile() {
+; CHECK-LABEL: acquire_i8_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot904[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot904;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b8 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i8, ptr addrspace(5) %slot syncscope("") acquire, align 1
+ ret i8 %retval
+}
+
+define i8 @acquire_i8_local_cta_volatile() {
+; CHECK-LABEL: acquire_i8_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot905[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot905;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b8 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i8, ptr addrspace(5) %slot syncscope("block") acquire, align 1
+ ret i8 %retval
+}
+
+define i8 @acquire_i8_local_cluster_volatile() {
+; CHECK-LABEL: acquire_i8_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot906[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot906;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b8 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i8, ptr addrspace(5) %slot syncscope("cluster") acquire, align 1
+ ret i8 %retval
+}
+
+define i8 @acquire_i8_local_gpu_volatile() {
+; CHECK-LABEL: acquire_i8_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot907[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot907;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b8 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i8, ptr addrspace(5) %slot syncscope("device") acquire, align 1
+ ret i8 %retval
+}
+
+define i16 @acquire_i16_local_sys() {
+; CHECK-LABEL: acquire_i16_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot908[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot908;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i16, ptr addrspace(5) %slot syncscope("") acquire, align 2
+ ret i16 %retval
+}
+
+define i16 @acquire_i16_local_cta() {
+; CHECK-LABEL: acquire_i16_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot909[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot909;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i16, ptr addrspace(5) %slot syncscope("block") acquire, align 2
+ ret i16 %retval
+}
+
+define i16 @acquire_i16_local_cluster() {
+; CHECK-LABEL: acquire_i16_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot910[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot910;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i16, ptr addrspace(5) %slot syncscope("cluster") acquire, align 2
+ ret i16 %retval
+}
+
+define i16 @acquire_i16_local_gpu() {
+; CHECK-LABEL: acquire_i16_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot911[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot911;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i16, ptr addrspace(5) %slot syncscope("device") acquire, align 2
+ ret i16 %retval
+}
+
+define i16 @acquire_i16_local_sys_volatile() {
+; CHECK-LABEL: acquire_i16_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot912[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot912;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i16, ptr addrspace(5) %slot syncscope("") acquire, align 2
+ ret i16 %retval
+}
+
+define i16 @acquire_i16_local_cta_volatile() {
+; CHECK-LABEL: acquire_i16_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot913[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot913;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i16, ptr addrspace(5) %slot syncscope("block") acquire, align 2
+ ret i16 %retval
+}
+
+define i16 @acquire_i16_local_cluster_volatile() {
+; CHECK-LABEL: acquire_i16_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot914[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot914;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i16, ptr addrspace(5) %slot syncscope("cluster") acquire, align 2
+ ret i16 %retval
+}
+
+define i16 @acquire_i16_local_gpu_volatile() {
+; CHECK-LABEL: acquire_i16_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot915[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot915;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i16, ptr addrspace(5) %slot syncscope("device") acquire, align 2
+ ret i16 %retval
+}
+
+define i32 @acquire_i32_local_sys() {
+; CHECK-LABEL: acquire_i32_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot916[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot916;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i32, ptr addrspace(5) %slot syncscope("") acquire, align 4
+ ret i32 %retval
+}
+
+define i32 @acquire_i32_local_cta() {
+; CHECK-LABEL: acquire_i32_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot917[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot917;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i32, ptr addrspace(5) %slot syncscope("block") acquire, align 4
+ ret i32 %retval
+}
+
+define i32 @acquire_i32_local_cluster() {
+; CHECK-LABEL: acquire_i32_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot918[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot918;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i32, ptr addrspace(5) %slot syncscope("cluster") acquire, align 4
+ ret i32 %retval
+}
+
+define i32 @acquire_i32_local_gpu() {
+; CHECK-LABEL: acquire_i32_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot919[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot919;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i32, ptr addrspace(5) %slot syncscope("device") acquire, align 4
+ ret i32 %retval
+}
+
+define i32 @acquire_i32_local_sys_volatile() {
+; CHECK-LABEL: acquire_i32_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot920[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot920;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i32, ptr addrspace(5) %slot syncscope("") acquire, align 4
+ ret i32 %retval
+}
+
+define i32 @acquire_i32_local_cta_volatile() {
+; CHECK-LABEL: acquire_i32_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot921[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot921;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i32, ptr addrspace(5) %slot syncscope("block") acquire, align 4
+ ret i32 %retval
+}
+
+define i32 @acquire_i32_local_cluster_volatile() {
+; CHECK-LABEL: acquire_i32_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot922[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot922;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i32, ptr addrspace(5) %slot syncscope("cluster") acquire, align 4
+ ret i32 %retval
+}
+
+define i32 @acquire_i32_local_gpu_volatile() {
+; CHECK-LABEL: acquire_i32_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot923[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot923;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i32, ptr addrspace(5) %slot syncscope("device") acquire, align 4
+ ret i32 %retval
+}
+
+define i64 @acquire_i64_local_sys() {
+; CHECK-LABEL: acquire_i64_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot924[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot924;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i64, ptr addrspace(5) %slot syncscope("") acquire, align 8
+ ret i64 %retval
+}
+
+define i64 @acquire_i64_local_cta() {
+; CHECK-LABEL: acquire_i64_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot925[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot925;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i64, ptr addrspace(5) %slot syncscope("block") acquire, align 8
+ ret i64 %retval
+}
+
+define i64 @acquire_i64_local_cluster() {
+; CHECK-LABEL: acquire_i64_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot926[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot926;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i64, ptr addrspace(5) %slot syncscope("cluster") acquire, align 8
+ ret i64 %retval
+}
+
+define i64 @acquire_i64_local_gpu() {
+; CHECK-LABEL: acquire_i64_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot927[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot927;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i64, ptr addrspace(5) %slot syncscope("device") acquire, align 8
+ ret i64 %retval
+}
+
+define i64 @acquire_i64_local_sys_volatile() {
+; CHECK-LABEL: acquire_i64_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot928[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot928;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i64, ptr addrspace(5) %slot syncscope("") acquire, align 8
+ ret i64 %retval
+}
+
+define i64 @acquire_i64_local_cta_volatile() {
+; CHECK-LABEL: acquire_i64_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot929[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot929;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i64, ptr addrspace(5) %slot syncscope("block") acquire, align 8
+ ret i64 %retval
+}
+
+define i64 @acquire_i64_local_cluster_volatile() {
+; CHECK-LABEL: acquire_i64_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot930[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot930;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i64, ptr addrspace(5) %slot syncscope("cluster") acquire, align 8
+ ret i64 %retval
+}
+
+define i64 @acquire_i64_local_gpu_volatile() {
+; CHECK-LABEL: acquire_i64_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot931[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot931;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i64, ptr addrspace(5) %slot syncscope("device") acquire, align 8
+ ret i64 %retval
+}
+
+define i128 @acquire_i128_local_sys() {
+; CHECK-LABEL: acquire_i128_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 16 .b8 __local_depot932[16];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot932;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.v2.b64 {%rd2, %rd3}, [%SP];
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd2, %rd3};
+; CHECK-NEXT: ret;
+ %slot = alloca i128, align 16, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i128, ptr addrspace(5) %slot syncscope("") acquire, align 16
+ ret i128 %retval
+}
+
+define i128 @acquire_i128_local_cta() {
+; CHECK-LABEL: acquire_i128_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 16 .b8 __local_depot933[16];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot933;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.v2.b64 {%rd2, %rd3}, [%SP];
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd2, %rd3};
+; CHECK-NEXT: ret;
+ %slot = alloca i128, align 16, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i128, ptr addrspace(5) %slot syncscope("block") acquire, align 16
+ ret i128 %retval
+}
+
+define i128 @acquire_i128_local_cluster() {
+; CHECK-LABEL: acquire_i128_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 16 .b8 __local_depot934[16];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot934;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.v2.b64 {%rd2, %rd3}, [%SP];
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd2, %rd3};
+; CHECK-NEXT: ret;
+ %slot = alloca i128, align 16, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i128, ptr addrspace(5) %slot syncscope("cluster") acquire, align 16
+ ret i128 %retval
+}
+
+define i128 @acquire_i128_local_gpu() {
+; CHECK-LABEL: acquire_i128_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 16 .b8 __local_depot935[16];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot935;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.v2.b64 {%rd2, %rd3}, [%SP];
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd2, %rd3};
+; CHECK-NEXT: ret;
+ %slot = alloca i128, align 16, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i128, ptr addrspace(5) %slot syncscope("device") acquire, align 16
+ ret i128 %retval
+}
+
+define i128 @acquire_i128_local_sys_volatile() {
+; CHECK-LABEL: acquire_i128_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 16 .b8 __local_depot936[16];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot936;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.v2.b64 {%rd2, %rd3}, [%SP];
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd2, %rd3};
+; CHECK-NEXT: ret;
+ %slot = alloca i128, align 16, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i128, ptr addrspace(5) %slot syncscope("") acquire, align 16
+ ret i128 %retval
+}
+
+define i128 @acquire_i128_local_cta_volatile() {
+; CHECK-LABEL: acquire_i128_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 16 .b8 __local_depot937[16];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot937;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.v2.b64 {%rd2, %rd3}, [%SP];
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd2, %rd3};
+; CHECK-NEXT: ret;
+ %slot = alloca i128, align 16, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i128, ptr addrspace(5) %slot syncscope("block") acquire, align 16
+ ret i128 %retval
+}
+
+define i128 @acquire_i128_local_cluster_volatile() {
+; CHECK-LABEL: acquire_i128_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 16 .b8 __local_depot938[16];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot938;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.v2.b64 {%rd2, %rd3}, [%SP];
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd2, %rd3};
+; CHECK-NEXT: ret;
+ %slot = alloca i128, align 16, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i128, ptr addrspace(5) %slot syncscope("cluster") acquire, align 16
+ ret i128 %retval
+}
+
+define i128 @acquire_i128_local_gpu_volatile() {
+; CHECK-LABEL: acquire_i128_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 16 .b8 __local_depot939[16];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot939;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.v2.b64 {%rd2, %rd3}, [%SP];
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd2, %rd3};
+; CHECK-NEXT: ret;
+ %slot = alloca i128, align 16, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i128, ptr addrspace(5) %slot syncscope("device") acquire, align 16
+ ret i128 %retval
+}
+
+define half @acquire_half_local_sys() {
+; CHECK-LABEL: acquire_half_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot940[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot940;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic half, ptr addrspace(5) %slot syncscope("") acquire, align 2
+ ret half %retval
+}
+
+define half @acquire_half_local_cta() {
+; CHECK-LABEL: acquire_half_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot941[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot941;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic half, ptr addrspace(5) %slot syncscope("block") acquire, align 2
+ ret half %retval
+}
+
+define half @acquire_half_local_cluster() {
+; CHECK-LABEL: acquire_half_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot942[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot942;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic half, ptr addrspace(5) %slot syncscope("cluster") acquire, align 2
+ ret half %retval
+}
+
+define half @acquire_half_local_gpu() {
+; CHECK-LABEL: acquire_half_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot943[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot943;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic half, ptr addrspace(5) %slot syncscope("device") acquire, align 2
+ ret half %retval
+}
+
+define half @acquire_half_local_sys_volatile() {
+; CHECK-LABEL: acquire_half_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot944[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot944;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile half, ptr addrspace(5) %slot syncscope("") acquire, align 2
+ ret half %retval
+}
+
+define half @acquire_half_local_cta_volatile() {
+; CHECK-LABEL: acquire_half_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot945[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot945;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile half, ptr addrspace(5) %slot syncscope("block") acquire, align 2
+ ret half %retval
+}
+
+define half @acquire_half_local_cluster_volatile() {
+; CHECK-LABEL: acquire_half_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot946[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot946;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile half, ptr addrspace(5) %slot syncscope("cluster") acquire, align 2
+ ret half %retval
+}
+
+define half @acquire_half_local_gpu_volatile() {
+; CHECK-LABEL: acquire_half_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot947[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot947;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile half, ptr addrspace(5) %slot syncscope("device") acquire, align 2
+ ret half %retval
+}
+
+define bfloat @acquire_bfloat_local_sys() {
+; CHECK-LABEL: acquire_bfloat_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot948[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot948;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic bfloat, ptr addrspace(5) %slot syncscope("") acquire, align 2
+ ret bfloat %retval
+}
+
+define bfloat @acquire_bfloat_local_cta() {
+; CHECK-LABEL: acquire_bfloat_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot949[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot949;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic bfloat, ptr addrspace(5) %slot syncscope("block") acquire, align 2
+ ret bfloat %retval
+}
+
+define bfloat @acquire_bfloat_local_cluster() {
+; CHECK-LABEL: acquire_bfloat_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot950[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot950;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic bfloat, ptr addrspace(5) %slot syncscope("cluster") acquire, align 2
+ ret bfloat %retval
+}
+
+define bfloat @acquire_bfloat_local_gpu() {
+; CHECK-LABEL: acquire_bfloat_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot951[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot951;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic bfloat, ptr addrspace(5) %slot syncscope("device") acquire, align 2
+ ret bfloat %retval
+}
+
+define bfloat @acquire_bfloat_local_sys_volatile() {
+; CHECK-LABEL: acquire_bfloat_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot952[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot952;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile bfloat, ptr addrspace(5) %slot syncscope("") acquire, align 2
+ ret bfloat %retval
+}
+
+define bfloat @acquire_bfloat_local_cta_volatile() {
+; CHECK-LABEL: acquire_bfloat_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot953[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot953;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile bfloat, ptr addrspace(5) %slot syncscope("block") acquire, align 2
+ ret bfloat %retval
+}
+
+define bfloat @acquire_bfloat_local_cluster_volatile() {
+; CHECK-LABEL: acquire_bfloat_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot954[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot954;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile bfloat, ptr addrspace(5) %slot syncscope("cluster") acquire, align 2
+ ret bfloat %retval
+}
+
+define bfloat @acquire_bfloat_local_gpu_volatile() {
+; CHECK-LABEL: acquire_bfloat_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot955[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot955;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile bfloat, ptr addrspace(5) %slot syncscope("device") acquire, align 2
+ ret bfloat %retval
+}
+
+define float @acquire_float_local_sys() {
+; CHECK-LABEL: acquire_float_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot956[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot956;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic float, ptr addrspace(5) %slot syncscope("") acquire, align 4
+ ret float %retval
+}
+
+define float @acquire_float_local_cta() {
+; CHECK-LABEL: acquire_float_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot957[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot957;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic float, ptr addrspace(5) %slot syncscope("block") acquire, align 4
+ ret float %retval
+}
+
+define float @acquire_float_local_cluster() {
+; CHECK-LABEL: acquire_float_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot958[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot958;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic float, ptr addrspace(5) %slot syncscope("cluster") acquire, align 4
+ ret float %retval
+}
+
+define float @acquire_float_local_gpu() {
+; CHECK-LABEL: acquire_float_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot959[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot959;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic float, ptr addrspace(5) %slot syncscope("device") acquire, align 4
+ ret float %retval
+}
+
+define float @acquire_float_local_sys_volatile() {
+; CHECK-LABEL: acquire_float_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot960[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot960;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile float, ptr addrspace(5) %slot syncscope("") acquire, align 4
+ ret float %retval
+}
+
+define float @acquire_float_local_cta_volatile() {
+; CHECK-LABEL: acquire_float_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot961[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot961;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile float, ptr addrspace(5) %slot syncscope("block") acquire, align 4
+ ret float %retval
+}
+
+define float @acquire_float_local_cluster_volatile() {
+; CHECK-LABEL: acquire_float_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot962[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot962;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile float, ptr addrspace(5) %slot syncscope("cluster") acquire, align 4
+ ret float %retval
+}
+
+define float @acquire_float_local_gpu_volatile() {
+; CHECK-LABEL: acquire_float_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot963[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot963;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile float, ptr addrspace(5) %slot syncscope("device") acquire, align 4
+ ret float %retval
+}
+
+define double @acquire_double_local_sys() {
+; CHECK-LABEL: acquire_double_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot964[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot964;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic double, ptr addrspace(5) %slot syncscope("") acquire, align 8
+ ret double %retval
+}
+
+define double @acquire_double_local_cta() {
+; CHECK-LABEL: acquire_double_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot965[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot965;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic double, ptr addrspace(5) %slot syncscope("block") acquire, align 8
+ ret double %retval
+}
+
+define double @acquire_double_local_cluster() {
+; CHECK-LABEL: acquire_double_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot966[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot966;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic double, ptr addrspace(5) %slot syncscope("cluster") acquire, align 8
+ ret double %retval
+}
+
+define double @acquire_double_local_gpu() {
+; CHECK-LABEL: acquire_double_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot967[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot967;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic double, ptr addrspace(5) %slot syncscope("device") acquire, align 8
+ ret double %retval
+}
+
+define double @acquire_double_local_sys_volatile() {
+; CHECK-LABEL: acquire_double_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot968[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot968;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile double, ptr addrspace(5) %slot syncscope("") acquire, align 8
+ ret double %retval
+}
+
+define double @acquire_double_local_cta_volatile() {
+; CHECK-LABEL: acquire_double_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot969[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot969;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile double, ptr addrspace(5) %slot syncscope("block") acquire, align 8
+ ret double %retval
+}
+
+define double @acquire_double_local_cluster_volatile() {
+; CHECK-LABEL: acquire_double_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot970[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot970;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile double, ptr addrspace(5) %slot syncscope("cluster") acquire, align 8
+ ret double %retval
+}
+
+define double @acquire_double_local_gpu_volatile() {
+; CHECK-LABEL: acquire_double_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot971[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot971;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile double, ptr addrspace(5) %slot syncscope("device") acquire, align 8
+ ret double %retval
+}
+
+define i8 @seq_cst_i8_generic_sys(ptr %addr) {
+; CHECK-LABEL: seq_cst_i8_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_generic_sys_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr %addr syncscope("") seq_cst, align 1
+ ret i8 %retval
+}
+
+define i8 @seq_cst_i8_generic_cta(ptr %addr) {
+; CHECK-LABEL: seq_cst_i8_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_generic_cta_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.acquire.cta.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr %addr syncscope("block") seq_cst, align 1
+ ret i8 %retval
+}
+
+define i8 @seq_cst_i8_generic_cluster(ptr %addr) {
+; CHECK-LABEL: seq_cst_i8_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_generic_cluster_param_0];
+; CHECK-NEXT: fence.sc.cluster;
+; CHECK-NEXT: ld.acquire.cluster.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr %addr syncscope("cluster") seq_cst, align 1
+ ret i8 %retval
+}
+
+define i8 @seq_cst_i8_generic_gpu(ptr %addr) {
+; CHECK-LABEL: seq_cst_i8_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_generic_gpu_param_0];
+; CHECK-NEXT: fence.sc.gpu;
+; CHECK-NEXT: ld.acquire.gpu.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr %addr syncscope("device") seq_cst, align 1
+ ret i8 %retval
+}
+
+define i8 @seq_cst_i8_generic_sys_volatile(ptr %addr) {
+; CHECK-LABEL: seq_cst_i8_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_generic_sys_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr %addr syncscope("") seq_cst, align 1
+ ret i8 %retval
+}
+
+define i8 @seq_cst_i8_generic_cta_volatile(ptr %addr) {
+; CHECK-LABEL: seq_cst_i8_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_generic_cta_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr %addr syncscope("block") seq_cst, align 1
+ ret i8 %retval
+}
+
+define i8 @seq_cst_i8_generic_cluster_volatile(ptr %addr) {
+; CHECK-LABEL: seq_cst_i8_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_generic_cluster_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr %addr syncscope("cluster") seq_cst, align 1
+ ret i8 %retval
+}
+
+define i8 @seq_cst_i8_generic_gpu_volatile(ptr %addr) {
+; CHECK-LABEL: seq_cst_i8_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_generic_gpu_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr %addr syncscope("device") seq_cst, align 1
+ ret i8 %retval
+}
+
+define i16 @seq_cst_i16_generic_sys(ptr %addr) {
+; CHECK-LABEL: seq_cst_i16_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_generic_sys_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr %addr syncscope("") seq_cst, align 2
+ ret i16 %retval
+}
+
+define i16 @seq_cst_i16_generic_cta(ptr %addr) {
+; CHECK-LABEL: seq_cst_i16_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_generic_cta_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.acquire.cta.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr %addr syncscope("block") seq_cst, align 2
+ ret i16 %retval
+}
+
+define i16 @seq_cst_i16_generic_cluster(ptr %addr) {
+; CHECK-LABEL: seq_cst_i16_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_generic_cluster_param_0];
+; CHECK-NEXT: fence.sc.cluster;
+; CHECK-NEXT: ld.acquire.cluster.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr %addr syncscope("cluster") seq_cst, align 2
+ ret i16 %retval
+}
+
+define i16 @seq_cst_i16_generic_gpu(ptr %addr) {
+; CHECK-LABEL: seq_cst_i16_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_generic_gpu_param_0];
+; CHECK-NEXT: fence.sc.gpu;
+; CHECK-NEXT: ld.acquire.gpu.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr %addr syncscope("device") seq_cst, align 2
+ ret i16 %retval
+}
+
+define i16 @seq_cst_i16_generic_sys_volatile(ptr %addr) {
+; CHECK-LABEL: seq_cst_i16_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_generic_sys_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr %addr syncscope("") seq_cst, align 2
+ ret i16 %retval
+}
+
+define i16 @seq_cst_i16_generic_cta_volatile(ptr %addr) {
+; CHECK-LABEL: seq_cst_i16_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_generic_cta_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr %addr syncscope("block") seq_cst, align 2
+ ret i16 %retval
+}
+
+define i16 @seq_cst_i16_generic_cluster_volatile(ptr %addr) {
+; CHECK-LABEL: seq_cst_i16_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_generic_cluster_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr %addr syncscope("cluster") seq_cst, align 2
+ ret i16 %retval
+}
+
+define i16 @seq_cst_i16_generic_gpu_volatile(ptr %addr) {
+; CHECK-LABEL: seq_cst_i16_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_generic_gpu_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr %addr syncscope("device") seq_cst, align 2
+ ret i16 %retval
+}
+
+define i32 @seq_cst_i32_generic_sys(ptr %addr) {
+; CHECK-LABEL: seq_cst_i32_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_generic_sys_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr %addr syncscope("") seq_cst, align 4
+ ret i32 %retval
+}
+
+define i32 @seq_cst_i32_generic_cta(ptr %addr) {
+; CHECK-LABEL: seq_cst_i32_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_generic_cta_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.acquire.cta.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr %addr syncscope("block") seq_cst, align 4
+ ret i32 %retval
+}
+
+define i32 @seq_cst_i32_generic_cluster(ptr %addr) {
+; CHECK-LABEL: seq_cst_i32_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_generic_cluster_param_0];
+; CHECK-NEXT: fence.sc.cluster;
+; CHECK-NEXT: ld.acquire.cluster.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr %addr syncscope("cluster") seq_cst, align 4
+ ret i32 %retval
+}
+
+define i32 @seq_cst_i32_generic_gpu(ptr %addr) {
+; CHECK-LABEL: seq_cst_i32_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_generic_gpu_param_0];
+; CHECK-NEXT: fence.sc.gpu;
+; CHECK-NEXT: ld.acquire.gpu.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr %addr syncscope("device") seq_cst, align 4
+ ret i32 %retval
+}
+
+define i32 @seq_cst_i32_generic_sys_volatile(ptr %addr) {
+; CHECK-LABEL: seq_cst_i32_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_generic_sys_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr %addr syncscope("") seq_cst, align 4
+ ret i32 %retval
+}
+
+define i32 @seq_cst_i32_generic_cta_volatile(ptr %addr) {
+; CHECK-LABEL: seq_cst_i32_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_generic_cta_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr %addr syncscope("block") seq_cst, align 4
+ ret i32 %retval
+}
+
+define i32 @seq_cst_i32_generic_cluster_volatile(ptr %addr) {
+; CHECK-LABEL: seq_cst_i32_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_generic_cluster_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr %addr syncscope("cluster") seq_cst, align 4
+ ret i32 %retval
+}
+
+define i32 @seq_cst_i32_generic_gpu_volatile(ptr %addr) {
+; CHECK-LABEL: seq_cst_i32_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_generic_gpu_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr %addr syncscope("device") seq_cst, align 4
+ ret i32 %retval
+}
+
+define i64 @seq_cst_i64_generic_sys(ptr %addr) {
+; CHECK-LABEL: seq_cst_i64_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_generic_sys_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr %addr syncscope("") seq_cst, align 8
+ ret i64 %retval
+}
+
+define i64 @seq_cst_i64_generic_cta(ptr %addr) {
+; CHECK-LABEL: seq_cst_i64_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_generic_cta_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.acquire.cta.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr %addr syncscope("block") seq_cst, align 8
+ ret i64 %retval
+}
+
+define i64 @seq_cst_i64_generic_cluster(ptr %addr) {
+; CHECK-LABEL: seq_cst_i64_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_generic_cluster_param_0];
+; CHECK-NEXT: fence.sc.cluster;
+; CHECK-NEXT: ld.acquire.cluster.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr %addr syncscope("cluster") seq_cst, align 8
+ ret i64 %retval
+}
+
+define i64 @seq_cst_i64_generic_gpu(ptr %addr) {
+; CHECK-LABEL: seq_cst_i64_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_generic_gpu_param_0];
+; CHECK-NEXT: fence.sc.gpu;
+; CHECK-NEXT: ld.acquire.gpu.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr %addr syncscope("device") seq_cst, align 8
+ ret i64 %retval
+}
+
+define i64 @seq_cst_i64_generic_sys_volatile(ptr %addr) {
+; CHECK-LABEL: seq_cst_i64_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_generic_sys_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr %addr syncscope("") seq_cst, align 8
+ ret i64 %retval
+}
+
+define i64 @seq_cst_i64_generic_cta_volatile(ptr %addr) {
+; CHECK-LABEL: seq_cst_i64_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_generic_cta_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr %addr syncscope("block") seq_cst, align 8
+ ret i64 %retval
+}
+
+define i64 @seq_cst_i64_generic_cluster_volatile(ptr %addr) {
+; CHECK-LABEL: seq_cst_i64_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_generic_cluster_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr %addr syncscope("cluster") seq_cst, align 8
+ ret i64 %retval
+}
+
+define i64 @seq_cst_i64_generic_gpu_volatile(ptr %addr) {
+; CHECK-LABEL: seq_cst_i64_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_generic_gpu_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr %addr syncscope("device") seq_cst, align 8
+ ret i64 %retval
+}
+
+define i128 @seq_cst_i128_generic_sys(ptr %addr) {
+; CHECK-LABEL: seq_cst_i128_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i128_generic_sys_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: mov.b64 %rd2, 0;
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 cmp, swap, dst;
+; CHECK-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; CHECK-NEXT: mov.b128 swap, {%rd2, %rd2};
+; CHECK-NEXT: atom.acquire.sys.cas.b128 dst, [%rd1], cmp, swap;
+; CHECK-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; CHECK-NEXT: ret;
+ %retval = load atomic i128, ptr %addr syncscope("") seq_cst, align 16
+ ret i128 %retval
+}
+
+define i128 @seq_cst_i128_generic_cta(ptr %addr) {
+; CHECK-LABEL: seq_cst_i128_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i128_generic_cta_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: mov.b64 %rd2, 0;
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 cmp, swap, dst;
+; CHECK-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; CHECK-NEXT: mov.b128 swap, {%rd2, %rd2};
+; CHECK-NEXT: atom.acquire.cta.cas.b128 dst, [%rd1], cmp, swap;
+; CHECK-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; CHECK-NEXT: ret;
+ %retval = load atomic i128, ptr %addr syncscope("block") seq_cst, align 16
+ ret i128 %retval
+}
+
+define i128 @seq_cst_i128_generic_cluster(ptr %addr) {
+; CHECK-LABEL: seq_cst_i128_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i128_generic_cluster_param_0];
+; CHECK-NEXT: fence.sc.cluster;
+; CHECK-NEXT: mov.b64 %rd2, 0;
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 cmp, swap, dst;
+; CHECK-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; CHECK-NEXT: mov.b128 swap, {%rd2, %rd2};
+; CHECK-NEXT: atom.acquire.cluster.cas.b128 dst, [%rd1], cmp, swap;
+; CHECK-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; CHECK-NEXT: ret;
+ %retval = load atomic i128, ptr %addr syncscope("cluster") seq_cst, align 16
+ ret i128 %retval
+}
+
+define i128 @seq_cst_i128_generic_gpu(ptr %addr) {
+; CHECK-LABEL: seq_cst_i128_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i128_generic_gpu_param_0];
+; CHECK-NEXT: fence.sc.gpu;
+; CHECK-NEXT: mov.b64 %rd2, 0;
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 cmp, swap, dst;
+; CHECK-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; CHECK-NEXT: mov.b128 swap, {%rd2, %rd2};
+; CHECK-NEXT: atom.acquire.gpu.cas.b128 dst, [%rd1], cmp, swap;
+; CHECK-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; CHECK-NEXT: ret;
+ %retval = load atomic i128, ptr %addr syncscope("device") seq_cst, align 16
+ ret i128 %retval
+}
+
+define i128 @seq_cst_i128_generic_sys_volatile(ptr %addr) {
+; CHECK-LABEL: seq_cst_i128_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i128_generic_sys_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: mov.b64 %rd2, 0;
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 cmp, swap, dst;
+; CHECK-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; CHECK-NEXT: mov.b128 swap, {%rd2, %rd2};
+; CHECK-NEXT: atom.acquire.sys.cas.b128 dst, [%rd1], cmp, swap;
+; CHECK-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i128, ptr %addr syncscope("") seq_cst, align 16
+ ret i128 %retval
+}
+
+define i128 @seq_cst_i128_generic_cta_volatile(ptr %addr) {
+; CHECK-LABEL: seq_cst_i128_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i128_generic_cta_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: mov.b64 %rd2, 0;
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 cmp, swap, dst;
+; CHECK-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; CHECK-NEXT: mov.b128 swap, {%rd2, %rd2};
+; CHECK-NEXT: atom.acquire.cta.cas.b128 dst, [%rd1], cmp, swap;
+; CHECK-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i128, ptr %addr syncscope("block") seq_cst, align 16
+ ret i128 %retval
+}
+
+define i128 @seq_cst_i128_generic_cluster_volatile(ptr %addr) {
+; CHECK-LABEL: seq_cst_i128_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i128_generic_cluster_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: mov.b64 %rd2, 0;
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 cmp, swap, dst;
+; CHECK-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; CHECK-NEXT: mov.b128 swap, {%rd2, %rd2};
+; CHECK-NEXT: atom.acquire.cluster.cas.b128 dst, [%rd1], cmp, swap;
+; CHECK-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i128, ptr %addr syncscope("cluster") seq_cst, align 16
+ ret i128 %retval
+}
+
+define i128 @seq_cst_i128_generic_gpu_volatile(ptr %addr) {
+; CHECK-LABEL: seq_cst_i128_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i128_generic_gpu_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: mov.b64 %rd2, 0;
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 cmp, swap, dst;
+; CHECK-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; CHECK-NEXT: mov.b128 swap, {%rd2, %rd2};
+; CHECK-NEXT: atom.acquire.gpu.cas.b128 dst, [%rd1], cmp, swap;
+; CHECK-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i128, ptr %addr syncscope("device") seq_cst, align 16
+ ret i128 %retval
+}
+
+define half @seq_cst_half_generic_sys(ptr %addr) {
+; CHECK-LABEL: seq_cst_half_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_generic_sys_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr %addr syncscope("") seq_cst, align 2
+ ret half %retval
+}
+
+define half @seq_cst_half_generic_cta(ptr %addr) {
+; CHECK-LABEL: seq_cst_half_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_generic_cta_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.acquire.cta.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr %addr syncscope("block") seq_cst, align 2
+ ret half %retval
+}
+
+define half @seq_cst_half_generic_cluster(ptr %addr) {
+; CHECK-LABEL: seq_cst_half_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_generic_cluster_param_0];
+; CHECK-NEXT: fence.sc.cluster;
+; CHECK-NEXT: ld.acquire.cluster.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr %addr syncscope("cluster") seq_cst, align 2
+ ret half %retval
+}
+
+define half @seq_cst_half_generic_gpu(ptr %addr) {
+; CHECK-LABEL: seq_cst_half_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_generic_gpu_param_0];
+; CHECK-NEXT: fence.sc.gpu;
+; CHECK-NEXT: ld.acquire.gpu.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr %addr syncscope("device") seq_cst, align 2
+ ret half %retval
+}
+
+define half @seq_cst_half_generic_sys_volatile(ptr %addr) {
+; CHECK-LABEL: seq_cst_half_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_generic_sys_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr %addr syncscope("") seq_cst, align 2
+ ret half %retval
+}
+
+define half @seq_cst_half_generic_cta_volatile(ptr %addr) {
+; CHECK-LABEL: seq_cst_half_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_generic_cta_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr %addr syncscope("block") seq_cst, align 2
+ ret half %retval
+}
+
+define half @seq_cst_half_generic_cluster_volatile(ptr %addr) {
+; CHECK-LABEL: seq_cst_half_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_generic_cluster_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr %addr syncscope("cluster") seq_cst, align 2
+ ret half %retval
+}
+
+define half @seq_cst_half_generic_gpu_volatile(ptr %addr) {
+; CHECK-LABEL: seq_cst_half_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_generic_gpu_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr %addr syncscope("device") seq_cst, align 2
+ ret half %retval
+}
+
+define bfloat @seq_cst_bfloat_generic_sys(ptr %addr) {
+; CHECK-LABEL: seq_cst_bfloat_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_generic_sys_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr %addr syncscope("") seq_cst, align 2
+ ret bfloat %retval
+}
+
+define bfloat @seq_cst_bfloat_generic_cta(ptr %addr) {
+; CHECK-LABEL: seq_cst_bfloat_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_generic_cta_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.acquire.cta.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr %addr syncscope("block") seq_cst, align 2
+ ret bfloat %retval
+}
+
+define bfloat @seq_cst_bfloat_generic_cluster(ptr %addr) {
+; CHECK-LABEL: seq_cst_bfloat_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_generic_cluster_param_0];
+; CHECK-NEXT: fence.sc.cluster;
+; CHECK-NEXT: ld.acquire.cluster.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr %addr syncscope("cluster") seq_cst, align 2
+ ret bfloat %retval
+}
+
+define bfloat @seq_cst_bfloat_generic_gpu(ptr %addr) {
+; CHECK-LABEL: seq_cst_bfloat_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_generic_gpu_param_0];
+; CHECK-NEXT: fence.sc.gpu;
+; CHECK-NEXT: ld.acquire.gpu.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr %addr syncscope("device") seq_cst, align 2
+ ret bfloat %retval
+}
+
+define bfloat @seq_cst_bfloat_generic_sys_volatile(ptr %addr) {
+; CHECK-LABEL: seq_cst_bfloat_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_generic_sys_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr %addr syncscope("") seq_cst, align 2
+ ret bfloat %retval
+}
+
+define bfloat @seq_cst_bfloat_generic_cta_volatile(ptr %addr) {
+; CHECK-LABEL: seq_cst_bfloat_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_generic_cta_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr %addr syncscope("block") seq_cst, align 2
+ ret bfloat %retval
+}
+
+define bfloat @seq_cst_bfloat_generic_cluster_volatile(ptr %addr) {
+; CHECK-LABEL: seq_cst_bfloat_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_generic_cluster_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr %addr syncscope("cluster") seq_cst, align 2
+ ret bfloat %retval
+}
+
+define bfloat @seq_cst_bfloat_generic_gpu_volatile(ptr %addr) {
+; CHECK-LABEL: seq_cst_bfloat_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_generic_gpu_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr %addr syncscope("device") seq_cst, align 2
+ ret bfloat %retval
+}
+
+define float @seq_cst_float_generic_sys(ptr %addr) {
+; CHECK-LABEL: seq_cst_float_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_generic_sys_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr %addr syncscope("") seq_cst, align 4
+ ret float %retval
+}
+
+define float @seq_cst_float_generic_cta(ptr %addr) {
+; CHECK-LABEL: seq_cst_float_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_generic_cta_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.acquire.cta.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr %addr syncscope("block") seq_cst, align 4
+ ret float %retval
+}
+
+define float @seq_cst_float_generic_cluster(ptr %addr) {
+; CHECK-LABEL: seq_cst_float_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_generic_cluster_param_0];
+; CHECK-NEXT: fence.sc.cluster;
+; CHECK-NEXT: ld.acquire.cluster.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr %addr syncscope("cluster") seq_cst, align 4
+ ret float %retval
+}
+
+define float @seq_cst_float_generic_gpu(ptr %addr) {
+; CHECK-LABEL: seq_cst_float_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_generic_gpu_param_0];
+; CHECK-NEXT: fence.sc.gpu;
+; CHECK-NEXT: ld.acquire.gpu.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr %addr syncscope("device") seq_cst, align 4
+ ret float %retval
+}
+
+define float @seq_cst_float_generic_sys_volatile(ptr %addr) {
+; CHECK-LABEL: seq_cst_float_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_generic_sys_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr %addr syncscope("") seq_cst, align 4
+ ret float %retval
+}
+
+define float @seq_cst_float_generic_cta_volatile(ptr %addr) {
+; CHECK-LABEL: seq_cst_float_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_generic_cta_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr %addr syncscope("block") seq_cst, align 4
+ ret float %retval
+}
+
+define float @seq_cst_float_generic_cluster_volatile(ptr %addr) {
+; CHECK-LABEL: seq_cst_float_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_generic_cluster_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr %addr syncscope("cluster") seq_cst, align 4
+ ret float %retval
+}
+
+define float @seq_cst_float_generic_gpu_volatile(ptr %addr) {
+; CHECK-LABEL: seq_cst_float_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_generic_gpu_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr %addr syncscope("device") seq_cst, align 4
+ ret float %retval
+}
+
+define double @seq_cst_double_generic_sys(ptr %addr) {
+; CHECK-LABEL: seq_cst_double_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_generic_sys_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr %addr syncscope("") seq_cst, align 8
+ ret double %retval
+}
+
+define double @seq_cst_double_generic_cta(ptr %addr) {
+; CHECK-LABEL: seq_cst_double_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_generic_cta_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.acquire.cta.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr %addr syncscope("block") seq_cst, align 8
+ ret double %retval
+}
+
+define double @seq_cst_double_generic_cluster(ptr %addr) {
+; CHECK-LABEL: seq_cst_double_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_generic_cluster_param_0];
+; CHECK-NEXT: fence.sc.cluster;
+; CHECK-NEXT: ld.acquire.cluster.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr %addr syncscope("cluster") seq_cst, align 8
+ ret double %retval
+}
+
+define double @seq_cst_double_generic_gpu(ptr %addr) {
+; CHECK-LABEL: seq_cst_double_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_generic_gpu_param_0];
+; CHECK-NEXT: fence.sc.gpu;
+; CHECK-NEXT: ld.acquire.gpu.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr %addr syncscope("device") seq_cst, align 8
+ ret double %retval
+}
+
+define double @seq_cst_double_generic_sys_volatile(ptr %addr) {
+; CHECK-LABEL: seq_cst_double_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_generic_sys_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr %addr syncscope("") seq_cst, align 8
+ ret double %retval
+}
+
+define double @seq_cst_double_generic_cta_volatile(ptr %addr) {
+; CHECK-LABEL: seq_cst_double_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_generic_cta_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr %addr syncscope("block") seq_cst, align 8
+ ret double %retval
+}
+
+define double @seq_cst_double_generic_cluster_volatile(ptr %addr) {
+; CHECK-LABEL: seq_cst_double_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_generic_cluster_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr %addr syncscope("cluster") seq_cst, align 8
+ ret double %retval
+}
+
+define double @seq_cst_double_generic_gpu_volatile(ptr %addr) {
+; CHECK-LABEL: seq_cst_double_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_generic_gpu_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr %addr syncscope("device") seq_cst, align 8
+ ret double %retval
+}
+
+define i8 @seq_cst_i8_global_sys(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i8_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_global_sys_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.global.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr addrspace(1) %addr syncscope("") seq_cst, align 1
+ ret i8 %retval
+}
+
+define i8 @seq_cst_i8_global_cta(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i8_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_global_cta_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.acquire.cta.global.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr addrspace(1) %addr syncscope("block") seq_cst, align 1
+ ret i8 %retval
+}
+
+define i8 @seq_cst_i8_global_cluster(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i8_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_global_cluster_param_0];
+; CHECK-NEXT: fence.sc.cluster;
+; CHECK-NEXT: ld.acquire.cluster.global.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr addrspace(1) %addr syncscope("cluster") seq_cst, align 1
+ ret i8 %retval
+}
+
+define i8 @seq_cst_i8_global_gpu(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i8_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_global_gpu_param_0];
+; CHECK-NEXT: fence.sc.gpu;
+; CHECK-NEXT: ld.acquire.gpu.global.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr addrspace(1) %addr syncscope("device") seq_cst, align 1
+ ret i8 %retval
+}
+
+define i8 @seq_cst_i8_global_sys_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i8_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_global_sys_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.global.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr addrspace(1) %addr syncscope("") seq_cst, align 1
+ ret i8 %retval
+}
+
+define i8 @seq_cst_i8_global_cta_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i8_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_global_cta_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.global.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr addrspace(1) %addr syncscope("block") seq_cst, align 1
+ ret i8 %retval
+}
+
+define i8 @seq_cst_i8_global_cluster_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i8_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_global_cluster_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.global.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr addrspace(1) %addr syncscope("cluster") seq_cst, align 1
+ ret i8 %retval
+}
+
+define i8 @seq_cst_i8_global_gpu_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i8_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_global_gpu_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.global.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr addrspace(1) %addr syncscope("device") seq_cst, align 1
+ ret i8 %retval
+}
+
+define i16 @seq_cst_i16_global_sys(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i16_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_global_sys_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr addrspace(1) %addr syncscope("") seq_cst, align 2
+ ret i16 %retval
+}
+
+define i16 @seq_cst_i16_global_cta(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i16_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_global_cta_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.acquire.cta.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr addrspace(1) %addr syncscope("block") seq_cst, align 2
+ ret i16 %retval
+}
+
+define i16 @seq_cst_i16_global_cluster(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i16_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_global_cluster_param_0];
+; CHECK-NEXT: fence.sc.cluster;
+; CHECK-NEXT: ld.acquire.cluster.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr addrspace(1) %addr syncscope("cluster") seq_cst, align 2
+ ret i16 %retval
+}
+
+define i16 @seq_cst_i16_global_gpu(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i16_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_global_gpu_param_0];
+; CHECK-NEXT: fence.sc.gpu;
+; CHECK-NEXT: ld.acquire.gpu.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr addrspace(1) %addr syncscope("device") seq_cst, align 2
+ ret i16 %retval
+}
+
+define i16 @seq_cst_i16_global_sys_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i16_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_global_sys_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr addrspace(1) %addr syncscope("") seq_cst, align 2
+ ret i16 %retval
+}
+
+define i16 @seq_cst_i16_global_cta_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i16_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_global_cta_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr addrspace(1) %addr syncscope("block") seq_cst, align 2
+ ret i16 %retval
+}
+
+define i16 @seq_cst_i16_global_cluster_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i16_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_global_cluster_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr addrspace(1) %addr syncscope("cluster") seq_cst, align 2
+ ret i16 %retval
+}
+
+define i16 @seq_cst_i16_global_gpu_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i16_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_global_gpu_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr addrspace(1) %addr syncscope("device") seq_cst, align 2
+ ret i16 %retval
+}
+
+define i32 @seq_cst_i32_global_sys(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i32_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_global_sys_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr addrspace(1) %addr syncscope("") seq_cst, align 4
+ ret i32 %retval
+}
+
+define i32 @seq_cst_i32_global_cta(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i32_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_global_cta_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.acquire.cta.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr addrspace(1) %addr syncscope("block") seq_cst, align 4
+ ret i32 %retval
+}
+
+define i32 @seq_cst_i32_global_cluster(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i32_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_global_cluster_param_0];
+; CHECK-NEXT: fence.sc.cluster;
+; CHECK-NEXT: ld.acquire.cluster.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr addrspace(1) %addr syncscope("cluster") seq_cst, align 4
+ ret i32 %retval
+}
+
+define i32 @seq_cst_i32_global_gpu(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i32_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_global_gpu_param_0];
+; CHECK-NEXT: fence.sc.gpu;
+; CHECK-NEXT: ld.acquire.gpu.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr addrspace(1) %addr syncscope("device") seq_cst, align 4
+ ret i32 %retval
+}
+
+define i32 @seq_cst_i32_global_sys_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i32_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_global_sys_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr addrspace(1) %addr syncscope("") seq_cst, align 4
+ ret i32 %retval
+}
+
+define i32 @seq_cst_i32_global_cta_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i32_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_global_cta_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr addrspace(1) %addr syncscope("block") seq_cst, align 4
+ ret i32 %retval
+}
+
+define i32 @seq_cst_i32_global_cluster_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i32_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_global_cluster_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr addrspace(1) %addr syncscope("cluster") seq_cst, align 4
+ ret i32 %retval
+}
+
+define i32 @seq_cst_i32_global_gpu_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i32_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_global_gpu_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr addrspace(1) %addr syncscope("device") seq_cst, align 4
+ ret i32 %retval
+}
+
+define i64 @seq_cst_i64_global_sys(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i64_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_global_sys_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr addrspace(1) %addr syncscope("") seq_cst, align 8
+ ret i64 %retval
+}
+
+define i64 @seq_cst_i64_global_cta(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i64_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_global_cta_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.acquire.cta.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr addrspace(1) %addr syncscope("block") seq_cst, align 8
+ ret i64 %retval
+}
+
+define i64 @seq_cst_i64_global_cluster(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i64_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_global_cluster_param_0];
+; CHECK-NEXT: fence.sc.cluster;
+; CHECK-NEXT: ld.acquire.cluster.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr addrspace(1) %addr syncscope("cluster") seq_cst, align 8
+ ret i64 %retval
+}
+
+define i64 @seq_cst_i64_global_gpu(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i64_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_global_gpu_param_0];
+; CHECK-NEXT: fence.sc.gpu;
+; CHECK-NEXT: ld.acquire.gpu.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr addrspace(1) %addr syncscope("device") seq_cst, align 8
+ ret i64 %retval
+}
+
+define i64 @seq_cst_i64_global_sys_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i64_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_global_sys_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr addrspace(1) %addr syncscope("") seq_cst, align 8
+ ret i64 %retval
+}
+
+define i64 @seq_cst_i64_global_cta_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i64_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_global_cta_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr addrspace(1) %addr syncscope("block") seq_cst, align 8
+ ret i64 %retval
+}
+
+define i64 @seq_cst_i64_global_cluster_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i64_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_global_cluster_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr addrspace(1) %addr syncscope("cluster") seq_cst, align 8
+ ret i64 %retval
+}
+
+define i64 @seq_cst_i64_global_gpu_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i64_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_global_gpu_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr addrspace(1) %addr syncscope("device") seq_cst, align 8
+ ret i64 %retval
+}
+
+define i128 @seq_cst_i128_global_sys(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i128_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i128_global_sys_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: mov.b64 %rd2, 0;
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 cmp, swap, dst;
+; CHECK-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; CHECK-NEXT: mov.b128 swap, {%rd2, %rd2};
+; CHECK-NEXT: atom.acquire.sys.global.cas.b128 dst, [%rd1], cmp, swap;
+; CHECK-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; CHECK-NEXT: ret;
+ %retval = load atomic i128, ptr addrspace(1) %addr syncscope("") seq_cst, align 16
+ ret i128 %retval
+}
+
+define i128 @seq_cst_i128_global_cta(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i128_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i128_global_cta_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: mov.b64 %rd2, 0;
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 cmp, swap, dst;
+; CHECK-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; CHECK-NEXT: mov.b128 swap, {%rd2, %rd2};
+; CHECK-NEXT: atom.acquire.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; CHECK-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; CHECK-NEXT: ret;
+ %retval = load atomic i128, ptr addrspace(1) %addr syncscope("block") seq_cst, align 16
+ ret i128 %retval
+}
+
+define i128 @seq_cst_i128_global_cluster(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i128_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i128_global_cluster_param_0];
+; CHECK-NEXT: fence.sc.cluster;
+; CHECK-NEXT: mov.b64 %rd2, 0;
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 cmp, swap, dst;
+; CHECK-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; CHECK-NEXT: mov.b128 swap, {%rd2, %rd2};
+; CHECK-NEXT: atom.acquire.cluster.global.cas.b128 dst, [%rd1], cmp, swap;
+; CHECK-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; CHECK-NEXT: ret;
+ %retval = load atomic i128, ptr addrspace(1) %addr syncscope("cluster") seq_cst, align 16
+ ret i128 %retval
+}
+
+define i128 @seq_cst_i128_global_gpu(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i128_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i128_global_gpu_param_0];
+; CHECK-NEXT: fence.sc.gpu;
+; CHECK-NEXT: mov.b64 %rd2, 0;
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 cmp, swap, dst;
+; CHECK-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; CHECK-NEXT: mov.b128 swap, {%rd2, %rd2};
+; CHECK-NEXT: atom.acquire.gpu.global.cas.b128 dst, [%rd1], cmp, swap;
+; CHECK-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; CHECK-NEXT: ret;
+ %retval = load atomic i128, ptr addrspace(1) %addr syncscope("device") seq_cst, align 16
+ ret i128 %retval
+}
+
+define i128 @seq_cst_i128_global_sys_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i128_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i128_global_sys_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: mov.b64 %rd2, 0;
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 cmp, swap, dst;
+; CHECK-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; CHECK-NEXT: mov.b128 swap, {%rd2, %rd2};
+; CHECK-NEXT: atom.acquire.sys.global.cas.b128 dst, [%rd1], cmp, swap;
+; CHECK-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i128, ptr addrspace(1) %addr syncscope("") seq_cst, align 16
+ ret i128 %retval
+}
+
+define i128 @seq_cst_i128_global_cta_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i128_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i128_global_cta_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: mov.b64 %rd2, 0;
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 cmp, swap, dst;
+; CHECK-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; CHECK-NEXT: mov.b128 swap, {%rd2, %rd2};
+; CHECK-NEXT: atom.acquire.cta.global.cas.b128 dst, [%rd1], cmp, swap;
+; CHECK-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i128, ptr addrspace(1) %addr syncscope("block") seq_cst, align 16
+ ret i128 %retval
+}
+
+define i128 @seq_cst_i128_global_cluster_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i128_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i128_global_cluster_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: mov.b64 %rd2, 0;
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 cmp, swap, dst;
+; CHECK-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; CHECK-NEXT: mov.b128 swap, {%rd2, %rd2};
+; CHECK-NEXT: atom.acquire.cluster.global.cas.b128 dst, [%rd1], cmp, swap;
+; CHECK-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i128, ptr addrspace(1) %addr syncscope("cluster") seq_cst, align 16
+ ret i128 %retval
+}
+
+define i128 @seq_cst_i128_global_gpu_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i128_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i128_global_gpu_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: mov.b64 %rd2, 0;
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 cmp, swap, dst;
+; CHECK-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; CHECK-NEXT: mov.b128 swap, {%rd2, %rd2};
+; CHECK-NEXT: atom.acquire.gpu.global.cas.b128 dst, [%rd1], cmp, swap;
+; CHECK-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i128, ptr addrspace(1) %addr syncscope("device") seq_cst, align 16
+ ret i128 %retval
+}
+
+define half @seq_cst_half_global_sys(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_half_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_global_sys_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr addrspace(1) %addr syncscope("") seq_cst, align 2
+ ret half %retval
+}
+
+define half @seq_cst_half_global_cta(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_half_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_global_cta_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.acquire.cta.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr addrspace(1) %addr syncscope("block") seq_cst, align 2
+ ret half %retval
+}
+
+define half @seq_cst_half_global_cluster(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_half_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_global_cluster_param_0];
+; CHECK-NEXT: fence.sc.cluster;
+; CHECK-NEXT: ld.acquire.cluster.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr addrspace(1) %addr syncscope("cluster") seq_cst, align 2
+ ret half %retval
+}
+
+define half @seq_cst_half_global_gpu(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_half_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_global_gpu_param_0];
+; CHECK-NEXT: fence.sc.gpu;
+; CHECK-NEXT: ld.acquire.gpu.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr addrspace(1) %addr syncscope("device") seq_cst, align 2
+ ret half %retval
+}
+
+define half @seq_cst_half_global_sys_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_half_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_global_sys_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr addrspace(1) %addr syncscope("") seq_cst, align 2
+ ret half %retval
+}
+
+define half @seq_cst_half_global_cta_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_half_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_global_cta_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr addrspace(1) %addr syncscope("block") seq_cst, align 2
+ ret half %retval
+}
+
+define half @seq_cst_half_global_cluster_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_half_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_global_cluster_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr addrspace(1) %addr syncscope("cluster") seq_cst, align 2
+ ret half %retval
+}
+
+define half @seq_cst_half_global_gpu_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_half_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_global_gpu_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr addrspace(1) %addr syncscope("device") seq_cst, align 2
+ ret half %retval
+}
+
+define bfloat @seq_cst_bfloat_global_sys(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_bfloat_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_global_sys_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr addrspace(1) %addr syncscope("") seq_cst, align 2
+ ret bfloat %retval
+}
+
+define bfloat @seq_cst_bfloat_global_cta(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_bfloat_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_global_cta_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.acquire.cta.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr addrspace(1) %addr syncscope("block") seq_cst, align 2
+ ret bfloat %retval
+}
+
+define bfloat @seq_cst_bfloat_global_cluster(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_bfloat_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_global_cluster_param_0];
+; CHECK-NEXT: fence.sc.cluster;
+; CHECK-NEXT: ld.acquire.cluster.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr addrspace(1) %addr syncscope("cluster") seq_cst, align 2
+ ret bfloat %retval
+}
+
+define bfloat @seq_cst_bfloat_global_gpu(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_bfloat_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_global_gpu_param_0];
+; CHECK-NEXT: fence.sc.gpu;
+; CHECK-NEXT: ld.acquire.gpu.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr addrspace(1) %addr syncscope("device") seq_cst, align 2
+ ret bfloat %retval
+}
+
+define bfloat @seq_cst_bfloat_global_sys_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_bfloat_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_global_sys_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr addrspace(1) %addr syncscope("") seq_cst, align 2
+ ret bfloat %retval
+}
+
+define bfloat @seq_cst_bfloat_global_cta_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_bfloat_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_global_cta_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr addrspace(1) %addr syncscope("block") seq_cst, align 2
+ ret bfloat %retval
+}
+
+define bfloat @seq_cst_bfloat_global_cluster_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_bfloat_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_global_cluster_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr addrspace(1) %addr syncscope("cluster") seq_cst, align 2
+ ret bfloat %retval
+}
+
+define bfloat @seq_cst_bfloat_global_gpu_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_bfloat_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_global_gpu_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.global.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr addrspace(1) %addr syncscope("device") seq_cst, align 2
+ ret bfloat %retval
+}
+
+define float @seq_cst_float_global_sys(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_float_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_global_sys_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr addrspace(1) %addr syncscope("") seq_cst, align 4
+ ret float %retval
+}
+
+define float @seq_cst_float_global_cta(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_float_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_global_cta_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.acquire.cta.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr addrspace(1) %addr syncscope("block") seq_cst, align 4
+ ret float %retval
+}
+
+define float @seq_cst_float_global_cluster(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_float_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_global_cluster_param_0];
+; CHECK-NEXT: fence.sc.cluster;
+; CHECK-NEXT: ld.acquire.cluster.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr addrspace(1) %addr syncscope("cluster") seq_cst, align 4
+ ret float %retval
+}
+
+define float @seq_cst_float_global_gpu(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_float_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_global_gpu_param_0];
+; CHECK-NEXT: fence.sc.gpu;
+; CHECK-NEXT: ld.acquire.gpu.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr addrspace(1) %addr syncscope("device") seq_cst, align 4
+ ret float %retval
+}
+
+define float @seq_cst_float_global_sys_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_float_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_global_sys_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr addrspace(1) %addr syncscope("") seq_cst, align 4
+ ret float %retval
+}
+
+define float @seq_cst_float_global_cta_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_float_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_global_cta_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr addrspace(1) %addr syncscope("block") seq_cst, align 4
+ ret float %retval
+}
+
+define float @seq_cst_float_global_cluster_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_float_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_global_cluster_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr addrspace(1) %addr syncscope("cluster") seq_cst, align 4
+ ret float %retval
+}
+
+define float @seq_cst_float_global_gpu_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_float_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_global_gpu_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.global.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr addrspace(1) %addr syncscope("device") seq_cst, align 4
+ ret float %retval
+}
+
+define double @seq_cst_double_global_sys(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_double_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_global_sys_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr addrspace(1) %addr syncscope("") seq_cst, align 8
+ ret double %retval
+}
+
+define double @seq_cst_double_global_cta(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_double_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_global_cta_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.acquire.cta.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr addrspace(1) %addr syncscope("block") seq_cst, align 8
+ ret double %retval
+}
+
+define double @seq_cst_double_global_cluster(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_double_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_global_cluster_param_0];
+; CHECK-NEXT: fence.sc.cluster;
+; CHECK-NEXT: ld.acquire.cluster.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr addrspace(1) %addr syncscope("cluster") seq_cst, align 8
+ ret double %retval
+}
+
+define double @seq_cst_double_global_gpu(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_double_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_global_gpu_param_0];
+; CHECK-NEXT: fence.sc.gpu;
+; CHECK-NEXT: ld.acquire.gpu.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr addrspace(1) %addr syncscope("device") seq_cst, align 8
+ ret double %retval
+}
+
+define double @seq_cst_double_global_sys_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_double_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_global_sys_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr addrspace(1) %addr syncscope("") seq_cst, align 8
+ ret double %retval
+}
+
+define double @seq_cst_double_global_cta_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_double_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_global_cta_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr addrspace(1) %addr syncscope("block") seq_cst, align 8
+ ret double %retval
+}
+
+define double @seq_cst_double_global_cluster_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_double_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_global_cluster_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr addrspace(1) %addr syncscope("cluster") seq_cst, align 8
+ ret double %retval
+}
+
+define double @seq_cst_double_global_gpu_volatile(ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_double_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_global_gpu_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.global.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr addrspace(1) %addr syncscope("device") seq_cst, align 8
+ ret double %retval
+}
+
+define i8 @seq_cst_i8_shared_sys(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i8_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_shared_sys_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.shared.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr addrspace(3) %addr syncscope("") seq_cst, align 1
+ ret i8 %retval
+}
+
+define i8 @seq_cst_i8_shared_cta(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i8_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_shared_cta_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.acquire.cta.shared.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr addrspace(3) %addr syncscope("block") seq_cst, align 1
+ ret i8 %retval
+}
+
+define i8 @seq_cst_i8_shared_cluster(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i8_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_shared_cluster_param_0];
+; CHECK-NEXT: fence.sc.cluster;
+; CHECK-NEXT: ld.acquire.cluster.shared.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr addrspace(3) %addr syncscope("cluster") seq_cst, align 1
+ ret i8 %retval
+}
+
+define i8 @seq_cst_i8_shared_gpu(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i8_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_shared_gpu_param_0];
+; CHECK-NEXT: fence.sc.gpu;
+; CHECK-NEXT: ld.acquire.gpu.shared.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i8, ptr addrspace(3) %addr syncscope("device") seq_cst, align 1
+ ret i8 %retval
+}
+
+define i8 @seq_cst_i8_shared_sys_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i8_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_shared_sys_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.shared.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr addrspace(3) %addr syncscope("") seq_cst, align 1
+ ret i8 %retval
+}
+
+define i8 @seq_cst_i8_shared_cta_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i8_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_shared_cta_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.shared.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr addrspace(3) %addr syncscope("block") seq_cst, align 1
+ ret i8 %retval
+}
+
+define i8 @seq_cst_i8_shared_cluster_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i8_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_shared_cluster_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.shared.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr addrspace(3) %addr syncscope("cluster") seq_cst, align 1
+ ret i8 %retval
+}
+
+define i8 @seq_cst_i8_shared_gpu_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i8_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_shared_gpu_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.shared.b8 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i8, ptr addrspace(3) %addr syncscope("device") seq_cst, align 1
+ ret i8 %retval
+}
+
+define i16 @seq_cst_i16_shared_sys(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i16_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_shared_sys_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr addrspace(3) %addr syncscope("") seq_cst, align 2
+ ret i16 %retval
+}
+
+define i16 @seq_cst_i16_shared_cta(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i16_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_shared_cta_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.acquire.cta.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr addrspace(3) %addr syncscope("block") seq_cst, align 2
+ ret i16 %retval
+}
+
+define i16 @seq_cst_i16_shared_cluster(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i16_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_shared_cluster_param_0];
+; CHECK-NEXT: fence.sc.cluster;
+; CHECK-NEXT: ld.acquire.cluster.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr addrspace(3) %addr syncscope("cluster") seq_cst, align 2
+ ret i16 %retval
+}
+
+define i16 @seq_cst_i16_shared_gpu(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i16_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_shared_gpu_param_0];
+; CHECK-NEXT: fence.sc.gpu;
+; CHECK-NEXT: ld.acquire.gpu.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i16, ptr addrspace(3) %addr syncscope("device") seq_cst, align 2
+ ret i16 %retval
+}
+
+define i16 @seq_cst_i16_shared_sys_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i16_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_shared_sys_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr addrspace(3) %addr syncscope("") seq_cst, align 2
+ ret i16 %retval
+}
+
+define i16 @seq_cst_i16_shared_cta_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i16_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_shared_cta_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr addrspace(3) %addr syncscope("block") seq_cst, align 2
+ ret i16 %retval
+}
+
+define i16 @seq_cst_i16_shared_cluster_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i16_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_shared_cluster_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr addrspace(3) %addr syncscope("cluster") seq_cst, align 2
+ ret i16 %retval
+}
+
+define i16 @seq_cst_i16_shared_gpu_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i16_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_shared_gpu_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i16, ptr addrspace(3) %addr syncscope("device") seq_cst, align 2
+ ret i16 %retval
+}
+
+define i32 @seq_cst_i32_shared_sys(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i32_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_shared_sys_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr addrspace(3) %addr syncscope("") seq_cst, align 4
+ ret i32 %retval
+}
+
+define i32 @seq_cst_i32_shared_cta(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i32_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_shared_cta_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.acquire.cta.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr addrspace(3) %addr syncscope("block") seq_cst, align 4
+ ret i32 %retval
+}
+
+define i32 @seq_cst_i32_shared_cluster(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i32_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_shared_cluster_param_0];
+; CHECK-NEXT: fence.sc.cluster;
+; CHECK-NEXT: ld.acquire.cluster.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr addrspace(3) %addr syncscope("cluster") seq_cst, align 4
+ ret i32 %retval
+}
+
+define i32 @seq_cst_i32_shared_gpu(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i32_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_shared_gpu_param_0];
+; CHECK-NEXT: fence.sc.gpu;
+; CHECK-NEXT: ld.acquire.gpu.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic i32, ptr addrspace(3) %addr syncscope("device") seq_cst, align 4
+ ret i32 %retval
+}
+
+define i32 @seq_cst_i32_shared_sys_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i32_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_shared_sys_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr addrspace(3) %addr syncscope("") seq_cst, align 4
+ ret i32 %retval
+}
+
+define i32 @seq_cst_i32_shared_cta_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i32_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_shared_cta_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr addrspace(3) %addr syncscope("block") seq_cst, align 4
+ ret i32 %retval
+}
+
+define i32 @seq_cst_i32_shared_cluster_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i32_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_shared_cluster_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr addrspace(3) %addr syncscope("cluster") seq_cst, align 4
+ ret i32 %retval
+}
+
+define i32 @seq_cst_i32_shared_gpu_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i32_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_shared_gpu_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i32, ptr addrspace(3) %addr syncscope("device") seq_cst, align 4
+ ret i32 %retval
+}
+
+define i64 @seq_cst_i64_shared_sys(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i64_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_shared_sys_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr addrspace(3) %addr syncscope("") seq_cst, align 8
+ ret i64 %retval
+}
+
+define i64 @seq_cst_i64_shared_cta(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i64_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_shared_cta_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.acquire.cta.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr addrspace(3) %addr syncscope("block") seq_cst, align 8
+ ret i64 %retval
+}
+
+define i64 @seq_cst_i64_shared_cluster(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i64_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_shared_cluster_param_0];
+; CHECK-NEXT: fence.sc.cluster;
+; CHECK-NEXT: ld.acquire.cluster.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr addrspace(3) %addr syncscope("cluster") seq_cst, align 8
+ ret i64 %retval
+}
+
+define i64 @seq_cst_i64_shared_gpu(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i64_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_shared_gpu_param_0];
+; CHECK-NEXT: fence.sc.gpu;
+; CHECK-NEXT: ld.acquire.gpu.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic i64, ptr addrspace(3) %addr syncscope("device") seq_cst, align 8
+ ret i64 %retval
+}
+
+define i64 @seq_cst_i64_shared_sys_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i64_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_shared_sys_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr addrspace(3) %addr syncscope("") seq_cst, align 8
+ ret i64 %retval
+}
+
+define i64 @seq_cst_i64_shared_cta_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i64_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_shared_cta_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr addrspace(3) %addr syncscope("block") seq_cst, align 8
+ ret i64 %retval
+}
+
+define i64 @seq_cst_i64_shared_cluster_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i64_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_shared_cluster_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr addrspace(3) %addr syncscope("cluster") seq_cst, align 8
+ ret i64 %retval
+}
+
+define i64 @seq_cst_i64_shared_gpu_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i64_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_shared_gpu_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i64, ptr addrspace(3) %addr syncscope("device") seq_cst, align 8
+ ret i64 %retval
+}
+
+define i128 @seq_cst_i128_shared_sys(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i128_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i128_shared_sys_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: mov.b64 %rd2, 0;
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 cmp, swap, dst;
+; CHECK-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; CHECK-NEXT: mov.b128 swap, {%rd2, %rd2};
+; CHECK-NEXT: atom.acquire.sys.shared.cas.b128 dst, [%rd1], cmp, swap;
+; CHECK-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; CHECK-NEXT: ret;
+ %retval = load atomic i128, ptr addrspace(3) %addr syncscope("") seq_cst, align 16
+ ret i128 %retval
+}
+
+define i128 @seq_cst_i128_shared_cta(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i128_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i128_shared_cta_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: mov.b64 %rd2, 0;
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 cmp, swap, dst;
+; CHECK-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; CHECK-NEXT: mov.b128 swap, {%rd2, %rd2};
+; CHECK-NEXT: atom.acquire.cta.shared.cas.b128 dst, [%rd1], cmp, swap;
+; CHECK-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; CHECK-NEXT: ret;
+ %retval = load atomic i128, ptr addrspace(3) %addr syncscope("block") seq_cst, align 16
+ ret i128 %retval
+}
+
+define i128 @seq_cst_i128_shared_cluster(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i128_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i128_shared_cluster_param_0];
+; CHECK-NEXT: fence.sc.cluster;
+; CHECK-NEXT: mov.b64 %rd2, 0;
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 cmp, swap, dst;
+; CHECK-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; CHECK-NEXT: mov.b128 swap, {%rd2, %rd2};
+; CHECK-NEXT: atom.acquire.cluster.shared.cas.b128 dst, [%rd1], cmp, swap;
+; CHECK-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; CHECK-NEXT: ret;
+ %retval = load atomic i128, ptr addrspace(3) %addr syncscope("cluster") seq_cst, align 16
+ ret i128 %retval
+}
+
+define i128 @seq_cst_i128_shared_gpu(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i128_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i128_shared_gpu_param_0];
+; CHECK-NEXT: fence.sc.gpu;
+; CHECK-NEXT: mov.b64 %rd2, 0;
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 cmp, swap, dst;
+; CHECK-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; CHECK-NEXT: mov.b128 swap, {%rd2, %rd2};
+; CHECK-NEXT: atom.acquire.gpu.shared.cas.b128 dst, [%rd1], cmp, swap;
+; CHECK-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; CHECK-NEXT: ret;
+ %retval = load atomic i128, ptr addrspace(3) %addr syncscope("device") seq_cst, align 16
+ ret i128 %retval
+}
+
+define i128 @seq_cst_i128_shared_sys_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i128_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i128_shared_sys_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: mov.b64 %rd2, 0;
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 cmp, swap, dst;
+; CHECK-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; CHECK-NEXT: mov.b128 swap, {%rd2, %rd2};
+; CHECK-NEXT: atom.acquire.sys.shared.cas.b128 dst, [%rd1], cmp, swap;
+; CHECK-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i128, ptr addrspace(3) %addr syncscope("") seq_cst, align 16
+ ret i128 %retval
+}
+
+define i128 @seq_cst_i128_shared_cta_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i128_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i128_shared_cta_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: mov.b64 %rd2, 0;
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 cmp, swap, dst;
+; CHECK-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; CHECK-NEXT: mov.b128 swap, {%rd2, %rd2};
+; CHECK-NEXT: atom.acquire.cta.shared.cas.b128 dst, [%rd1], cmp, swap;
+; CHECK-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i128, ptr addrspace(3) %addr syncscope("block") seq_cst, align 16
+ ret i128 %retval
+}
+
+define i128 @seq_cst_i128_shared_cluster_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i128_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i128_shared_cluster_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: mov.b64 %rd2, 0;
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 cmp, swap, dst;
+; CHECK-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; CHECK-NEXT: mov.b128 swap, {%rd2, %rd2};
+; CHECK-NEXT: atom.acquire.cluster.shared.cas.b128 dst, [%rd1], cmp, swap;
+; CHECK-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i128, ptr addrspace(3) %addr syncscope("cluster") seq_cst, align 16
+ ret i128 %retval
+}
+
+define i128 @seq_cst_i128_shared_gpu_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i128_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i128_shared_gpu_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: mov.b64 %rd2, 0;
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 cmp, swap, dst;
+; CHECK-NEXT: mov.b128 cmp, {%rd2, %rd2};
+; CHECK-NEXT: mov.b128 swap, {%rd2, %rd2};
+; CHECK-NEXT: atom.acquire.gpu.shared.cas.b128 dst, [%rd1], cmp, swap;
+; CHECK-NEXT: mov.b128 {%rd3, %rd4}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile i128, ptr addrspace(3) %addr syncscope("device") seq_cst, align 16
+ ret i128 %retval
+}
+
+define half @seq_cst_half_shared_sys(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_half_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_shared_sys_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr addrspace(3) %addr syncscope("") seq_cst, align 2
+ ret half %retval
+}
+
+define half @seq_cst_half_shared_cta(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_half_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_shared_cta_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.acquire.cta.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr addrspace(3) %addr syncscope("block") seq_cst, align 2
+ ret half %retval
+}
+
+define half @seq_cst_half_shared_cluster(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_half_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_shared_cluster_param_0];
+; CHECK-NEXT: fence.sc.cluster;
+; CHECK-NEXT: ld.acquire.cluster.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr addrspace(3) %addr syncscope("cluster") seq_cst, align 2
+ ret half %retval
+}
+
+define half @seq_cst_half_shared_gpu(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_half_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_shared_gpu_param_0];
+; CHECK-NEXT: fence.sc.gpu;
+; CHECK-NEXT: ld.acquire.gpu.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic half, ptr addrspace(3) %addr syncscope("device") seq_cst, align 2
+ ret half %retval
+}
+
+define half @seq_cst_half_shared_sys_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_half_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_shared_sys_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr addrspace(3) %addr syncscope("") seq_cst, align 2
+ ret half %retval
+}
+
+define half @seq_cst_half_shared_cta_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_half_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_shared_cta_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr addrspace(3) %addr syncscope("block") seq_cst, align 2
+ ret half %retval
+}
+
+define half @seq_cst_half_shared_cluster_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_half_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_shared_cluster_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr addrspace(3) %addr syncscope("cluster") seq_cst, align 2
+ ret half %retval
+}
+
+define half @seq_cst_half_shared_gpu_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_half_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_shared_gpu_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile half, ptr addrspace(3) %addr syncscope("device") seq_cst, align 2
+ ret half %retval
+}
+
+define bfloat @seq_cst_bfloat_shared_sys(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_bfloat_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_shared_sys_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr addrspace(3) %addr syncscope("") seq_cst, align 2
+ ret bfloat %retval
+}
+
+define bfloat @seq_cst_bfloat_shared_cta(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_bfloat_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_shared_cta_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.acquire.cta.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr addrspace(3) %addr syncscope("block") seq_cst, align 2
+ ret bfloat %retval
+}
+
+define bfloat @seq_cst_bfloat_shared_cluster(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_bfloat_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_shared_cluster_param_0];
+; CHECK-NEXT: fence.sc.cluster;
+; CHECK-NEXT: ld.acquire.cluster.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr addrspace(3) %addr syncscope("cluster") seq_cst, align 2
+ ret bfloat %retval
+}
+
+define bfloat @seq_cst_bfloat_shared_gpu(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_bfloat_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_shared_gpu_param_0];
+; CHECK-NEXT: fence.sc.gpu;
+; CHECK-NEXT: ld.acquire.gpu.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic bfloat, ptr addrspace(3) %addr syncscope("device") seq_cst, align 2
+ ret bfloat %retval
+}
+
+define bfloat @seq_cst_bfloat_shared_sys_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_bfloat_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_shared_sys_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr addrspace(3) %addr syncscope("") seq_cst, align 2
+ ret bfloat %retval
+}
+
+define bfloat @seq_cst_bfloat_shared_cta_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_bfloat_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_shared_cta_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr addrspace(3) %addr syncscope("block") seq_cst, align 2
+ ret bfloat %retval
+}
+
+define bfloat @seq_cst_bfloat_shared_cluster_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_bfloat_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_shared_cluster_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr addrspace(3) %addr syncscope("cluster") seq_cst, align 2
+ ret bfloat %retval
+}
+
+define bfloat @seq_cst_bfloat_shared_gpu_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_bfloat_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_shared_gpu_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.shared.b16 %rs1, [%rd1];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile bfloat, ptr addrspace(3) %addr syncscope("device") seq_cst, align 2
+ ret bfloat %retval
+}
+
+define float @seq_cst_float_shared_sys(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_float_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_shared_sys_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr addrspace(3) %addr syncscope("") seq_cst, align 4
+ ret float %retval
+}
+
+define float @seq_cst_float_shared_cta(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_float_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_shared_cta_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.acquire.cta.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr addrspace(3) %addr syncscope("block") seq_cst, align 4
+ ret float %retval
+}
+
+define float @seq_cst_float_shared_cluster(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_float_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_shared_cluster_param_0];
+; CHECK-NEXT: fence.sc.cluster;
+; CHECK-NEXT: ld.acquire.cluster.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr addrspace(3) %addr syncscope("cluster") seq_cst, align 4
+ ret float %retval
+}
+
+define float @seq_cst_float_shared_gpu(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_float_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_shared_gpu_param_0];
+; CHECK-NEXT: fence.sc.gpu;
+; CHECK-NEXT: ld.acquire.gpu.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic float, ptr addrspace(3) %addr syncscope("device") seq_cst, align 4
+ ret float %retval
+}
+
+define float @seq_cst_float_shared_sys_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_float_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_shared_sys_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr addrspace(3) %addr syncscope("") seq_cst, align 4
+ ret float %retval
+}
+
+define float @seq_cst_float_shared_cta_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_float_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_shared_cta_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr addrspace(3) %addr syncscope("block") seq_cst, align 4
+ ret float %retval
+}
+
+define float @seq_cst_float_shared_cluster_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_float_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_shared_cluster_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr addrspace(3) %addr syncscope("cluster") seq_cst, align 4
+ ret float %retval
+}
+
+define float @seq_cst_float_shared_gpu_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_float_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_shared_gpu_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.shared.b32 %r1, [%rd1];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile float, ptr addrspace(3) %addr syncscope("device") seq_cst, align 4
+ ret float %retval
+}
+
+define double @seq_cst_double_shared_sys(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_double_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_shared_sys_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr addrspace(3) %addr syncscope("") seq_cst, align 8
+ ret double %retval
+}
+
+define double @seq_cst_double_shared_cta(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_double_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_shared_cta_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.acquire.cta.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr addrspace(3) %addr syncscope("block") seq_cst, align 8
+ ret double %retval
+}
+
+define double @seq_cst_double_shared_cluster(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_double_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_shared_cluster_param_0];
+; CHECK-NEXT: fence.sc.cluster;
+; CHECK-NEXT: ld.acquire.cluster.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr addrspace(3) %addr syncscope("cluster") seq_cst, align 8
+ ret double %retval
+}
+
+define double @seq_cst_double_shared_gpu(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_double_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_shared_gpu_param_0];
+; CHECK-NEXT: fence.sc.gpu;
+; CHECK-NEXT: ld.acquire.gpu.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic double, ptr addrspace(3) %addr syncscope("device") seq_cst, align 8
+ ret double %retval
+}
+
+define double @seq_cst_double_shared_sys_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_double_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_shared_sys_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr addrspace(3) %addr syncscope("") seq_cst, align 8
+ ret double %retval
+}
+
+define double @seq_cst_double_shared_cta_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_double_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_shared_cta_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr addrspace(3) %addr syncscope("block") seq_cst, align 8
+ ret double %retval
+}
+
+define double @seq_cst_double_shared_cluster_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_double_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_shared_cluster_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr addrspace(3) %addr syncscope("cluster") seq_cst, align 8
+ ret double %retval
+}
+
+define double @seq_cst_double_shared_gpu_volatile(ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_double_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_shared_gpu_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.shared.b64 %rd2, [%rd1];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %retval = load atomic volatile double, ptr addrspace(3) %addr syncscope("device") seq_cst, align 8
+ ret double %retval
+}
+
+define i8 @seq_cst_i8_local_sys() {
+; CHECK-LABEL: seq_cst_i8_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot1188[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1188;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b8 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i8, ptr addrspace(5) %slot syncscope("") seq_cst, align 1
+ ret i8 %retval
+}
+
+define i8 @seq_cst_i8_local_cta() {
+; CHECK-LABEL: seq_cst_i8_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot1189[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1189;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b8 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i8, ptr addrspace(5) %slot syncscope("block") seq_cst, align 1
+ ret i8 %retval
+}
+
+define i8 @seq_cst_i8_local_cluster() {
+; CHECK-LABEL: seq_cst_i8_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot1190[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1190;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b8 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i8, ptr addrspace(5) %slot syncscope("cluster") seq_cst, align 1
+ ret i8 %retval
+}
+
+define i8 @seq_cst_i8_local_gpu() {
+; CHECK-LABEL: seq_cst_i8_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot1191[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1191;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b8 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i8, ptr addrspace(5) %slot syncscope("device") seq_cst, align 1
+ ret i8 %retval
+}
+
+define i8 @seq_cst_i8_local_sys_volatile() {
+; CHECK-LABEL: seq_cst_i8_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot1192[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1192;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b8 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i8, ptr addrspace(5) %slot syncscope("") seq_cst, align 1
+ ret i8 %retval
+}
+
+define i8 @seq_cst_i8_local_cta_volatile() {
+; CHECK-LABEL: seq_cst_i8_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot1193[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1193;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b8 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i8, ptr addrspace(5) %slot syncscope("block") seq_cst, align 1
+ ret i8 %retval
+}
+
+define i8 @seq_cst_i8_local_cluster_volatile() {
+; CHECK-LABEL: seq_cst_i8_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot1194[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1194;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b8 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i8, ptr addrspace(5) %slot syncscope("cluster") seq_cst, align 1
+ ret i8 %retval
+}
+
+define i8 @seq_cst_i8_local_gpu_volatile() {
+; CHECK-LABEL: seq_cst_i8_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot1195[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1195;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b8 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i8, ptr addrspace(5) %slot syncscope("device") seq_cst, align 1
+ ret i8 %retval
+}
+
+define i16 @seq_cst_i16_local_sys() {
+; CHECK-LABEL: seq_cst_i16_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1196[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1196;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i16, ptr addrspace(5) %slot syncscope("") seq_cst, align 2
+ ret i16 %retval
+}
+
+define i16 @seq_cst_i16_local_cta() {
+; CHECK-LABEL: seq_cst_i16_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1197[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1197;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i16, ptr addrspace(5) %slot syncscope("block") seq_cst, align 2
+ ret i16 %retval
+}
+
+define i16 @seq_cst_i16_local_cluster() {
+; CHECK-LABEL: seq_cst_i16_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1198[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1198;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i16, ptr addrspace(5) %slot syncscope("cluster") seq_cst, align 2
+ ret i16 %retval
+}
+
+define i16 @seq_cst_i16_local_gpu() {
+; CHECK-LABEL: seq_cst_i16_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1199[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1199;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i16, ptr addrspace(5) %slot syncscope("device") seq_cst, align 2
+ ret i16 %retval
+}
+
+define i16 @seq_cst_i16_local_sys_volatile() {
+; CHECK-LABEL: seq_cst_i16_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1200[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1200;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i16, ptr addrspace(5) %slot syncscope("") seq_cst, align 2
+ ret i16 %retval
+}
+
+define i16 @seq_cst_i16_local_cta_volatile() {
+; CHECK-LABEL: seq_cst_i16_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1201[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1201;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i16, ptr addrspace(5) %slot syncscope("block") seq_cst, align 2
+ ret i16 %retval
+}
+
+define i16 @seq_cst_i16_local_cluster_volatile() {
+; CHECK-LABEL: seq_cst_i16_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1202[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1202;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i16, ptr addrspace(5) %slot syncscope("cluster") seq_cst, align 2
+ ret i16 %retval
+}
+
+define i16 @seq_cst_i16_local_gpu_volatile() {
+; CHECK-LABEL: seq_cst_i16_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1203[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1203;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i16, ptr addrspace(5) %slot syncscope("device") seq_cst, align 2
+ ret i16 %retval
+}
+
+define i32 @seq_cst_i32_local_sys() {
+; CHECK-LABEL: seq_cst_i32_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot1204[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1204;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i32, ptr addrspace(5) %slot syncscope("") seq_cst, align 4
+ ret i32 %retval
+}
+
+define i32 @seq_cst_i32_local_cta() {
+; CHECK-LABEL: seq_cst_i32_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot1205[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1205;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i32, ptr addrspace(5) %slot syncscope("block") seq_cst, align 4
+ ret i32 %retval
+}
+
+define i32 @seq_cst_i32_local_cluster() {
+; CHECK-LABEL: seq_cst_i32_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot1206[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1206;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i32, ptr addrspace(5) %slot syncscope("cluster") seq_cst, align 4
+ ret i32 %retval
+}
+
+define i32 @seq_cst_i32_local_gpu() {
+; CHECK-LABEL: seq_cst_i32_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot1207[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1207;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i32, ptr addrspace(5) %slot syncscope("device") seq_cst, align 4
+ ret i32 %retval
+}
+
+define i32 @seq_cst_i32_local_sys_volatile() {
+; CHECK-LABEL: seq_cst_i32_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot1208[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1208;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i32, ptr addrspace(5) %slot syncscope("") seq_cst, align 4
+ ret i32 %retval
+}
+
+define i32 @seq_cst_i32_local_cta_volatile() {
+; CHECK-LABEL: seq_cst_i32_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot1209[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1209;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i32, ptr addrspace(5) %slot syncscope("block") seq_cst, align 4
+ ret i32 %retval
+}
+
+define i32 @seq_cst_i32_local_cluster_volatile() {
+; CHECK-LABEL: seq_cst_i32_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot1210[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1210;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i32, ptr addrspace(5) %slot syncscope("cluster") seq_cst, align 4
+ ret i32 %retval
+}
+
+define i32 @seq_cst_i32_local_gpu_volatile() {
+; CHECK-LABEL: seq_cst_i32_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot1211[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1211;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i32, ptr addrspace(5) %slot syncscope("device") seq_cst, align 4
+ ret i32 %retval
+}
+
+define i64 @seq_cst_i64_local_sys() {
+; CHECK-LABEL: seq_cst_i64_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot1212[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1212;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i64, ptr addrspace(5) %slot syncscope("") seq_cst, align 8
+ ret i64 %retval
+}
+
+define i64 @seq_cst_i64_local_cta() {
+; CHECK-LABEL: seq_cst_i64_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot1213[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1213;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i64, ptr addrspace(5) %slot syncscope("block") seq_cst, align 8
+ ret i64 %retval
+}
+
+define i64 @seq_cst_i64_local_cluster() {
+; CHECK-LABEL: seq_cst_i64_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot1214[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1214;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i64, ptr addrspace(5) %slot syncscope("cluster") seq_cst, align 8
+ ret i64 %retval
+}
+
+define i64 @seq_cst_i64_local_gpu() {
+; CHECK-LABEL: seq_cst_i64_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot1215[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1215;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i64, ptr addrspace(5) %slot syncscope("device") seq_cst, align 8
+ ret i64 %retval
+}
+
+define i64 @seq_cst_i64_local_sys_volatile() {
+; CHECK-LABEL: seq_cst_i64_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot1216[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1216;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i64, ptr addrspace(5) %slot syncscope("") seq_cst, align 8
+ ret i64 %retval
+}
+
+define i64 @seq_cst_i64_local_cta_volatile() {
+; CHECK-LABEL: seq_cst_i64_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot1217[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1217;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i64, ptr addrspace(5) %slot syncscope("block") seq_cst, align 8
+ ret i64 %retval
+}
+
+define i64 @seq_cst_i64_local_cluster_volatile() {
+; CHECK-LABEL: seq_cst_i64_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot1218[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1218;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i64, ptr addrspace(5) %slot syncscope("cluster") seq_cst, align 8
+ ret i64 %retval
+}
+
+define i64 @seq_cst_i64_local_gpu_volatile() {
+; CHECK-LABEL: seq_cst_i64_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot1219[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1219;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i64, ptr addrspace(5) %slot syncscope("device") seq_cst, align 8
+ ret i64 %retval
+}
+
+define i128 @seq_cst_i128_local_sys() {
+; CHECK-LABEL: seq_cst_i128_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 16 .b8 __local_depot1220[16];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1220;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.v2.b64 {%rd2, %rd3}, [%SP];
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd2, %rd3};
+; CHECK-NEXT: ret;
+ %slot = alloca i128, align 16, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i128, ptr addrspace(5) %slot syncscope("") seq_cst, align 16
+ ret i128 %retval
+}
+
+define i128 @seq_cst_i128_local_cta() {
+; CHECK-LABEL: seq_cst_i128_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 16 .b8 __local_depot1221[16];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1221;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.v2.b64 {%rd2, %rd3}, [%SP];
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd2, %rd3};
+; CHECK-NEXT: ret;
+ %slot = alloca i128, align 16, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i128, ptr addrspace(5) %slot syncscope("block") seq_cst, align 16
+ ret i128 %retval
+}
+
+define i128 @seq_cst_i128_local_cluster() {
+; CHECK-LABEL: seq_cst_i128_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 16 .b8 __local_depot1222[16];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1222;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.v2.b64 {%rd2, %rd3}, [%SP];
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd2, %rd3};
+; CHECK-NEXT: ret;
+ %slot = alloca i128, align 16, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i128, ptr addrspace(5) %slot syncscope("cluster") seq_cst, align 16
+ ret i128 %retval
+}
+
+define i128 @seq_cst_i128_local_gpu() {
+; CHECK-LABEL: seq_cst_i128_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 16 .b8 __local_depot1223[16];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1223;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.v2.b64 {%rd2, %rd3}, [%SP];
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd2, %rd3};
+; CHECK-NEXT: ret;
+ %slot = alloca i128, align 16, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic i128, ptr addrspace(5) %slot syncscope("device") seq_cst, align 16
+ ret i128 %retval
+}
+
+define i128 @seq_cst_i128_local_sys_volatile() {
+; CHECK-LABEL: seq_cst_i128_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 16 .b8 __local_depot1224[16];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1224;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.v2.b64 {%rd2, %rd3}, [%SP];
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd2, %rd3};
+; CHECK-NEXT: ret;
+ %slot = alloca i128, align 16, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i128, ptr addrspace(5) %slot syncscope("") seq_cst, align 16
+ ret i128 %retval
+}
+
+define i128 @seq_cst_i128_local_cta_volatile() {
+; CHECK-LABEL: seq_cst_i128_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 16 .b8 __local_depot1225[16];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1225;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.v2.b64 {%rd2, %rd3}, [%SP];
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd2, %rd3};
+; CHECK-NEXT: ret;
+ %slot = alloca i128, align 16, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i128, ptr addrspace(5) %slot syncscope("block") seq_cst, align 16
+ ret i128 %retval
+}
+
+define i128 @seq_cst_i128_local_cluster_volatile() {
+; CHECK-LABEL: seq_cst_i128_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 16 .b8 __local_depot1226[16];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1226;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.v2.b64 {%rd2, %rd3}, [%SP];
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd2, %rd3};
+; CHECK-NEXT: ret;
+ %slot = alloca i128, align 16, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i128, ptr addrspace(5) %slot syncscope("cluster") seq_cst, align 16
+ ret i128 %retval
+}
+
+define i128 @seq_cst_i128_local_gpu_volatile() {
+; CHECK-LABEL: seq_cst_i128_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 16 .b8 __local_depot1227[16];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1227;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.v2.b64 {%rd2, %rd3}, [%SP];
+; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd2, %rd3};
+; CHECK-NEXT: ret;
+ %slot = alloca i128, align 16, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile i128, ptr addrspace(5) %slot syncscope("device") seq_cst, align 16
+ ret i128 %retval
+}
+
+define half @seq_cst_half_local_sys() {
+; CHECK-LABEL: seq_cst_half_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1228[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1228;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic half, ptr addrspace(5) %slot syncscope("") seq_cst, align 2
+ ret half %retval
+}
+
+define half @seq_cst_half_local_cta() {
+; CHECK-LABEL: seq_cst_half_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1229[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1229;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic half, ptr addrspace(5) %slot syncscope("block") seq_cst, align 2
+ ret half %retval
+}
+
+define half @seq_cst_half_local_cluster() {
+; CHECK-LABEL: seq_cst_half_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1230[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1230;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic half, ptr addrspace(5) %slot syncscope("cluster") seq_cst, align 2
+ ret half %retval
+}
+
+define half @seq_cst_half_local_gpu() {
+; CHECK-LABEL: seq_cst_half_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1231[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1231;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic half, ptr addrspace(5) %slot syncscope("device") seq_cst, align 2
+ ret half %retval
+}
+
+define half @seq_cst_half_local_sys_volatile() {
+; CHECK-LABEL: seq_cst_half_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1232[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1232;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile half, ptr addrspace(5) %slot syncscope("") seq_cst, align 2
+ ret half %retval
+}
+
+define half @seq_cst_half_local_cta_volatile() {
+; CHECK-LABEL: seq_cst_half_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1233[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1233;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile half, ptr addrspace(5) %slot syncscope("block") seq_cst, align 2
+ ret half %retval
+}
+
+define half @seq_cst_half_local_cluster_volatile() {
+; CHECK-LABEL: seq_cst_half_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1234[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1234;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile half, ptr addrspace(5) %slot syncscope("cluster") seq_cst, align 2
+ ret half %retval
+}
+
+define half @seq_cst_half_local_gpu_volatile() {
+; CHECK-LABEL: seq_cst_half_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1235[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1235;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile half, ptr addrspace(5) %slot syncscope("device") seq_cst, align 2
+ ret half %retval
+}
+
+define bfloat @seq_cst_bfloat_local_sys() {
+; CHECK-LABEL: seq_cst_bfloat_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1236[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1236;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic bfloat, ptr addrspace(5) %slot syncscope("") seq_cst, align 2
+ ret bfloat %retval
+}
+
+define bfloat @seq_cst_bfloat_local_cta() {
+; CHECK-LABEL: seq_cst_bfloat_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1237[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1237;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic bfloat, ptr addrspace(5) %slot syncscope("block") seq_cst, align 2
+ ret bfloat %retval
+}
+
+define bfloat @seq_cst_bfloat_local_cluster() {
+; CHECK-LABEL: seq_cst_bfloat_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1238[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1238;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic bfloat, ptr addrspace(5) %slot syncscope("cluster") seq_cst, align 2
+ ret bfloat %retval
+}
+
+define bfloat @seq_cst_bfloat_local_gpu() {
+; CHECK-LABEL: seq_cst_bfloat_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1239[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1239;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic bfloat, ptr addrspace(5) %slot syncscope("device") seq_cst, align 2
+ ret bfloat %retval
+}
+
+define bfloat @seq_cst_bfloat_local_sys_volatile() {
+; CHECK-LABEL: seq_cst_bfloat_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1240[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1240;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile bfloat, ptr addrspace(5) %slot syncscope("") seq_cst, align 2
+ ret bfloat %retval
+}
+
+define bfloat @seq_cst_bfloat_local_cta_volatile() {
+; CHECK-LABEL: seq_cst_bfloat_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1241[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1241;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile bfloat, ptr addrspace(5) %slot syncscope("block") seq_cst, align 2
+ ret bfloat %retval
+}
+
+define bfloat @seq_cst_bfloat_local_cluster_volatile() {
+; CHECK-LABEL: seq_cst_bfloat_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1242[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1242;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile bfloat, ptr addrspace(5) %slot syncscope("cluster") seq_cst, align 2
+ ret bfloat %retval
+}
+
+define bfloat @seq_cst_bfloat_local_gpu_volatile() {
+; CHECK-LABEL: seq_cst_bfloat_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1243[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1243;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b16 %rs1, [%SP];
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile bfloat, ptr addrspace(5) %slot syncscope("device") seq_cst, align 2
+ ret bfloat %retval
+}
+
+define float @seq_cst_float_local_sys() {
+; CHECK-LABEL: seq_cst_float_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot1244[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1244;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic float, ptr addrspace(5) %slot syncscope("") seq_cst, align 4
+ ret float %retval
+}
+
+define float @seq_cst_float_local_cta() {
+; CHECK-LABEL: seq_cst_float_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot1245[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1245;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic float, ptr addrspace(5) %slot syncscope("block") seq_cst, align 4
+ ret float %retval
+}
+
+define float @seq_cst_float_local_cluster() {
+; CHECK-LABEL: seq_cst_float_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot1246[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1246;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic float, ptr addrspace(5) %slot syncscope("cluster") seq_cst, align 4
+ ret float %retval
+}
+
+define float @seq_cst_float_local_gpu() {
+; CHECK-LABEL: seq_cst_float_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot1247[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1247;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic float, ptr addrspace(5) %slot syncscope("device") seq_cst, align 4
+ ret float %retval
+}
+
+define float @seq_cst_float_local_sys_volatile() {
+; CHECK-LABEL: seq_cst_float_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot1248[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1248;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile float, ptr addrspace(5) %slot syncscope("") seq_cst, align 4
+ ret float %retval
+}
+
+define float @seq_cst_float_local_cta_volatile() {
+; CHECK-LABEL: seq_cst_float_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot1249[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1249;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile float, ptr addrspace(5) %slot syncscope("block") seq_cst, align 4
+ ret float %retval
+}
+
+define float @seq_cst_float_local_cluster_volatile() {
+; CHECK-LABEL: seq_cst_float_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot1250[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1250;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile float, ptr addrspace(5) %slot syncscope("cluster") seq_cst, align 4
+ ret float %retval
+}
+
+define float @seq_cst_float_local_gpu_volatile() {
+; CHECK-LABEL: seq_cst_float_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot1251[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1251;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b32 %r2, [%SP];
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile float, ptr addrspace(5) %slot syncscope("device") seq_cst, align 4
+ ret float %retval
+}
+
+define double @seq_cst_double_local_sys() {
+; CHECK-LABEL: seq_cst_double_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot1252[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1252;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic double, ptr addrspace(5) %slot syncscope("") seq_cst, align 8
+ ret double %retval
+}
+
+define double @seq_cst_double_local_cta() {
+; CHECK-LABEL: seq_cst_double_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot1253[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1253;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic double, ptr addrspace(5) %slot syncscope("block") seq_cst, align 8
+ ret double %retval
+}
+
+define double @seq_cst_double_local_cluster() {
+; CHECK-LABEL: seq_cst_double_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot1254[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1254;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic double, ptr addrspace(5) %slot syncscope("cluster") seq_cst, align 8
+ ret double %retval
+}
+
+define double @seq_cst_double_local_gpu() {
+; CHECK-LABEL: seq_cst_double_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot1255[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1255;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic double, ptr addrspace(5) %slot syncscope("device") seq_cst, align 8
+ ret double %retval
+}
+
+define double @seq_cst_double_local_sys_volatile() {
+; CHECK-LABEL: seq_cst_double_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot1256[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1256;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile double, ptr addrspace(5) %slot syncscope("") seq_cst, align 8
+ ret double %retval
+}
+
+define double @seq_cst_double_local_cta_volatile() {
+; CHECK-LABEL: seq_cst_double_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot1257[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1257;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile double, ptr addrspace(5) %slot syncscope("block") seq_cst, align 8
+ ret double %retval
+}
+
+define double @seq_cst_double_local_cluster_volatile() {
+; CHECK-LABEL: seq_cst_double_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot1258[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1258;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile double, ptr addrspace(5) %slot syncscope("cluster") seq_cst, align 8
+ ret double %retval
+}
+
+define double @seq_cst_double_local_gpu_volatile() {
+; CHECK-LABEL: seq_cst_double_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot1259[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1259;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: ld.local.b64 %rd2, [%SP];
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic volatile double, ptr addrspace(5) %slot syncscope("device") seq_cst, align 8
+ ret double %retval
+}
+
diff --git a/llvm/test/CodeGen/NVPTX/load-store-sm-70.ll b/llvm/test/CodeGen/NVPTX/load-store-sm-70.ll
index 7373b50477d22..1bfd702bf486d 100644
--- a/llvm/test/CodeGen/NVPTX/load-store-sm-70.ll
+++ b/llvm/test/CodeGen/NVPTX/load-store-sm-70.ll
@@ -884,39 +884,39 @@ define void @generic_sc_sys(ptr %a, ptr %b, ptr %c, ptr %d, ptr %e) local_unname
; CHECK-NEXT: // %bb.0:
; CHECK-NEXT: ld.param.b64 %rd1, [generic_sc_sys_param_0];
; CHECK-NEXT: fence.sc.sys;
-; CHECK-NEXT: ld.acquire.sys.b8 %rs1, [%rd1];
; CHECK-NEXT: ld.param.b64 %rd2, [generic_sc_sys_param_1];
-; CHECK-NEXT: add.s16 %rs2, %rs1, 1;
+; CHECK-NEXT: ld.acquire.sys.b8 %rs1, [%rd1];
; CHECK-NEXT: ld.param.b64 %rd3, [generic_sc_sys_param_2];
-; CHECK-NEXT: fence.sc.sys;
-; CHECK-NEXT: st.release.sys.b8 [%rd1], %rs2;
+; CHECK-NEXT: add.s16 %rs2, %rs1, 1;
; CHECK-NEXT: ld.param.b64 %rd4, [generic_sc_sys_param_3];
; CHECK-NEXT: fence.sc.sys;
-; CHECK-NEXT: ld.acquire.sys.b16 %rs3, [%rd2];
; CHECK-NEXT: ld.param.b64 %rd5, [generic_sc_sys_param_4];
+; CHECK-NEXT: st.relaxed.sys.b8 [%rd1], %rs2;
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.b16 %rs3, [%rd2];
; CHECK-NEXT: add.s16 %rs4, %rs3, 1;
; CHECK-NEXT: fence.sc.sys;
-; CHECK-NEXT: st.release.sys.b16 [%rd2], %rs4;
+; CHECK-NEXT: st.relaxed.sys.b16 [%rd2], %rs4;
; CHECK-NEXT: fence.sc.sys;
; CHECK-NEXT: ld.acquire.sys.b32 %r1, [%rd3];
; CHECK-NEXT: add.s32 %r2, %r1, 1;
; CHECK-NEXT: fence.sc.sys;
-; CHECK-NEXT: st.release.sys.b32 [%rd3], %r2;
+; CHECK-NEXT: st.relaxed.sys.b32 [%rd3], %r2;
; CHECK-NEXT: fence.sc.sys;
; CHECK-NEXT: ld.acquire.sys.b64 %rd6, [%rd4];
; CHECK-NEXT: add.s64 %rd7, %rd6, 1;
; CHECK-NEXT: fence.sc.sys;
-; CHECK-NEXT: st.release.sys.b64 [%rd4], %rd7;
+; CHECK-NEXT: st.relaxed.sys.b64 [%rd4], %rd7;
; CHECK-NEXT: fence.sc.sys;
; CHECK-NEXT: ld.acquire.sys.b32 %r3, [%rd5];
; CHECK-NEXT: add.rn.f32 %r4, %r3, 0f3F800000;
; CHECK-NEXT: fence.sc.sys;
-; CHECK-NEXT: st.release.sys.b32 [%rd5], %r4;
+; CHECK-NEXT: st.relaxed.sys.b32 [%rd5], %r4;
; CHECK-NEXT: fence.sc.sys;
; CHECK-NEXT: ld.acquire.sys.b64 %rd8, [%rd5];
; CHECK-NEXT: add.rn.f64 %rd9, %rd8, 0d3FF0000000000000;
; CHECK-NEXT: fence.sc.sys;
-; CHECK-NEXT: st.release.sys.b64 [%rd5], %rd9;
+; CHECK-NEXT: st.relaxed.sys.b64 [%rd5], %rd9;
; CHECK-NEXT: ret;
%a.load = load atomic i8, ptr %a seq_cst, align 1
%a.add = add i8 %a.load, 1
@@ -956,39 +956,39 @@ define void @generic_sc_volatile_sys(ptr %a, ptr %b, ptr %c, ptr %d, ptr %e) loc
; CHECK-NEXT: // %bb.0:
; CHECK-NEXT: ld.param.b64 %rd1, [generic_sc_volatile_sys_param_0];
; CHECK-NEXT: fence.sc.sys;
-; CHECK-NEXT: ld.acquire.sys.b8 %rs1, [%rd1];
; CHECK-NEXT: ld.param.b64 %rd2, [generic_sc_volatile_sys_param_1];
-; CHECK-NEXT: add.s16 %rs2, %rs1, 1;
+; CHECK-NEXT: ld.acquire.sys.b8 %rs1, [%rd1];
; CHECK-NEXT: ld.param.b64 %rd3, [generic_sc_volatile_sys_param_2];
-; CHECK-NEXT: fence.sc.sys;
-; CHECK-NEXT: st.release.sys.b8 [%rd1], %rs2;
+; CHECK-NEXT: add.s16 %rs2, %rs1, 1;
; CHECK-NEXT: ld.param.b64 %rd4, [generic_sc_volatile_sys_param_3];
; CHECK-NEXT: fence.sc.sys;
-; CHECK-NEXT: ld.acquire.sys.b16 %rs3, [%rd2];
; CHECK-NEXT: ld.param.b64 %rd5, [generic_sc_volatile_sys_param_4];
+; CHECK-NEXT: st.volatile.b8 [%rd1], %rs2;
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.b16 %rs3, [%rd2];
; CHECK-NEXT: add.s16 %rs4, %rs3, 1;
; CHECK-NEXT: fence.sc.sys;
-; CHECK-NEXT: st.release.sys.b16 [%rd2], %rs4;
+; CHECK-NEXT: st.volatile.b16 [%rd2], %rs4;
; CHECK-NEXT: fence.sc.sys;
; CHECK-NEXT: ld.acquire.sys.b32 %r1, [%rd3];
; CHECK-NEXT: add.s32 %r2, %r1, 1;
; CHECK-NEXT: fence.sc.sys;
-; CHECK-NEXT: st.release.sys.b32 [%rd3], %r2;
+; CHECK-NEXT: st.volatile.b32 [%rd3], %r2;
; CHECK-NEXT: fence.sc.sys;
; CHECK-NEXT: ld.acquire.sys.b64 %rd6, [%rd4];
; CHECK-NEXT: add.s64 %rd7, %rd6, 1;
; CHECK-NEXT: fence.sc.sys;
-; CHECK-NEXT: st.release.sys.b64 [%rd4], %rd7;
+; CHECK-NEXT: st.volatile.b64 [%rd4], %rd7;
; CHECK-NEXT: fence.sc.sys;
; CHECK-NEXT: ld.acquire.sys.b32 %r3, [%rd5];
; CHECK-NEXT: add.rn.f32 %r4, %r3, 0f3F800000;
; CHECK-NEXT: fence.sc.sys;
-; CHECK-NEXT: st.release.sys.b32 [%rd5], %r4;
+; CHECK-NEXT: st.volatile.b32 [%rd5], %r4;
; CHECK-NEXT: fence.sc.sys;
; CHECK-NEXT: ld.acquire.sys.b64 %rd8, [%rd5];
; CHECK-NEXT: add.rn.f64 %rd9, %rd8, 0d3FF0000000000000;
; CHECK-NEXT: fence.sc.sys;
-; CHECK-NEXT: st.release.sys.b64 [%rd5], %rd9;
+; CHECK-NEXT: st.volatile.b64 [%rd5], %rd9;
; CHECK-NEXT: ret;
%a.load = load atomic volatile i8, ptr %a seq_cst, align 1
%a.add = add i8 %a.load, 1
@@ -1028,39 +1028,39 @@ define void @generic_sc_gpu(ptr %a, ptr %b, ptr %c, ptr %d, ptr %e) local_unname
; CHECK-NEXT: // %bb.0:
; CHECK-NEXT: ld.param.b64 %rd1, [generic_sc_gpu_param_0];
; CHECK-NEXT: fence.sc.gpu;
-; CHECK-NEXT: ld.acquire.gpu.b8 %rs1, [%rd1];
; CHECK-NEXT: ld.param.b64 %rd2, [generic_sc_gpu_param_1];
-; CHECK-NEXT: add.s16 %rs2, %rs1, 1;
+; CHECK-NEXT: ld.acquire.gpu.b8 %rs1, [%rd1];
; CHECK-NEXT: ld.param.b64 %rd3, [generic_sc_gpu_param_2];
-; CHECK-NEXT: fence.sc.gpu;
-; CHECK-NEXT: st.release.gpu.b8 [%rd1], %rs2;
+; CHECK-NEXT: add.s16 %rs2, %rs1, 1;
; CHECK-NEXT: ld.param.b64 %rd4, [generic_sc_gpu_param_3];
; CHECK-NEXT: fence.sc.gpu;
-; CHECK-NEXT: ld.acquire.gpu.b16 %rs3, [%rd2];
; CHECK-NEXT: ld.param.b64 %rd5, [generic_sc_gpu_param_4];
+; CHECK-NEXT: st.relaxed.gpu.b8 [%rd1], %rs2;
+; CHECK-NEXT: fence.sc.gpu;
+; CHECK-NEXT: ld.acquire.gpu.b16 %rs3, [%rd2];
; CHECK-NEXT: add.s16 %rs4, %rs3, 1;
; CHECK-NEXT: fence.sc.gpu;
-; CHECK-NEXT: st.release.gpu.b16 [%rd2], %rs4;
+; CHECK-NEXT: st.relaxed.gpu.b16 [%rd2], %rs4;
; CHECK-NEXT: fence.sc.gpu;
; CHECK-NEXT: ld.acquire.gpu.b32 %r1, [%rd3];
; CHECK-NEXT: add.s32 %r2, %r1, 1;
; CHECK-NEXT: fence.sc.gpu;
-; CHECK-NEXT: st.release.gpu.b32 [%rd3], %r2;
+; CHECK-NEXT: st.relaxed.gpu.b32 [%rd3], %r2;
; CHECK-NEXT: fence.sc.gpu;
; CHECK-NEXT: ld.acquire.gpu.b64 %rd6, [%rd4];
; CHECK-NEXT: add.s64 %rd7, %rd6, 1;
; CHECK-NEXT: fence.sc.gpu;
-; CHECK-NEXT: st.release.gpu.b64 [%rd4], %rd7;
+; CHECK-NEXT: st.relaxed.gpu.b64 [%rd4], %rd7;
; CHECK-NEXT: fence.sc.gpu;
; CHECK-NEXT: ld.acquire.gpu.b32 %r3, [%rd5];
; CHECK-NEXT: add.rn.f32 %r4, %r3, 0f3F800000;
; CHECK-NEXT: fence.sc.gpu;
-; CHECK-NEXT: st.release.gpu.b32 [%rd5], %r4;
+; CHECK-NEXT: st.relaxed.gpu.b32 [%rd5], %r4;
; CHECK-NEXT: fence.sc.gpu;
; CHECK-NEXT: ld.acquire.gpu.b64 %rd8, [%rd5];
; CHECK-NEXT: add.rn.f64 %rd9, %rd8, 0d3FF0000000000000;
; CHECK-NEXT: fence.sc.gpu;
-; CHECK-NEXT: st.release.gpu.b64 [%rd5], %rd9;
+; CHECK-NEXT: st.relaxed.gpu.b64 [%rd5], %rd9;
; CHECK-NEXT: ret;
%a.load = load atomic i8, ptr %a syncscope("device") seq_cst, align 1
%a.add = add i8 %a.load, 1
@@ -1100,39 +1100,39 @@ define void @generic_sc_volatile_gpu(ptr %a, ptr %b, ptr %c, ptr %d, ptr %e) loc
; CHECK-NEXT: // %bb.0:
; CHECK-NEXT: ld.param.b64 %rd1, [generic_sc_volatile_gpu_param_0];
; CHECK-NEXT: fence.sc.sys;
-; CHECK-NEXT: ld.acquire.sys.b8 %rs1, [%rd1];
; CHECK-NEXT: ld.param.b64 %rd2, [generic_sc_volatile_gpu_param_1];
-; CHECK-NEXT: add.s16 %rs2, %rs1, 1;
+; CHECK-NEXT: ld.acquire.sys.b8 %rs1, [%rd1];
; CHECK-NEXT: ld.param.b64 %rd3, [generic_sc_volatile_gpu_param_2];
-; CHECK-NEXT: fence.sc.sys;
-; CHECK-NEXT: st.release.sys.b8 [%rd1], %rs2;
+; CHECK-NEXT: add.s16 %rs2, %rs1, 1;
; CHECK-NEXT: ld.param.b64 %rd4, [generic_sc_volatile_gpu_param_3];
; CHECK-NEXT: fence.sc.sys;
-; CHECK-NEXT: ld.acquire.sys.b16 %rs3, [%rd2];
; CHECK-NEXT: ld.param.b64 %rd5, [generic_sc_volatile_gpu_param_4];
+; CHECK-NEXT: st.volatile.b8 [%rd1], %rs2;
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.b16 %rs3, [%rd2];
; CHECK-NEXT: add.s16 %rs4, %rs3, 1;
; CHECK-NEXT: fence.sc.sys;
-; CHECK-NEXT: st.release.sys.b16 [%rd2], %rs4;
+; CHECK-NEXT: st.volatile.b16 [%rd2], %rs4;
; CHECK-NEXT: fence.sc.sys;
; CHECK-NEXT: ld.acquire.sys.b32 %r1, [%rd3];
; CHECK-NEXT: add.s32 %r2, %r1, 1;
; CHECK-NEXT: fence.sc.sys;
-; CHECK-NEXT: st.release.sys.b32 [%rd3], %r2;
+; CHECK-NEXT: st.volatile.b32 [%rd3], %r2;
; CHECK-NEXT: fence.sc.sys;
; CHECK-NEXT: ld.acquire.sys.b64 %rd6, [%rd4];
; CHECK-NEXT: add.s64 %rd7, %rd6, 1;
; CHECK-NEXT: fence.sc.sys;
-; CHECK-NEXT: st.release.sys.b64 [%rd4], %rd7;
+; CHECK-NEXT: st.volatile.b64 [%rd4], %rd7;
; CHECK-NEXT: fence.sc.sys;
; CHECK-NEXT: ld.acquire.sys.b32 %r3, [%rd5];
; CHECK-NEXT: add.rn.f32 %r4, %r3, 0f3F800000;
; CHECK-NEXT: fence.sc.sys;
-; CHECK-NEXT: st.release.sys.b32 [%rd5], %r4;
+; CHECK-NEXT: st.volatile.b32 [%rd5], %r4;
; CHECK-NEXT: fence.sc.sys;
; CHECK-NEXT: ld.acquire.sys.b64 %rd8, [%rd5];
; CHECK-NEXT: add.rn.f64 %rd9, %rd8, 0d3FF0000000000000;
; CHECK-NEXT: fence.sc.sys;
-; CHECK-NEXT: st.release.sys.b64 [%rd5], %rd9;
+; CHECK-NEXT: st.volatile.b64 [%rd5], %rd9;
; CHECK-NEXT: ret;
%a.load = load atomic volatile i8, ptr %a syncscope("device") seq_cst, align 1
%a.add = add i8 %a.load, 1
@@ -1172,39 +1172,39 @@ define void @generic_sc_cta(ptr %a, ptr %b, ptr %c, ptr %d, ptr %e) local_unname
; CHECK-NEXT: // %bb.0:
; CHECK-NEXT: ld.param.b64 %rd1, [generic_sc_cta_param_0];
; CHECK-NEXT: fence.sc.cta;
-; CHECK-NEXT: ld.acquire.cta.b8 %rs1, [%rd1];
; CHECK-NEXT: ld.param.b64 %rd2, [generic_sc_cta_param_1];
-; CHECK-NEXT: add.s16 %rs2, %rs1, 1;
+; CHECK-NEXT: ld.acquire.cta.b8 %rs1, [%rd1];
; CHECK-NEXT: ld.param.b64 %rd3, [generic_sc_cta_param_2];
-; CHECK-NEXT: fence.sc.cta;
-; CHECK-NEXT: st.release.cta.b8 [%rd1], %rs2;
+; CHECK-NEXT: add.s16 %rs2, %rs1, 1;
; CHECK-NEXT: ld.param.b64 %rd4, [generic_sc_cta_param_3];
; CHECK-NEXT: fence.sc.cta;
-; CHECK-NEXT: ld.acquire.cta.b16 %rs3, [%rd2];
; CHECK-NEXT: ld.param.b64 %rd5, [generic_sc_cta_param_4];
+; CHECK-NEXT: st.relaxed.cta.b8 [%rd1], %rs2;
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.acquire.cta.b16 %rs3, [%rd2];
; CHECK-NEXT: add.s16 %rs4, %rs3, 1;
; CHECK-NEXT: fence.sc.cta;
-; CHECK-NEXT: st.release.cta.b16 [%rd2], %rs4;
+; CHECK-NEXT: st.relaxed.cta.b16 [%rd2], %rs4;
; CHECK-NEXT: fence.sc.cta;
; CHECK-NEXT: ld.acquire.cta.b32 %r1, [%rd3];
; CHECK-NEXT: add.s32 %r2, %r1, 1;
; CHECK-NEXT: fence.sc.cta;
-; CHECK-NEXT: st.release.cta.b32 [%rd3], %r2;
+; CHECK-NEXT: st.relaxed.cta.b32 [%rd3], %r2;
; CHECK-NEXT: fence.sc.cta;
; CHECK-NEXT: ld.acquire.cta.b64 %rd6, [%rd4];
; CHECK-NEXT: add.s64 %rd7, %rd6, 1;
; CHECK-NEXT: fence.sc.cta;
-; CHECK-NEXT: st.release.cta.b64 [%rd4], %rd7;
+; CHECK-NEXT: st.relaxed.cta.b64 [%rd4], %rd7;
; CHECK-NEXT: fence.sc.cta;
; CHECK-NEXT: ld.acquire.cta.b32 %r3, [%rd5];
; CHECK-NEXT: add.rn.f32 %r4, %r3, 0f3F800000;
; CHECK-NEXT: fence.sc.cta;
-; CHECK-NEXT: st.release.cta.b32 [%rd5], %r4;
+; CHECK-NEXT: st.relaxed.cta.b32 [%rd5], %r4;
; CHECK-NEXT: fence.sc.cta;
; CHECK-NEXT: ld.acquire.cta.b64 %rd8, [%rd5];
; CHECK-NEXT: add.rn.f64 %rd9, %rd8, 0d3FF0000000000000;
; CHECK-NEXT: fence.sc.cta;
-; CHECK-NEXT: st.release.cta.b64 [%rd5], %rd9;
+; CHECK-NEXT: st.relaxed.cta.b64 [%rd5], %rd9;
; CHECK-NEXT: ret;
%a.load = load atomic i8, ptr %a syncscope("block") seq_cst, align 1
%a.add = add i8 %a.load, 1
@@ -1244,39 +1244,39 @@ define void @generic_sc_volatile_cta(ptr %a, ptr %b, ptr %c, ptr %d, ptr %e) loc
; CHECK-NEXT: // %bb.0:
; CHECK-NEXT: ld.param.b64 %rd1, [generic_sc_volatile_cta_param_0];
; CHECK-NEXT: fence.sc.sys;
-; CHECK-NEXT: ld.acquire.sys.b8 %rs1, [%rd1];
; CHECK-NEXT: ld.param.b64 %rd2, [generic_sc_volatile_cta_param_1];
-; CHECK-NEXT: add.s16 %rs2, %rs1, 1;
+; CHECK-NEXT: ld.acquire.sys.b8 %rs1, [%rd1];
; CHECK-NEXT: ld.param.b64 %rd3, [generic_sc_volatile_cta_param_2];
-; CHECK-NEXT: fence.sc.sys;
-; CHECK-NEXT: st.release.sys.b8 [%rd1], %rs2;
+; CHECK-NEXT: add.s16 %rs2, %rs1, 1;
; CHECK-NEXT: ld.param.b64 %rd4, [generic_sc_volatile_cta_param_3];
; CHECK-NEXT: fence.sc.sys;
-; CHECK-NEXT: ld.acquire.sys.b16 %rs3, [%rd2];
; CHECK-NEXT: ld.param.b64 %rd5, [generic_sc_volatile_cta_param_4];
+; CHECK-NEXT: st.volatile.b8 [%rd1], %rs2;
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.b16 %rs3, [%rd2];
; CHECK-NEXT: add.s16 %rs4, %rs3, 1;
; CHECK-NEXT: fence.sc.sys;
-; CHECK-NEXT: st.release.sys.b16 [%rd2], %rs4;
+; CHECK-NEXT: st.volatile.b16 [%rd2], %rs4;
; CHECK-NEXT: fence.sc.sys;
; CHECK-NEXT: ld.acquire.sys.b32 %r1, [%rd3];
; CHECK-NEXT: add.s32 %r2, %r1, 1;
; CHECK-NEXT: fence.sc.sys;
-; CHECK-NEXT: st.release.sys.b32 [%rd3], %r2;
+; CHECK-NEXT: st.volatile.b32 [%rd3], %r2;
; CHECK-NEXT: fence.sc.sys;
; CHECK-NEXT: ld.acquire.sys.b64 %rd6, [%rd4];
; CHECK-NEXT: add.s64 %rd7, %rd6, 1;
; CHECK-NEXT: fence.sc.sys;
-; CHECK-NEXT: st.release.sys.b64 [%rd4], %rd7;
+; CHECK-NEXT: st.volatile.b64 [%rd4], %rd7;
; CHECK-NEXT: fence.sc.sys;
; CHECK-NEXT: ld.acquire.sys.b32 %r3, [%rd5];
; CHECK-NEXT: add.rn.f32 %r4, %r3, 0f3F800000;
; CHECK-NEXT: fence.sc.sys;
-; CHECK-NEXT: st.release.sys.b32 [%rd5], %r4;
+; CHECK-NEXT: st.volatile.b32 [%rd5], %r4;
; CHECK-NEXT: fence.sc.sys;
; CHECK-NEXT: ld.acquire.sys.b64 %rd8, [%rd5];
; CHECK-NEXT: add.rn.f64 %rd9, %rd8, 0d3FF0000000000000;
; CHECK-NEXT: fence.sc.sys;
-; CHECK-NEXT: st.release.sys.b64 [%rd5], %rd9;
+; CHECK-NEXT: st.volatile.b64 [%rd5], %rd9;
; CHECK-NEXT: ret;
%a.load = load atomic volatile i8, ptr %a syncscope("block") seq_cst, align 1
%a.add = add i8 %a.load, 1
@@ -2158,39 +2158,39 @@ define void @global_seq_cst_sys(ptr addrspace(1) %a, ptr addrspace(1) %b, ptr ad
; CHECK-NEXT: // %bb.0:
; CHECK-NEXT: ld.param.b64 %rd1, [global_seq_cst_sys_param_0];
; CHECK-NEXT: fence.sc.sys;
-; CHECK-NEXT: ld.acquire.sys.global.b8 %rs1, [%rd1];
; CHECK-NEXT: ld.param.b64 %rd2, [global_seq_cst_sys_param_1];
-; CHECK-NEXT: add.s16 %rs2, %rs1, 1;
+; CHECK-NEXT: ld.acquire.sys.global.b8 %rs1, [%rd1];
; CHECK-NEXT: ld.param.b64 %rd3, [global_seq_cst_sys_param_2];
-; CHECK-NEXT: fence.sc.sys;
-; CHECK-NEXT: st.release.sys.global.b8 [%rd1], %rs2;
+; CHECK-NEXT: add.s16 %rs2, %rs1, 1;
; CHECK-NEXT: ld.param.b64 %rd4, [global_seq_cst_sys_param_3];
; CHECK-NEXT: fence.sc.sys;
-; CHECK-NEXT: ld.acquire.sys.global.b16 %rs3, [%rd2];
; CHECK-NEXT: ld.param.b64 %rd5, [global_seq_cst_sys_param_4];
+; CHECK-NEXT: st.relaxed.sys.global.b8 [%rd1], %rs2;
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.global.b16 %rs3, [%rd2];
; CHECK-NEXT: add.s16 %rs4, %rs3, 1;
; CHECK-NEXT: fence.sc.sys;
-; CHECK-NEXT: st.release.sys.global.b16 [%rd2], %rs4;
+; CHECK-NEXT: st.relaxed.sys.global.b16 [%rd2], %rs4;
; CHECK-NEXT: fence.sc.sys;
; CHECK-NEXT: ld.acquire.sys.global.b32 %r1, [%rd3];
; CHECK-NEXT: add.s32 %r2, %r1, 1;
; CHECK-NEXT: fence.sc.sys;
-; CHECK-NEXT: st.release.sys.global.b32 [%rd3], %r2;
+; CHECK-NEXT: st.relaxed.sys.global.b32 [%rd3], %r2;
; CHECK-NEXT: fence.sc.sys;
; CHECK-NEXT: ld.acquire.sys.global.b64 %rd6, [%rd4];
; CHECK-NEXT: add.s64 %rd7, %rd6, 1;
; CHECK-NEXT: fence.sc.sys;
-; CHECK-NEXT: st.release.sys.global.b64 [%rd4], %rd7;
+; CHECK-NEXT: st.relaxed.sys.global.b64 [%rd4], %rd7;
; CHECK-NEXT: fence.sc.sys;
; CHECK-NEXT: ld.acquire.sys.global.b32 %r3, [%rd5];
; CHECK-NEXT: add.rn.f32 %r4, %r3, 0f3F800000;
; CHECK-NEXT: fence.sc.sys;
-; CHECK-NEXT: st.release.sys.global.b32 [%rd5], %r4;
+; CHECK-NEXT: st.relaxed.sys.global.b32 [%rd5], %r4;
; CHECK-NEXT: fence.sc.sys;
; CHECK-NEXT: ld.acquire.sys.global.b64 %rd8, [%rd5];
; CHECK-NEXT: add.rn.f64 %rd9, %rd8, 0d3FF0000000000000;
; CHECK-NEXT: fence.sc.sys;
-; CHECK-NEXT: st.release.sys.global.b64 [%rd5], %rd9;
+; CHECK-NEXT: st.relaxed.sys.global.b64 [%rd5], %rd9;
; CHECK-NEXT: ret;
%a.load = load atomic i8, ptr addrspace(1) %a seq_cst, align 1
%a.add = add i8 %a.load, 1
@@ -2230,39 +2230,39 @@ define void @global_seq_cst_volatile_sys(ptr addrspace(1) %a, ptr addrspace(1) %
; CHECK-NEXT: // %bb.0:
; CHECK-NEXT: ld.param.b64 %rd1, [global_seq_cst_volatile_sys_param_0];
; CHECK-NEXT: fence.sc.sys;
-; CHECK-NEXT: ld.acquire.sys.global.b8 %rs1, [%rd1];
; CHECK-NEXT: ld.param.b64 %rd2, [global_seq_cst_volatile_sys_param_1];
-; CHECK-NEXT: add.s16 %rs2, %rs1, 1;
+; CHECK-NEXT: ld.acquire.sys.global.b8 %rs1, [%rd1];
; CHECK-NEXT: ld.param.b64 %rd3, [global_seq_cst_volatile_sys_param_2];
-; CHECK-NEXT: fence.sc.sys;
-; CHECK-NEXT: st.release.sys.global.b8 [%rd1], %rs2;
+; CHECK-NEXT: add.s16 %rs2, %rs1, 1;
; CHECK-NEXT: ld.param.b64 %rd4, [global_seq_cst_volatile_sys_param_3];
; CHECK-NEXT: fence.sc.sys;
-; CHECK-NEXT: ld.acquire.sys.global.b16 %rs3, [%rd2];
; CHECK-NEXT: ld.param.b64 %rd5, [global_seq_cst_volatile_sys_param_4];
+; CHECK-NEXT: st.mmio.relaxed.sys.global.b8 [%rd1], %rs2;
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.global.b16 %rs3, [%rd2];
; CHECK-NEXT: add.s16 %rs4, %rs3, 1;
; CHECK-NEXT: fence.sc.sys;
-; CHECK-NEXT: st.release.sys.global.b16 [%rd2], %rs4;
+; CHECK-NEXT: st.mmio.relaxed.sys.global.b16 [%rd2], %rs4;
; CHECK-NEXT: fence.sc.sys;
; CHECK-NEXT: ld.acquire.sys.global.b32 %r1, [%rd3];
; CHECK-NEXT: add.s32 %r2, %r1, 1;
; CHECK-NEXT: fence.sc.sys;
-; CHECK-NEXT: st.release.sys.global.b32 [%rd3], %r2;
+; CHECK-NEXT: st.mmio.relaxed.sys.global.b32 [%rd3], %r2;
; CHECK-NEXT: fence.sc.sys;
; CHECK-NEXT: ld.acquire.sys.global.b64 %rd6, [%rd4];
; CHECK-NEXT: add.s64 %rd7, %rd6, 1;
; CHECK-NEXT: fence.sc.sys;
-; CHECK-NEXT: st.release.sys.global.b64 [%rd4], %rd7;
+; CHECK-NEXT: st.mmio.relaxed.sys.global.b64 [%rd4], %rd7;
; CHECK-NEXT: fence.sc.sys;
; CHECK-NEXT: ld.acquire.sys.global.b32 %r3, [%rd5];
; CHECK-NEXT: add.rn.f32 %r4, %r3, 0f3F800000;
; CHECK-NEXT: fence.sc.sys;
-; CHECK-NEXT: st.release.sys.global.b32 [%rd5], %r4;
+; CHECK-NEXT: st.mmio.relaxed.sys.global.b32 [%rd5], %r4;
; CHECK-NEXT: fence.sc.sys;
; CHECK-NEXT: ld.acquire.sys.global.b64 %rd8, [%rd5];
; CHECK-NEXT: add.rn.f64 %rd9, %rd8, 0d3FF0000000000000;
; CHECK-NEXT: fence.sc.sys;
-; CHECK-NEXT: st.release.sys.global.b64 [%rd5], %rd9;
+; CHECK-NEXT: st.mmio.relaxed.sys.global.b64 [%rd5], %rd9;
; CHECK-NEXT: ret;
%a.load = load atomic volatile i8, ptr addrspace(1) %a seq_cst, align 1
%a.add = add i8 %a.load, 1
@@ -2302,39 +2302,39 @@ define void @global_seq_cst_gpu(ptr addrspace(1) %a, ptr addrspace(1) %b, ptr ad
; CHECK-NEXT: // %bb.0:
; CHECK-NEXT: ld.param.b64 %rd1, [global_seq_cst_gpu_param_0];
; CHECK-NEXT: fence.sc.gpu;
-; CHECK-NEXT: ld.acquire.gpu.global.b8 %rs1, [%rd1];
; CHECK-NEXT: ld.param.b64 %rd2, [global_seq_cst_gpu_param_1];
-; CHECK-NEXT: add.s16 %rs2, %rs1, 1;
+; CHECK-NEXT: ld.acquire.gpu.global.b8 %rs1, [%rd1];
; CHECK-NEXT: ld.param.b64 %rd3, [global_seq_cst_gpu_param_2];
-; CHECK-NEXT: fence.sc.gpu;
-; CHECK-NEXT: st.release.gpu.global.b8 [%rd1], %rs2;
+; CHECK-NEXT: add.s16 %rs2, %rs1, 1;
; CHECK-NEXT: ld.param.b64 %rd4, [global_seq_cst_gpu_param_3];
; CHECK-NEXT: fence.sc.gpu;
-; CHECK-NEXT: ld.acquire.gpu.global.b16 %rs3, [%rd2];
; CHECK-NEXT: ld.param.b64 %rd5, [global_seq_cst_gpu_param_4];
+; CHECK-NEXT: st.relaxed.gpu.global.b8 [%rd1], %rs2;
+; CHECK-NEXT: fence.sc.gpu;
+; CHECK-NEXT: ld.acquire.gpu.global.b16 %rs3, [%rd2];
; CHECK-NEXT: add.s16 %rs4, %rs3, 1;
; CHECK-NEXT: fence.sc.gpu;
-; CHECK-NEXT: st.release.gpu.global.b16 [%rd2], %rs4;
+; CHECK-NEXT: st.relaxed.gpu.global.b16 [%rd2], %rs4;
; CHECK-NEXT: fence.sc.gpu;
; CHECK-NEXT: ld.acquire.gpu.global.b32 %r1, [%rd3];
; CHECK-NEXT: add.s32 %r2, %r1, 1;
; CHECK-NEXT: fence.sc.gpu;
-; CHECK-NEXT: st.release.gpu.global.b32 [%rd3], %r2;
+; CHECK-NEXT: st.relaxed.gpu.global.b32 [%rd3], %r2;
; CHECK-NEXT: fence.sc.gpu;
; CHECK-NEXT: ld.acquire.gpu.global.b64 %rd6, [%rd4];
; CHECK-NEXT: add.s64 %rd7, %rd6, 1;
; CHECK-NEXT: fence.sc.gpu;
-; CHECK-NEXT: st.release.gpu.global.b64 [%rd4], %rd7;
+; CHECK-NEXT: st.relaxed.gpu.global.b64 [%rd4], %rd7;
; CHECK-NEXT: fence.sc.gpu;
; CHECK-NEXT: ld.acquire.gpu.global.b32 %r3, [%rd5];
; CHECK-NEXT: add.rn.f32 %r4, %r3, 0f3F800000;
; CHECK-NEXT: fence.sc.gpu;
-; CHECK-NEXT: st.release.gpu.global.b32 [%rd5], %r4;
+; CHECK-NEXT: st.relaxed.gpu.global.b32 [%rd5], %r4;
; CHECK-NEXT: fence.sc.gpu;
; CHECK-NEXT: ld.acquire.gpu.global.b64 %rd8, [%rd5];
; CHECK-NEXT: add.rn.f64 %rd9, %rd8, 0d3FF0000000000000;
; CHECK-NEXT: fence.sc.gpu;
-; CHECK-NEXT: st.release.gpu.global.b64 [%rd5], %rd9;
+; CHECK-NEXT: st.relaxed.gpu.global.b64 [%rd5], %rd9;
; CHECK-NEXT: ret;
%a.load = load atomic i8, ptr addrspace(1) %a syncscope("device") seq_cst, align 1
%a.add = add i8 %a.load, 1
@@ -2374,39 +2374,39 @@ define void @global_seq_cst_volatile_gpu(ptr addrspace(1) %a, ptr addrspace(1) %
; CHECK-NEXT: // %bb.0:
; CHECK-NEXT: ld.param.b64 %rd1, [global_seq_cst_volatile_gpu_param_0];
; CHECK-NEXT: fence.sc.sys;
-; CHECK-NEXT: ld.acquire.sys.global.b8 %rs1, [%rd1];
; CHECK-NEXT: ld.param.b64 %rd2, [global_seq_cst_volatile_gpu_param_1];
-; CHECK-NEXT: add.s16 %rs2, %rs1, 1;
+; CHECK-NEXT: ld.acquire.sys.global.b8 %rs1, [%rd1];
; CHECK-NEXT: ld.param.b64 %rd3, [global_seq_cst_volatile_gpu_param_2];
-; CHECK-NEXT: fence.sc.sys;
-; CHECK-NEXT: st.release.sys.global.b8 [%rd1], %rs2;
+; CHECK-NEXT: add.s16 %rs2, %rs1, 1;
; CHECK-NEXT: ld.param.b64 %rd4, [global_seq_cst_volatile_gpu_param_3];
; CHECK-NEXT: fence.sc.sys;
-; CHECK-NEXT: ld.acquire.sys.global.b16 %rs3, [%rd2];
; CHECK-NEXT: ld.param.b64 %rd5, [global_seq_cst_volatile_gpu_param_4];
+; CHECK-NEXT: st.mmio.relaxed.sys.global.b8 [%rd1], %rs2;
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.global.b16 %rs3, [%rd2];
; CHECK-NEXT: add.s16 %rs4, %rs3, 1;
; CHECK-NEXT: fence.sc.sys;
-; CHECK-NEXT: st.release.sys.global.b16 [%rd2], %rs4;
+; CHECK-NEXT: st.mmio.relaxed.sys.global.b16 [%rd2], %rs4;
; CHECK-NEXT: fence.sc.sys;
; CHECK-NEXT: ld.acquire.sys.global.b32 %r1, [%rd3];
; CHECK-NEXT: add.s32 %r2, %r1, 1;
; CHECK-NEXT: fence.sc.sys;
-; CHECK-NEXT: st.release.sys.global.b32 [%rd3], %r2;
+; CHECK-NEXT: st.mmio.relaxed.sys.global.b32 [%rd3], %r2;
; CHECK-NEXT: fence.sc.sys;
; CHECK-NEXT: ld.acquire.sys.global.b64 %rd6, [%rd4];
; CHECK-NEXT: add.s64 %rd7, %rd6, 1;
; CHECK-NEXT: fence.sc.sys;
-; CHECK-NEXT: st.release.sys.global.b64 [%rd4], %rd7;
+; CHECK-NEXT: st.mmio.relaxed.sys.global.b64 [%rd4], %rd7;
; CHECK-NEXT: fence.sc.sys;
; CHECK-NEXT: ld.acquire.sys.global.b32 %r3, [%rd5];
; CHECK-NEXT: add.rn.f32 %r4, %r3, 0f3F800000;
; CHECK-NEXT: fence.sc.sys;
-; CHECK-NEXT: st.release.sys.global.b32 [%rd5], %r4;
+; CHECK-NEXT: st.mmio.relaxed.sys.global.b32 [%rd5], %r4;
; CHECK-NEXT: fence.sc.sys;
; CHECK-NEXT: ld.acquire.sys.global.b64 %rd8, [%rd5];
; CHECK-NEXT: add.rn.f64 %rd9, %rd8, 0d3FF0000000000000;
; CHECK-NEXT: fence.sc.sys;
-; CHECK-NEXT: st.release.sys.global.b64 [%rd5], %rd9;
+; CHECK-NEXT: st.mmio.relaxed.sys.global.b64 [%rd5], %rd9;
; CHECK-NEXT: ret;
%a.load = load atomic volatile i8, ptr addrspace(1) %a syncscope("device") seq_cst, align 1
%a.add = add i8 %a.load, 1
@@ -2446,39 +2446,39 @@ define void @global_seq_cst_cta(ptr addrspace(1) %a, ptr addrspace(1) %b, ptr ad
; CHECK-NEXT: // %bb.0:
; CHECK-NEXT: ld.param.b64 %rd1, [global_seq_cst_cta_param_0];
; CHECK-NEXT: fence.sc.cta;
-; CHECK-NEXT: ld.acquire.cta.global.b8 %rs1, [%rd1];
; CHECK-NEXT: ld.param.b64 %rd2, [global_seq_cst_cta_param_1];
-; CHECK-NEXT: add.s16 %rs2, %rs1, 1;
+; CHECK-NEXT: ld.acquire.cta.global.b8 %rs1, [%rd1];
; CHECK-NEXT: ld.param.b64 %rd3, [global_seq_cst_cta_param_2];
-; CHECK-NEXT: fence.sc.cta;
-; CHECK-NEXT: st.release.cta.global.b8 [%rd1], %rs2;
+; CHECK-NEXT: add.s16 %rs2, %rs1, 1;
; CHECK-NEXT: ld.param.b64 %rd4, [global_seq_cst_cta_param_3];
; CHECK-NEXT: fence.sc.cta;
-; CHECK-NEXT: ld.acquire.cta.global.b16 %rs3, [%rd2];
; CHECK-NEXT: ld.param.b64 %rd5, [global_seq_cst_cta_param_4];
+; CHECK-NEXT: st.relaxed.cta.global.b8 [%rd1], %rs2;
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.acquire.cta.global.b16 %rs3, [%rd2];
; CHECK-NEXT: add.s16 %rs4, %rs3, 1;
; CHECK-NEXT: fence.sc.cta;
-; CHECK-NEXT: st.release.cta.global.b16 [%rd2], %rs4;
+; CHECK-NEXT: st.relaxed.cta.global.b16 [%rd2], %rs4;
; CHECK-NEXT: fence.sc.cta;
; CHECK-NEXT: ld.acquire.cta.global.b32 %r1, [%rd3];
; CHECK-NEXT: add.s32 %r2, %r1, 1;
; CHECK-NEXT: fence.sc.cta;
-; CHECK-NEXT: st.release.cta.global.b32 [%rd3], %r2;
+; CHECK-NEXT: st.relaxed.cta.global.b32 [%rd3], %r2;
; CHECK-NEXT: fence.sc.cta;
; CHECK-NEXT: ld.acquire.cta.global.b64 %rd6, [%rd4];
; CHECK-NEXT: add.s64 %rd7, %rd6, 1;
; CHECK-NEXT: fence.sc.cta;
-; CHECK-NEXT: st.release.cta.global.b64 [%rd4], %rd7;
+; CHECK-NEXT: st.relaxed.cta.global.b64 [%rd4], %rd7;
; CHECK-NEXT: fence.sc.cta;
; CHECK-NEXT: ld.acquire.cta.global.b32 %r3, [%rd5];
; CHECK-NEXT: add.rn.f32 %r4, %r3, 0f3F800000;
; CHECK-NEXT: fence.sc.cta;
-; CHECK-NEXT: st.release.cta.global.b32 [%rd5], %r4;
+; CHECK-NEXT: st.relaxed.cta.global.b32 [%rd5], %r4;
; CHECK-NEXT: fence.sc.cta;
; CHECK-NEXT: ld.acquire.cta.global.b64 %rd8, [%rd5];
; CHECK-NEXT: add.rn.f64 %rd9, %rd8, 0d3FF0000000000000;
; CHECK-NEXT: fence.sc.cta;
-; CHECK-NEXT: st.release.cta.global.b64 [%rd5], %rd9;
+; CHECK-NEXT: st.relaxed.cta.global.b64 [%rd5], %rd9;
; CHECK-NEXT: ret;
%a.load = load atomic i8, ptr addrspace(1) %a syncscope("block") seq_cst, align 1
%a.add = add i8 %a.load, 1
@@ -2518,39 +2518,39 @@ define void @global_seq_cst_volatile_cta(ptr addrspace(1) %a, ptr addrspace(1) %
; CHECK-NEXT: // %bb.0:
; CHECK-NEXT: ld.param.b64 %rd1, [global_seq_cst_volatile_cta_param_0];
; CHECK-NEXT: fence.sc.sys;
-; CHECK-NEXT: ld.acquire.sys.global.b8 %rs1, [%rd1];
; CHECK-NEXT: ld.param.b64 %rd2, [global_seq_cst_volatile_cta_param_1];
-; CHECK-NEXT: add.s16 %rs2, %rs1, 1;
+; CHECK-NEXT: ld.acquire.sys.global.b8 %rs1, [%rd1];
; CHECK-NEXT: ld.param.b64 %rd3, [global_seq_cst_volatile_cta_param_2];
-; CHECK-NEXT: fence.sc.sys;
-; CHECK-NEXT: st.release.sys.global.b8 [%rd1], %rs2;
+; CHECK-NEXT: add.s16 %rs2, %rs1, 1;
; CHECK-NEXT: ld.param.b64 %rd4, [global_seq_cst_volatile_cta_param_3];
; CHECK-NEXT: fence.sc.sys;
-; CHECK-NEXT: ld.acquire.sys.global.b16 %rs3, [%rd2];
; CHECK-NEXT: ld.param.b64 %rd5, [global_seq_cst_volatile_cta_param_4];
+; CHECK-NEXT: st.mmio.relaxed.sys.global.b8 [%rd1], %rs2;
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.global.b16 %rs3, [%rd2];
; CHECK-NEXT: add.s16 %rs4, %rs3, 1;
; CHECK-NEXT: fence.sc.sys;
-; CHECK-NEXT: st.release.sys.global.b16 [%rd2], %rs4;
+; CHECK-NEXT: st.mmio.relaxed.sys.global.b16 [%rd2], %rs4;
; CHECK-NEXT: fence.sc.sys;
; CHECK-NEXT: ld.acquire.sys.global.b32 %r1, [%rd3];
; CHECK-NEXT: add.s32 %r2, %r1, 1;
; CHECK-NEXT: fence.sc.sys;
-; CHECK-NEXT: st.release.sys.global.b32 [%rd3], %r2;
+; CHECK-NEXT: st.mmio.relaxed.sys.global.b32 [%rd3], %r2;
; CHECK-NEXT: fence.sc.sys;
; CHECK-NEXT: ld.acquire.sys.global.b64 %rd6, [%rd4];
; CHECK-NEXT: add.s64 %rd7, %rd6, 1;
; CHECK-NEXT: fence.sc.sys;
-; CHECK-NEXT: st.release.sys.global.b64 [%rd4], %rd7;
+; CHECK-NEXT: st.mmio.relaxed.sys.global.b64 [%rd4], %rd7;
; CHECK-NEXT: fence.sc.sys;
; CHECK-NEXT: ld.acquire.sys.global.b32 %r3, [%rd5];
; CHECK-NEXT: add.rn.f32 %r4, %r3, 0f3F800000;
; CHECK-NEXT: fence.sc.sys;
-; CHECK-NEXT: st.release.sys.global.b32 [%rd5], %r4;
+; CHECK-NEXT: st.mmio.relaxed.sys.global.b32 [%rd5], %r4;
; CHECK-NEXT: fence.sc.sys;
; CHECK-NEXT: ld.acquire.sys.global.b64 %rd8, [%rd5];
; CHECK-NEXT: add.rn.f64 %rd9, %rd8, 0d3FF0000000000000;
; CHECK-NEXT: fence.sc.sys;
-; CHECK-NEXT: st.release.sys.global.b64 [%rd5], %rd9;
+; CHECK-NEXT: st.mmio.relaxed.sys.global.b64 [%rd5], %rd9;
; CHECK-NEXT: ret;
%a.load = load atomic volatile i8, ptr addrspace(1) %a syncscope("block") seq_cst, align 1
%a.add = add i8 %a.load, 1
@@ -3432,39 +3432,39 @@ define void @shared_seq_cst_sys(ptr addrspace(3) %a, ptr addrspace(3) %b, ptr ad
; CHECK-NEXT: // %bb.0:
; CHECK-NEXT: ld.param.b64 %rd1, [shared_seq_cst_sys_param_0];
; CHECK-NEXT: fence.sc.sys;
-; CHECK-NEXT: ld.acquire.sys.shared.b8 %rs1, [%rd1];
; CHECK-NEXT: ld.param.b64 %rd2, [shared_seq_cst_sys_param_1];
-; CHECK-NEXT: add.s16 %rs2, %rs1, 1;
+; CHECK-NEXT: ld.acquire.sys.shared.b8 %rs1, [%rd1];
; CHECK-NEXT: ld.param.b64 %rd3, [shared_seq_cst_sys_param_2];
-; CHECK-NEXT: fence.sc.sys;
-; CHECK-NEXT: st.release.sys.shared.b8 [%rd1], %rs2;
+; CHECK-NEXT: add.s16 %rs2, %rs1, 1;
; CHECK-NEXT: ld.param.b64 %rd4, [shared_seq_cst_sys_param_3];
; CHECK-NEXT: fence.sc.sys;
-; CHECK-NEXT: ld.acquire.sys.shared.b16 %rs3, [%rd2];
; CHECK-NEXT: ld.param.b64 %rd5, [shared_seq_cst_sys_param_4];
+; CHECK-NEXT: st.relaxed.sys.shared.b8 [%rd1], %rs2;
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.shared.b16 %rs3, [%rd2];
; CHECK-NEXT: add.s16 %rs4, %rs3, 1;
; CHECK-NEXT: fence.sc.sys;
-; CHECK-NEXT: st.release.sys.shared.b16 [%rd2], %rs4;
+; CHECK-NEXT: st.relaxed.sys.shared.b16 [%rd2], %rs4;
; CHECK-NEXT: fence.sc.sys;
; CHECK-NEXT: ld.acquire.sys.shared.b32 %r1, [%rd3];
; CHECK-NEXT: add.s32 %r2, %r1, 1;
; CHECK-NEXT: fence.sc.sys;
-; CHECK-NEXT: st.release.sys.shared.b32 [%rd3], %r2;
+; CHECK-NEXT: st.relaxed.sys.shared.b32 [%rd3], %r2;
; CHECK-NEXT: fence.sc.sys;
; CHECK-NEXT: ld.acquire.sys.shared.b64 %rd6, [%rd4];
; CHECK-NEXT: add.s64 %rd7, %rd6, 1;
; CHECK-NEXT: fence.sc.sys;
-; CHECK-NEXT: st.release.sys.shared.b64 [%rd4], %rd7;
+; CHECK-NEXT: st.relaxed.sys.shared.b64 [%rd4], %rd7;
; CHECK-NEXT: fence.sc.sys;
; CHECK-NEXT: ld.acquire.sys.shared.b32 %r3, [%rd5];
; CHECK-NEXT: add.rn.f32 %r4, %r3, 0f3F800000;
; CHECK-NEXT: fence.sc.sys;
-; CHECK-NEXT: st.release.sys.shared.b32 [%rd5], %r4;
+; CHECK-NEXT: st.relaxed.sys.shared.b32 [%rd5], %r4;
; CHECK-NEXT: fence.sc.sys;
; CHECK-NEXT: ld.acquire.sys.shared.b64 %rd8, [%rd5];
; CHECK-NEXT: add.rn.f64 %rd9, %rd8, 0d3FF0000000000000;
; CHECK-NEXT: fence.sc.sys;
-; CHECK-NEXT: st.release.sys.shared.b64 [%rd5], %rd9;
+; CHECK-NEXT: st.relaxed.sys.shared.b64 [%rd5], %rd9;
; CHECK-NEXT: ret;
%a.load = load atomic i8, ptr addrspace(3) %a seq_cst, align 1
%a.add = add i8 %a.load, 1
@@ -3504,39 +3504,39 @@ define void @shared_seq_cst_volatile_sys(ptr addrspace(3) %a, ptr addrspace(3) %
; CHECK-NEXT: // %bb.0:
; CHECK-NEXT: ld.param.b64 %rd1, [shared_seq_cst_volatile_sys_param_0];
; CHECK-NEXT: fence.sc.sys;
-; CHECK-NEXT: ld.acquire.sys.shared.b8 %rs1, [%rd1];
; CHECK-NEXT: ld.param.b64 %rd2, [shared_seq_cst_volatile_sys_param_1];
-; CHECK-NEXT: add.s16 %rs2, %rs1, 1;
+; CHECK-NEXT: ld.acquire.sys.shared.b8 %rs1, [%rd1];
; CHECK-NEXT: ld.param.b64 %rd3, [shared_seq_cst_volatile_sys_param_2];
-; CHECK-NEXT: fence.sc.sys;
-; CHECK-NEXT: st.release.sys.shared.b8 [%rd1], %rs2;
+; CHECK-NEXT: add.s16 %rs2, %rs1, 1;
; CHECK-NEXT: ld.param.b64 %rd4, [shared_seq_cst_volatile_sys_param_3];
; CHECK-NEXT: fence.sc.sys;
-; CHECK-NEXT: ld.acquire.sys.shared.b16 %rs3, [%rd2];
; CHECK-NEXT: ld.param.b64 %rd5, [shared_seq_cst_volatile_sys_param_4];
+; CHECK-NEXT: st.volatile.shared.b8 [%rd1], %rs2;
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.shared.b16 %rs3, [%rd2];
; CHECK-NEXT: add.s16 %rs4, %rs3, 1;
; CHECK-NEXT: fence.sc.sys;
-; CHECK-NEXT: st.release.sys.shared.b16 [%rd2], %rs4;
+; CHECK-NEXT: st.volatile.shared.b16 [%rd2], %rs4;
; CHECK-NEXT: fence.sc.sys;
; CHECK-NEXT: ld.acquire.sys.shared.b32 %r1, [%rd3];
; CHECK-NEXT: add.s32 %r2, %r1, 1;
; CHECK-NEXT: fence.sc.sys;
-; CHECK-NEXT: st.release.sys.shared.b32 [%rd3], %r2;
+; CHECK-NEXT: st.volatile.shared.b32 [%rd3], %r2;
; CHECK-NEXT: fence.sc.sys;
; CHECK-NEXT: ld.acquire.sys.shared.b64 %rd6, [%rd4];
; CHECK-NEXT: add.s64 %rd7, %rd6, 1;
; CHECK-NEXT: fence.sc.sys;
-; CHECK-NEXT: st.release.sys.shared.b64 [%rd4], %rd7;
+; CHECK-NEXT: st.volatile.shared.b64 [%rd4], %rd7;
; CHECK-NEXT: fence.sc.sys;
; CHECK-NEXT: ld.acquire.sys.shared.b32 %r3, [%rd5];
; CHECK-NEXT: add.rn.f32 %r4, %r3, 0f3F800000;
; CHECK-NEXT: fence.sc.sys;
-; CHECK-NEXT: st.release.sys.shared.b32 [%rd5], %r4;
+; CHECK-NEXT: st.volatile.shared.b32 [%rd5], %r4;
; CHECK-NEXT: fence.sc.sys;
; CHECK-NEXT: ld.acquire.sys.shared.b64 %rd8, [%rd5];
; CHECK-NEXT: add.rn.f64 %rd9, %rd8, 0d3FF0000000000000;
; CHECK-NEXT: fence.sc.sys;
-; CHECK-NEXT: st.release.sys.shared.b64 [%rd5], %rd9;
+; CHECK-NEXT: st.volatile.shared.b64 [%rd5], %rd9;
; CHECK-NEXT: ret;
%a.load = load atomic volatile i8, ptr addrspace(3) %a seq_cst, align 1
%a.add = add i8 %a.load, 1
@@ -3576,39 +3576,39 @@ define void @shared_seq_cst_gpu(ptr addrspace(3) %a, ptr addrspace(3) %b, ptr ad
; CHECK-NEXT: // %bb.0:
; CHECK-NEXT: ld.param.b64 %rd1, [shared_seq_cst_gpu_param_0];
; CHECK-NEXT: fence.sc.gpu;
-; CHECK-NEXT: ld.acquire.gpu.shared.b8 %rs1, [%rd1];
; CHECK-NEXT: ld.param.b64 %rd2, [shared_seq_cst_gpu_param_1];
-; CHECK-NEXT: add.s16 %rs2, %rs1, 1;
+; CHECK-NEXT: ld.acquire.gpu.shared.b8 %rs1, [%rd1];
; CHECK-NEXT: ld.param.b64 %rd3, [shared_seq_cst_gpu_param_2];
-; CHECK-NEXT: fence.sc.gpu;
-; CHECK-NEXT: st.release.gpu.shared.b8 [%rd1], %rs2;
+; CHECK-NEXT: add.s16 %rs2, %rs1, 1;
; CHECK-NEXT: ld.param.b64 %rd4, [shared_seq_cst_gpu_param_3];
; CHECK-NEXT: fence.sc.gpu;
-; CHECK-NEXT: ld.acquire.gpu.shared.b16 %rs3, [%rd2];
; CHECK-NEXT: ld.param.b64 %rd5, [shared_seq_cst_gpu_param_4];
+; CHECK-NEXT: st.relaxed.gpu.shared.b8 [%rd1], %rs2;
+; CHECK-NEXT: fence.sc.gpu;
+; CHECK-NEXT: ld.acquire.gpu.shared.b16 %rs3, [%rd2];
; CHECK-NEXT: add.s16 %rs4, %rs3, 1;
; CHECK-NEXT: fence.sc.gpu;
-; CHECK-NEXT: st.release.gpu.shared.b16 [%rd2], %rs4;
+; CHECK-NEXT: st.relaxed.gpu.shared.b16 [%rd2], %rs4;
; CHECK-NEXT: fence.sc.gpu;
; CHECK-NEXT: ld.acquire.gpu.shared.b32 %r1, [%rd3];
; CHECK-NEXT: add.s32 %r2, %r1, 1;
; CHECK-NEXT: fence.sc.gpu;
-; CHECK-NEXT: st.release.gpu.shared.b32 [%rd3], %r2;
+; CHECK-NEXT: st.relaxed.gpu.shared.b32 [%rd3], %r2;
; CHECK-NEXT: fence.sc.gpu;
; CHECK-NEXT: ld.acquire.gpu.shared.b64 %rd6, [%rd4];
; CHECK-NEXT: add.s64 %rd7, %rd6, 1;
; CHECK-NEXT: fence.sc.gpu;
-; CHECK-NEXT: st.release.gpu.shared.b64 [%rd4], %rd7;
+; CHECK-NEXT: st.relaxed.gpu.shared.b64 [%rd4], %rd7;
; CHECK-NEXT: fence.sc.gpu;
; CHECK-NEXT: ld.acquire.gpu.shared.b32 %r3, [%rd5];
; CHECK-NEXT: add.rn.f32 %r4, %r3, 0f3F800000;
; CHECK-NEXT: fence.sc.gpu;
-; CHECK-NEXT: st.release.gpu.shared.b32 [%rd5], %r4;
+; CHECK-NEXT: st.relaxed.gpu.shared.b32 [%rd5], %r4;
; CHECK-NEXT: fence.sc.gpu;
; CHECK-NEXT: ld.acquire.gpu.shared.b64 %rd8, [%rd5];
; CHECK-NEXT: add.rn.f64 %rd9, %rd8, 0d3FF0000000000000;
; CHECK-NEXT: fence.sc.gpu;
-; CHECK-NEXT: st.release.gpu.shared.b64 [%rd5], %rd9;
+; CHECK-NEXT: st.relaxed.gpu.shared.b64 [%rd5], %rd9;
; CHECK-NEXT: ret;
%a.load = load atomic i8, ptr addrspace(3) %a syncscope("device") seq_cst, align 1
%a.add = add i8 %a.load, 1
@@ -3648,39 +3648,39 @@ define void @shared_seq_cst_volatile_gpu(ptr addrspace(3) %a, ptr addrspace(3) %
; CHECK-NEXT: // %bb.0:
; CHECK-NEXT: ld.param.b64 %rd1, [shared_seq_cst_volatile_gpu_param_0];
; CHECK-NEXT: fence.sc.sys;
-; CHECK-NEXT: ld.acquire.sys.shared.b8 %rs1, [%rd1];
; CHECK-NEXT: ld.param.b64 %rd2, [shared_seq_cst_volatile_gpu_param_1];
-; CHECK-NEXT: add.s16 %rs2, %rs1, 1;
+; CHECK-NEXT: ld.acquire.sys.shared.b8 %rs1, [%rd1];
; CHECK-NEXT: ld.param.b64 %rd3, [shared_seq_cst_volatile_gpu_param_2];
-; CHECK-NEXT: fence.sc.sys;
-; CHECK-NEXT: st.release.sys.shared.b8 [%rd1], %rs2;
+; CHECK-NEXT: add.s16 %rs2, %rs1, 1;
; CHECK-NEXT: ld.param.b64 %rd4, [shared_seq_cst_volatile_gpu_param_3];
; CHECK-NEXT: fence.sc.sys;
-; CHECK-NEXT: ld.acquire.sys.shared.b16 %rs3, [%rd2];
; CHECK-NEXT: ld.param.b64 %rd5, [shared_seq_cst_volatile_gpu_param_4];
+; CHECK-NEXT: st.volatile.shared.b8 [%rd1], %rs2;
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.shared.b16 %rs3, [%rd2];
; CHECK-NEXT: add.s16 %rs4, %rs3, 1;
; CHECK-NEXT: fence.sc.sys;
-; CHECK-NEXT: st.release.sys.shared.b16 [%rd2], %rs4;
+; CHECK-NEXT: st.volatile.shared.b16 [%rd2], %rs4;
; CHECK-NEXT: fence.sc.sys;
; CHECK-NEXT: ld.acquire.sys.shared.b32 %r1, [%rd3];
; CHECK-NEXT: add.s32 %r2, %r1, 1;
; CHECK-NEXT: fence.sc.sys;
-; CHECK-NEXT: st.release.sys.shared.b32 [%rd3], %r2;
+; CHECK-NEXT: st.volatile.shared.b32 [%rd3], %r2;
; CHECK-NEXT: fence.sc.sys;
; CHECK-NEXT: ld.acquire.sys.shared.b64 %rd6, [%rd4];
; CHECK-NEXT: add.s64 %rd7, %rd6, 1;
; CHECK-NEXT: fence.sc.sys;
-; CHECK-NEXT: st.release.sys.shared.b64 [%rd4], %rd7;
+; CHECK-NEXT: st.volatile.shared.b64 [%rd4], %rd7;
; CHECK-NEXT: fence.sc.sys;
; CHECK-NEXT: ld.acquire.sys.shared.b32 %r3, [%rd5];
; CHECK-NEXT: add.rn.f32 %r4, %r3, 0f3F800000;
; CHECK-NEXT: fence.sc.sys;
-; CHECK-NEXT: st.release.sys.shared.b32 [%rd5], %r4;
+; CHECK-NEXT: st.volatile.shared.b32 [%rd5], %r4;
; CHECK-NEXT: fence.sc.sys;
; CHECK-NEXT: ld.acquire.sys.shared.b64 %rd8, [%rd5];
; CHECK-NEXT: add.rn.f64 %rd9, %rd8, 0d3FF0000000000000;
; CHECK-NEXT: fence.sc.sys;
-; CHECK-NEXT: st.release.sys.shared.b64 [%rd5], %rd9;
+; CHECK-NEXT: st.volatile.shared.b64 [%rd5], %rd9;
; CHECK-NEXT: ret;
%a.load = load atomic volatile i8, ptr addrspace(3) %a syncscope("device") seq_cst, align 1
%a.add = add i8 %a.load, 1
@@ -3720,39 +3720,39 @@ define void @shared_seq_cst_cta(ptr addrspace(3) %a, ptr addrspace(3) %b, ptr ad
; CHECK-NEXT: // %bb.0:
; CHECK-NEXT: ld.param.b64 %rd1, [shared_seq_cst_cta_param_0];
; CHECK-NEXT: fence.sc.cta;
-; CHECK-NEXT: ld.acquire.cta.shared.b8 %rs1, [%rd1];
; CHECK-NEXT: ld.param.b64 %rd2, [shared_seq_cst_cta_param_1];
-; CHECK-NEXT: add.s16 %rs2, %rs1, 1;
+; CHECK-NEXT: ld.acquire.cta.shared.b8 %rs1, [%rd1];
; CHECK-NEXT: ld.param.b64 %rd3, [shared_seq_cst_cta_param_2];
-; CHECK-NEXT: fence.sc.cta;
-; CHECK-NEXT: st.release.cta.shared.b8 [%rd1], %rs2;
+; CHECK-NEXT: add.s16 %rs2, %rs1, 1;
; CHECK-NEXT: ld.param.b64 %rd4, [shared_seq_cst_cta_param_3];
; CHECK-NEXT: fence.sc.cta;
-; CHECK-NEXT: ld.acquire.cta.shared.b16 %rs3, [%rd2];
; CHECK-NEXT: ld.param.b64 %rd5, [shared_seq_cst_cta_param_4];
+; CHECK-NEXT: st.relaxed.cta.shared.b8 [%rd1], %rs2;
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.acquire.cta.shared.b16 %rs3, [%rd2];
; CHECK-NEXT: add.s16 %rs4, %rs3, 1;
; CHECK-NEXT: fence.sc.cta;
-; CHECK-NEXT: st.release.cta.shared.b16 [%rd2], %rs4;
+; CHECK-NEXT: st.relaxed.cta.shared.b16 [%rd2], %rs4;
; CHECK-NEXT: fence.sc.cta;
; CHECK-NEXT: ld.acquire.cta.shared.b32 %r1, [%rd3];
; CHECK-NEXT: add.s32 %r2, %r1, 1;
; CHECK-NEXT: fence.sc.cta;
-; CHECK-NEXT: st.release.cta.shared.b32 [%rd3], %r2;
+; CHECK-NEXT: st.relaxed.cta.shared.b32 [%rd3], %r2;
; CHECK-NEXT: fence.sc.cta;
; CHECK-NEXT: ld.acquire.cta.shared.b64 %rd6, [%rd4];
; CHECK-NEXT: add.s64 %rd7, %rd6, 1;
; CHECK-NEXT: fence.sc.cta;
-; CHECK-NEXT: st.release.cta.shared.b64 [%rd4], %rd7;
+; CHECK-NEXT: st.relaxed.cta.shared.b64 [%rd4], %rd7;
; CHECK-NEXT: fence.sc.cta;
; CHECK-NEXT: ld.acquire.cta.shared.b32 %r3, [%rd5];
; CHECK-NEXT: add.rn.f32 %r4, %r3, 0f3F800000;
; CHECK-NEXT: fence.sc.cta;
-; CHECK-NEXT: st.release.cta.shared.b32 [%rd5], %r4;
+; CHECK-NEXT: st.relaxed.cta.shared.b32 [%rd5], %r4;
; CHECK-NEXT: fence.sc.cta;
; CHECK-NEXT: ld.acquire.cta.shared.b64 %rd8, [%rd5];
; CHECK-NEXT: add.rn.f64 %rd9, %rd8, 0d3FF0000000000000;
; CHECK-NEXT: fence.sc.cta;
-; CHECK-NEXT: st.release.cta.shared.b64 [%rd5], %rd9;
+; CHECK-NEXT: st.relaxed.cta.shared.b64 [%rd5], %rd9;
; CHECK-NEXT: ret;
%a.load = load atomic i8, ptr addrspace(3) %a syncscope("block") seq_cst, align 1
%a.add = add i8 %a.load, 1
@@ -3792,39 +3792,39 @@ define void @shared_seq_cst_volatile_cta(ptr addrspace(3) %a, ptr addrspace(3) %
; CHECK-NEXT: // %bb.0:
; CHECK-NEXT: ld.param.b64 %rd1, [shared_seq_cst_volatile_cta_param_0];
; CHECK-NEXT: fence.sc.sys;
-; CHECK-NEXT: ld.acquire.sys.shared.b8 %rs1, [%rd1];
; CHECK-NEXT: ld.param.b64 %rd2, [shared_seq_cst_volatile_cta_param_1];
-; CHECK-NEXT: add.s16 %rs2, %rs1, 1;
+; CHECK-NEXT: ld.acquire.sys.shared.b8 %rs1, [%rd1];
; CHECK-NEXT: ld.param.b64 %rd3, [shared_seq_cst_volatile_cta_param_2];
-; CHECK-NEXT: fence.sc.sys;
-; CHECK-NEXT: st.release.sys.shared.b8 [%rd1], %rs2;
+; CHECK-NEXT: add.s16 %rs2, %rs1, 1;
; CHECK-NEXT: ld.param.b64 %rd4, [shared_seq_cst_volatile_cta_param_3];
; CHECK-NEXT: fence.sc.sys;
-; CHECK-NEXT: ld.acquire.sys.shared.b16 %rs3, [%rd2];
; CHECK-NEXT: ld.param.b64 %rd5, [shared_seq_cst_volatile_cta_param_4];
+; CHECK-NEXT: st.volatile.shared.b8 [%rd1], %rs2;
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.shared.b16 %rs3, [%rd2];
; CHECK-NEXT: add.s16 %rs4, %rs3, 1;
; CHECK-NEXT: fence.sc.sys;
-; CHECK-NEXT: st.release.sys.shared.b16 [%rd2], %rs4;
+; CHECK-NEXT: st.volatile.shared.b16 [%rd2], %rs4;
; CHECK-NEXT: fence.sc.sys;
; CHECK-NEXT: ld.acquire.sys.shared.b32 %r1, [%rd3];
; CHECK-NEXT: add.s32 %r2, %r1, 1;
; CHECK-NEXT: fence.sc.sys;
-; CHECK-NEXT: st.release.sys.shared.b32 [%rd3], %r2;
+; CHECK-NEXT: st.volatile.shared.b32 [%rd3], %r2;
; CHECK-NEXT: fence.sc.sys;
; CHECK-NEXT: ld.acquire.sys.shared.b64 %rd6, [%rd4];
; CHECK-NEXT: add.s64 %rd7, %rd6, 1;
; CHECK-NEXT: fence.sc.sys;
-; CHECK-NEXT: st.release.sys.shared.b64 [%rd4], %rd7;
+; CHECK-NEXT: st.volatile.shared.b64 [%rd4], %rd7;
; CHECK-NEXT: fence.sc.sys;
; CHECK-NEXT: ld.acquire.sys.shared.b32 %r3, [%rd5];
; CHECK-NEXT: add.rn.f32 %r4, %r3, 0f3F800000;
; CHECK-NEXT: fence.sc.sys;
-; CHECK-NEXT: st.release.sys.shared.b32 [%rd5], %r4;
+; CHECK-NEXT: st.volatile.shared.b32 [%rd5], %r4;
; CHECK-NEXT: fence.sc.sys;
; CHECK-NEXT: ld.acquire.sys.shared.b64 %rd8, [%rd5];
; CHECK-NEXT: add.rn.f64 %rd9, %rd8, 0d3FF0000000000000;
; CHECK-NEXT: fence.sc.sys;
-; CHECK-NEXT: st.release.sys.shared.b64 [%rd5], %rd9;
+; CHECK-NEXT: st.volatile.shared.b64 [%rd5], %rd9;
; CHECK-NEXT: ret;
%a.load = load atomic volatile i8, ptr addrspace(3) %a syncscope("block") seq_cst, align 1
%a.add = add i8 %a.load, 1
diff --git a/llvm/test/CodeGen/NVPTX/load-store-sm-90.ll b/llvm/test/CodeGen/NVPTX/load-store-sm-90.ll
index 5e85e989a2fd8..c846cd79f5066 100644
--- a/llvm/test/CodeGen/NVPTX/load-store-sm-90.ll
+++ b/llvm/test/CodeGen/NVPTX/load-store-sm-90.ll
@@ -404,39 +404,39 @@ define void @generic_sc_cluster(ptr %a, ptr %b, ptr %c, ptr %d, ptr %e) local_un
; CHECK-NEXT: // %bb.0:
; CHECK-NEXT: ld.param.b64 %rd1, [generic_sc_cluster_param_0];
; CHECK-NEXT: fence.sc.cluster;
-; CHECK-NEXT: ld.acquire.cluster.b8 %rs1, [%rd1];
; CHECK-NEXT: ld.param.b64 %rd2, [generic_sc_cluster_param_1];
-; CHECK-NEXT: add.s16 %rs2, %rs1, 1;
+; CHECK-NEXT: ld.acquire.cluster.b8 %rs1, [%rd1];
; CHECK-NEXT: ld.param.b64 %rd3, [generic_sc_cluster_param_2];
-; CHECK-NEXT: fence.sc.cluster;
-; CHECK-NEXT: st.release.cluster.b8 [%rd1], %rs2;
+; CHECK-NEXT: add.s16 %rs2, %rs1, 1;
; CHECK-NEXT: ld.param.b64 %rd4, [generic_sc_cluster_param_3];
; CHECK-NEXT: fence.sc.cluster;
-; CHECK-NEXT: ld.acquire.cluster.b16 %rs3, [%rd2];
; CHECK-NEXT: ld.param.b64 %rd5, [generic_sc_cluster_param_4];
+; CHECK-NEXT: st.relaxed.cluster.b8 [%rd1], %rs2;
+; CHECK-NEXT: fence.sc.cluster;
+; CHECK-NEXT: ld.acquire.cluster.b16 %rs3, [%rd2];
; CHECK-NEXT: add.s16 %rs4, %rs3, 1;
; CHECK-NEXT: fence.sc.cluster;
-; CHECK-NEXT: st.release.cluster.b16 [%rd2], %rs4;
+; CHECK-NEXT: st.relaxed.cluster.b16 [%rd2], %rs4;
; CHECK-NEXT: fence.sc.cluster;
; CHECK-NEXT: ld.acquire.cluster.b32 %r1, [%rd3];
; CHECK-NEXT: add.s32 %r2, %r1, 1;
; CHECK-NEXT: fence.sc.cluster;
-; CHECK-NEXT: st.release.cluster.b32 [%rd3], %r2;
+; CHECK-NEXT: st.relaxed.cluster.b32 [%rd3], %r2;
; CHECK-NEXT: fence.sc.cluster;
; CHECK-NEXT: ld.acquire.cluster.b64 %rd6, [%rd4];
; CHECK-NEXT: add.s64 %rd7, %rd6, 1;
; CHECK-NEXT: fence.sc.cluster;
-; CHECK-NEXT: st.release.cluster.b64 [%rd4], %rd7;
+; CHECK-NEXT: st.relaxed.cluster.b64 [%rd4], %rd7;
; CHECK-NEXT: fence.sc.cluster;
; CHECK-NEXT: ld.acquire.cluster.b32 %r3, [%rd5];
; CHECK-NEXT: add.rn.f32 %r4, %r3, 0f3F800000;
; CHECK-NEXT: fence.sc.cluster;
-; CHECK-NEXT: st.release.cluster.b32 [%rd5], %r4;
+; CHECK-NEXT: st.relaxed.cluster.b32 [%rd5], %r4;
; CHECK-NEXT: fence.sc.cluster;
; CHECK-NEXT: ld.acquire.cluster.b64 %rd8, [%rd5];
; CHECK-NEXT: add.rn.f64 %rd9, %rd8, 0d3FF0000000000000;
; CHECK-NEXT: fence.sc.cluster;
-; CHECK-NEXT: st.release.cluster.b64 [%rd5], %rd9;
+; CHECK-NEXT: st.relaxed.cluster.b64 [%rd5], %rd9;
; CHECK-NEXT: ret;
%a.load = load atomic i8, ptr %a syncscope("cluster") seq_cst, align 1
%a.add = add i8 %a.load, 1
@@ -476,39 +476,39 @@ define void @generic_sc_volatile_cluster(ptr %a, ptr %b, ptr %c, ptr %d, ptr %e)
; CHECK-NEXT: // %bb.0:
; CHECK-NEXT: ld.param.b64 %rd1, [generic_sc_volatile_cluster_param_0];
; CHECK-NEXT: fence.sc.sys;
-; CHECK-NEXT: ld.acquire.sys.b8 %rs1, [%rd1];
; CHECK-NEXT: ld.param.b64 %rd2, [generic_sc_volatile_cluster_param_1];
-; CHECK-NEXT: add.s16 %rs2, %rs1, 1;
+; CHECK-NEXT: ld.acquire.sys.b8 %rs1, [%rd1];
; CHECK-NEXT: ld.param.b64 %rd3, [generic_sc_volatile_cluster_param_2];
-; CHECK-NEXT: fence.sc.sys;
-; CHECK-NEXT: st.release.sys.b8 [%rd1], %rs2;
+; CHECK-NEXT: add.s16 %rs2, %rs1, 1;
; CHECK-NEXT: ld.param.b64 %rd4, [generic_sc_volatile_cluster_param_3];
; CHECK-NEXT: fence.sc.sys;
-; CHECK-NEXT: ld.acquire.sys.b16 %rs3, [%rd2];
; CHECK-NEXT: ld.param.b64 %rd5, [generic_sc_volatile_cluster_param_4];
+; CHECK-NEXT: st.volatile.b8 [%rd1], %rs2;
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.b16 %rs3, [%rd2];
; CHECK-NEXT: add.s16 %rs4, %rs3, 1;
; CHECK-NEXT: fence.sc.sys;
-; CHECK-NEXT: st.release.sys.b16 [%rd2], %rs4;
+; CHECK-NEXT: st.volatile.b16 [%rd2], %rs4;
; CHECK-NEXT: fence.sc.sys;
; CHECK-NEXT: ld.acquire.sys.b32 %r1, [%rd3];
; CHECK-NEXT: add.s32 %r2, %r1, 1;
; CHECK-NEXT: fence.sc.sys;
-; CHECK-NEXT: st.release.sys.b32 [%rd3], %r2;
+; CHECK-NEXT: st.volatile.b32 [%rd3], %r2;
; CHECK-NEXT: fence.sc.sys;
; CHECK-NEXT: ld.acquire.sys.b64 %rd6, [%rd4];
; CHECK-NEXT: add.s64 %rd7, %rd6, 1;
; CHECK-NEXT: fence.sc.sys;
-; CHECK-NEXT: st.release.sys.b64 [%rd4], %rd7;
+; CHECK-NEXT: st.volatile.b64 [%rd4], %rd7;
; CHECK-NEXT: fence.sc.sys;
; CHECK-NEXT: ld.acquire.sys.b32 %r3, [%rd5];
; CHECK-NEXT: add.rn.f32 %r4, %r3, 0f3F800000;
; CHECK-NEXT: fence.sc.sys;
-; CHECK-NEXT: st.release.sys.b32 [%rd5], %r4;
+; CHECK-NEXT: st.volatile.b32 [%rd5], %r4;
; CHECK-NEXT: fence.sc.sys;
; CHECK-NEXT: ld.acquire.sys.b64 %rd8, [%rd5];
; CHECK-NEXT: add.rn.f64 %rd9, %rd8, 0d3FF0000000000000;
; CHECK-NEXT: fence.sc.sys;
-; CHECK-NEXT: st.release.sys.b64 [%rd5], %rd9;
+; CHECK-NEXT: st.volatile.b64 [%rd5], %rd9;
; CHECK-NEXT: ret;
%a.load = load atomic volatile i8, ptr %a syncscope("cluster") seq_cst, align 1
%a.add = add i8 %a.load, 1
@@ -910,39 +910,39 @@ define void @global_seq_cst_cluster(ptr addrspace(1) %a, ptr addrspace(1) %b, pt
; CHECK-NEXT: // %bb.0:
; CHECK-NEXT: ld.param.b64 %rd1, [global_seq_cst_cluster_param_0];
; CHECK-NEXT: fence.sc.cluster;
-; CHECK-NEXT: ld.acquire.cluster.global.b8 %rs1, [%rd1];
; CHECK-NEXT: ld.param.b64 %rd2, [global_seq_cst_cluster_param_1];
-; CHECK-NEXT: add.s16 %rs2, %rs1, 1;
+; CHECK-NEXT: ld.acquire.cluster.global.b8 %rs1, [%rd1];
; CHECK-NEXT: ld.param.b64 %rd3, [global_seq_cst_cluster_param_2];
-; CHECK-NEXT: fence.sc.cluster;
-; CHECK-NEXT: st.release.cluster.global.b8 [%rd1], %rs2;
+; CHECK-NEXT: add.s16 %rs2, %rs1, 1;
; CHECK-NEXT: ld.param.b64 %rd4, [global_seq_cst_cluster_param_3];
; CHECK-NEXT: fence.sc.cluster;
-; CHECK-NEXT: ld.acquire.cluster.global.b16 %rs3, [%rd2];
; CHECK-NEXT: ld.param.b64 %rd5, [global_seq_cst_cluster_param_4];
+; CHECK-NEXT: st.relaxed.cluster.global.b8 [%rd1], %rs2;
+; CHECK-NEXT: fence.sc.cluster;
+; CHECK-NEXT: ld.acquire.cluster.global.b16 %rs3, [%rd2];
; CHECK-NEXT: add.s16 %rs4, %rs3, 1;
; CHECK-NEXT: fence.sc.cluster;
-; CHECK-NEXT: st.release.cluster.global.b16 [%rd2], %rs4;
+; CHECK-NEXT: st.relaxed.cluster.global.b16 [%rd2], %rs4;
; CHECK-NEXT: fence.sc.cluster;
; CHECK-NEXT: ld.acquire.cluster.global.b32 %r1, [%rd3];
; CHECK-NEXT: add.s32 %r2, %r1, 1;
; CHECK-NEXT: fence.sc.cluster;
-; CHECK-NEXT: st.release.cluster.global.b32 [%rd3], %r2;
+; CHECK-NEXT: st.relaxed.cluster.global.b32 [%rd3], %r2;
; CHECK-NEXT: fence.sc.cluster;
; CHECK-NEXT: ld.acquire.cluster.global.b64 %rd6, [%rd4];
; CHECK-NEXT: add.s64 %rd7, %rd6, 1;
; CHECK-NEXT: fence.sc.cluster;
-; CHECK-NEXT: st.release.cluster.global.b64 [%rd4], %rd7;
+; CHECK-NEXT: st.relaxed.cluster.global.b64 [%rd4], %rd7;
; CHECK-NEXT: fence.sc.cluster;
; CHECK-NEXT: ld.acquire.cluster.global.b32 %r3, [%rd5];
; CHECK-NEXT: add.rn.f32 %r4, %r3, 0f3F800000;
; CHECK-NEXT: fence.sc.cluster;
-; CHECK-NEXT: st.release.cluster.global.b32 [%rd5], %r4;
+; CHECK-NEXT: st.relaxed.cluster.global.b32 [%rd5], %r4;
; CHECK-NEXT: fence.sc.cluster;
; CHECK-NEXT: ld.acquire.cluster.global.b64 %rd8, [%rd5];
; CHECK-NEXT: add.rn.f64 %rd9, %rd8, 0d3FF0000000000000;
; CHECK-NEXT: fence.sc.cluster;
-; CHECK-NEXT: st.release.cluster.global.b64 [%rd5], %rd9;
+; CHECK-NEXT: st.relaxed.cluster.global.b64 [%rd5], %rd9;
; CHECK-NEXT: ret;
%a.load = load atomic i8, ptr addrspace(1) %a syncscope("cluster") seq_cst, align 1
%a.add = add i8 %a.load, 1
@@ -982,39 +982,39 @@ define void @global_seq_cst_volatile_cluster(ptr addrspace(1) %a, ptr addrspace(
; CHECK-NEXT: // %bb.0:
; CHECK-NEXT: ld.param.b64 %rd1, [global_seq_cst_volatile_cluster_param_0];
; CHECK-NEXT: fence.sc.sys;
-; CHECK-NEXT: ld.acquire.sys.global.b8 %rs1, [%rd1];
; CHECK-NEXT: ld.param.b64 %rd2, [global_seq_cst_volatile_cluster_param_1];
-; CHECK-NEXT: add.s16 %rs2, %rs1, 1;
+; CHECK-NEXT: ld.acquire.sys.global.b8 %rs1, [%rd1];
; CHECK-NEXT: ld.param.b64 %rd3, [global_seq_cst_volatile_cluster_param_2];
-; CHECK-NEXT: fence.sc.sys;
-; CHECK-NEXT: st.release.sys.global.b8 [%rd1], %rs2;
+; CHECK-NEXT: add.s16 %rs2, %rs1, 1;
; CHECK-NEXT: ld.param.b64 %rd4, [global_seq_cst_volatile_cluster_param_3];
; CHECK-NEXT: fence.sc.sys;
-; CHECK-NEXT: ld.acquire.sys.global.b16 %rs3, [%rd2];
; CHECK-NEXT: ld.param.b64 %rd5, [global_seq_cst_volatile_cluster_param_4];
+; CHECK-NEXT: st.volatile.global.b8 [%rd1], %rs2;
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.global.b16 %rs3, [%rd2];
; CHECK-NEXT: add.s16 %rs4, %rs3, 1;
; CHECK-NEXT: fence.sc.sys;
-; CHECK-NEXT: st.release.sys.global.b16 [%rd2], %rs4;
+; CHECK-NEXT: st.volatile.global.b16 [%rd2], %rs4;
; CHECK-NEXT: fence.sc.sys;
; CHECK-NEXT: ld.acquire.sys.global.b32 %r1, [%rd3];
; CHECK-NEXT: add.s32 %r2, %r1, 1;
; CHECK-NEXT: fence.sc.sys;
-; CHECK-NEXT: st.release.sys.global.b32 [%rd3], %r2;
+; CHECK-NEXT: st.volatile.global.b32 [%rd3], %r2;
; CHECK-NEXT: fence.sc.sys;
; CHECK-NEXT: ld.acquire.sys.global.b64 %rd6, [%rd4];
; CHECK-NEXT: add.s64 %rd7, %rd6, 1;
; CHECK-NEXT: fence.sc.sys;
-; CHECK-NEXT: st.release.sys.global.b64 [%rd4], %rd7;
+; CHECK-NEXT: st.volatile.global.b64 [%rd4], %rd7;
; CHECK-NEXT: fence.sc.sys;
; CHECK-NEXT: ld.acquire.sys.global.b32 %r3, [%rd5];
; CHECK-NEXT: add.rn.f32 %r4, %r3, 0f3F800000;
; CHECK-NEXT: fence.sc.sys;
-; CHECK-NEXT: st.release.sys.global.b32 [%rd5], %r4;
+; CHECK-NEXT: st.volatile.global.b32 [%rd5], %r4;
; CHECK-NEXT: fence.sc.sys;
; CHECK-NEXT: ld.acquire.sys.global.b64 %rd8, [%rd5];
; CHECK-NEXT: add.rn.f64 %rd9, %rd8, 0d3FF0000000000000;
; CHECK-NEXT: fence.sc.sys;
-; CHECK-NEXT: st.release.sys.global.b64 [%rd5], %rd9;
+; CHECK-NEXT: st.volatile.global.b64 [%rd5], %rd9;
; CHECK-NEXT: ret;
%a.load = load atomic volatile i8, ptr addrspace(1) %a syncscope("cluster") seq_cst, align 1
%a.add = add i8 %a.load, 1
@@ -1416,39 +1416,39 @@ define void @shared_seq_cst_cluster(ptr addrspace(3) %a, ptr addrspace(3) %b, pt
; CHECK-NEXT: // %bb.0:
; CHECK-NEXT: ld.param.b64 %rd1, [shared_seq_cst_cluster_param_0];
; CHECK-NEXT: fence.sc.cluster;
-; CHECK-NEXT: ld.acquire.cluster.shared.b8 %rs1, [%rd1];
; CHECK-NEXT: ld.param.b64 %rd2, [shared_seq_cst_cluster_param_1];
-; CHECK-NEXT: add.s16 %rs2, %rs1, 1;
+; CHECK-NEXT: ld.acquire.cluster.shared.b8 %rs1, [%rd1];
; CHECK-NEXT: ld.param.b64 %rd3, [shared_seq_cst_cluster_param_2];
-; CHECK-NEXT: fence.sc.cluster;
-; CHECK-NEXT: st.release.cluster.shared.b8 [%rd1], %rs2;
+; CHECK-NEXT: add.s16 %rs2, %rs1, 1;
; CHECK-NEXT: ld.param.b64 %rd4, [shared_seq_cst_cluster_param_3];
; CHECK-NEXT: fence.sc.cluster;
-; CHECK-NEXT: ld.acquire.cluster.shared.b16 %rs3, [%rd2];
; CHECK-NEXT: ld.param.b64 %rd5, [shared_seq_cst_cluster_param_4];
+; CHECK-NEXT: st.relaxed.cluster.shared.b8 [%rd1], %rs2;
+; CHECK-NEXT: fence.sc.cluster;
+; CHECK-NEXT: ld.acquire.cluster.shared.b16 %rs3, [%rd2];
; CHECK-NEXT: add.s16 %rs4, %rs3, 1;
; CHECK-NEXT: fence.sc.cluster;
-; CHECK-NEXT: st.release.cluster.shared.b16 [%rd2], %rs4;
+; CHECK-NEXT: st.relaxed.cluster.shared.b16 [%rd2], %rs4;
; CHECK-NEXT: fence.sc.cluster;
; CHECK-NEXT: ld.acquire.cluster.shared.b32 %r1, [%rd3];
; CHECK-NEXT: add.s32 %r2, %r1, 1;
; CHECK-NEXT: fence.sc.cluster;
-; CHECK-NEXT: st.release.cluster.shared.b32 [%rd3], %r2;
+; CHECK-NEXT: st.relaxed.cluster.shared.b32 [%rd3], %r2;
; CHECK-NEXT: fence.sc.cluster;
; CHECK-NEXT: ld.acquire.cluster.shared.b64 %rd6, [%rd4];
; CHECK-NEXT: add.s64 %rd7, %rd6, 1;
; CHECK-NEXT: fence.sc.cluster;
-; CHECK-NEXT: st.release.cluster.shared.b64 [%rd4], %rd7;
+; CHECK-NEXT: st.relaxed.cluster.shared.b64 [%rd4], %rd7;
; CHECK-NEXT: fence.sc.cluster;
; CHECK-NEXT: ld.acquire.cluster.shared.b32 %r3, [%rd5];
; CHECK-NEXT: add.rn.f32 %r4, %r3, 0f3F800000;
; CHECK-NEXT: fence.sc.cluster;
-; CHECK-NEXT: st.release.cluster.shared.b32 [%rd5], %r4;
+; CHECK-NEXT: st.relaxed.cluster.shared.b32 [%rd5], %r4;
; CHECK-NEXT: fence.sc.cluster;
; CHECK-NEXT: ld.acquire.cluster.shared.b64 %rd8, [%rd5];
; CHECK-NEXT: add.rn.f64 %rd9, %rd8, 0d3FF0000000000000;
; CHECK-NEXT: fence.sc.cluster;
-; CHECK-NEXT: st.release.cluster.shared.b64 [%rd5], %rd9;
+; CHECK-NEXT: st.relaxed.cluster.shared.b64 [%rd5], %rd9;
; CHECK-NEXT: ret;
%a.load = load atomic i8, ptr addrspace(3) %a syncscope("cluster") seq_cst, align 1
%a.add = add i8 %a.load, 1
@@ -1488,39 +1488,39 @@ define void @shared_seq_cst_volatile_cluster(ptr addrspace(3) %a, ptr addrspace(
; CHECK-NEXT: // %bb.0:
; CHECK-NEXT: ld.param.b64 %rd1, [shared_seq_cst_volatile_cluster_param_0];
; CHECK-NEXT: fence.sc.sys;
-; CHECK-NEXT: ld.acquire.sys.shared.b8 %rs1, [%rd1];
; CHECK-NEXT: ld.param.b64 %rd2, [shared_seq_cst_volatile_cluster_param_1];
-; CHECK-NEXT: add.s16 %rs2, %rs1, 1;
+; CHECK-NEXT: ld.acquire.sys.shared.b8 %rs1, [%rd1];
; CHECK-NEXT: ld.param.b64 %rd3, [shared_seq_cst_volatile_cluster_param_2];
-; CHECK-NEXT: fence.sc.sys;
-; CHECK-NEXT: st.release.sys.shared.b8 [%rd1], %rs2;
+; CHECK-NEXT: add.s16 %rs2, %rs1, 1;
; CHECK-NEXT: ld.param.b64 %rd4, [shared_seq_cst_volatile_cluster_param_3];
; CHECK-NEXT: fence.sc.sys;
-; CHECK-NEXT: ld.acquire.sys.shared.b16 %rs3, [%rd2];
; CHECK-NEXT: ld.param.b64 %rd5, [shared_seq_cst_volatile_cluster_param_4];
+; CHECK-NEXT: st.volatile.shared.b8 [%rd1], %rs2;
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.acquire.sys.shared.b16 %rs3, [%rd2];
; CHECK-NEXT: add.s16 %rs4, %rs3, 1;
; CHECK-NEXT: fence.sc.sys;
-; CHECK-NEXT: st.release.sys.shared.b16 [%rd2], %rs4;
+; CHECK-NEXT: st.volatile.shared.b16 [%rd2], %rs4;
; CHECK-NEXT: fence.sc.sys;
; CHECK-NEXT: ld.acquire.sys.shared.b32 %r1, [%rd3];
; CHECK-NEXT: add.s32 %r2, %r1, 1;
; CHECK-NEXT: fence.sc.sys;
-; CHECK-NEXT: st.release.sys.shared.b32 [%rd3], %r2;
+; CHECK-NEXT: st.volatile.shared.b32 [%rd3], %r2;
; CHECK-NEXT: fence.sc.sys;
; CHECK-NEXT: ld.acquire.sys.shared.b64 %rd6, [%rd4];
; CHECK-NEXT: add.s64 %rd7, %rd6, 1;
; CHECK-NEXT: fence.sc.sys;
-; CHECK-NEXT: st.release.sys.shared.b64 [%rd4], %rd7;
+; CHECK-NEXT: st.volatile.shared.b64 [%rd4], %rd7;
; CHECK-NEXT: fence.sc.sys;
; CHECK-NEXT: ld.acquire.sys.shared.b32 %r3, [%rd5];
; CHECK-NEXT: add.rn.f32 %r4, %r3, 0f3F800000;
; CHECK-NEXT: fence.sc.sys;
-; CHECK-NEXT: st.release.sys.shared.b32 [%rd5], %r4;
+; CHECK-NEXT: st.volatile.shared.b32 [%rd5], %r4;
; CHECK-NEXT: fence.sc.sys;
; CHECK-NEXT: ld.acquire.sys.shared.b64 %rd8, [%rd5];
; CHECK-NEXT: add.rn.f64 %rd9, %rd8, 0d3FF0000000000000;
; CHECK-NEXT: fence.sc.sys;
-; CHECK-NEXT: st.release.sys.shared.b64 [%rd5], %rd9;
+; CHECK-NEXT: st.volatile.shared.b64 [%rd5], %rd9;
; CHECK-NEXT: ret;
%a.load = load atomic volatile i8, ptr addrspace(3) %a syncscope("cluster") seq_cst, align 1
%a.add = add i8 %a.load, 1
diff --git a/llvm/test/CodeGen/NVPTX/load.py b/llvm/test/CodeGen/NVPTX/load.py
new file mode 100644
index 0000000000000..c2e5aae0a3736
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/load.py
@@ -0,0 +1,194 @@
+from string import Template
+from itertools import product
+
+# From https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-ld, Target ISA notes:
+
+# SM
+# Sub-qualifier ::cta requires sm_30 or higher.
+# Support for scope qualifier, .relaxed, .acquire, .weak qualifiers require sm_70 or higher.
+# Support for .b128 type requires sm_70 or higher.
+# Support for .cluster scope qualifier requires sm_90 or higher.
+# Sub-qualifier ::cluster requires sm_90 or higher.
+
+# PTX
+# Support for scope qualifier, .relaxed, .acquire, .weak qualifiers introduced in PTX ISA version 6.0. (prior to this, support will have to be through fences)
+# Support for .cluster scope qualifier introduced in PTX ISA version 7.8.
+# Support for ::cta and ::cluster sub-qualifiers introduced in PTX ISA version 7.8.
+# Support for .b128 type introduced in PTX ISA version 8.3.
+# Support for .sys scope with .b128 type introduced in PTX ISA version 8.4.
+# Support for .volatile qualifier with .local state space introduced in PTX ISA version 9.1.
+
+# Feature compatibility matrix
+# ----------------------------
+# Features (each gated by both an SM and a PTX threshold; the feature lights up
+# only when both thresholds are satisfied):
+# A = scope qualifier / .relaxed / .acquire / .weak sm_70 + PTX 6.0
+# B = .cluster scope sm_90 + PTX 7.8
+# C = .b128 type sm_70 + PTX 8.3
+# D = .sys + .b128 sm_70 + PTX 8.4
+# E = .volatile + .local (any) + PTX 9.1
+#
+# PTX 5.0 PTX 6.0 PTX 7.8 PTX 8.3 PTX 8.4 PTX 9.1
+# sm_30 -- -- -- -- -- E
+# sm_70 -- A A A C A C D A C D E
+# sm_90 -- A A B A B C A B C D A B C D E
+#
+# Each cell lists the features valid at that (SM, PTX); blank means no feature
+# from this set is supported. A feature's "subgrid" is the rectangle of cells
+# containing its letter; the subgrid's top-left cell is the minimal (SM, PTX)
+# pair that turns the feature on.
+
+# One PTX version per SM (matching cmpxchg.py / atomicrmw.py). The previous
+# multi-PTX sweep produced identical codegen on every slice axis -- the only
+# PTX-version-sensitive lowering (volatile-on-local at PTX 9.1) is exercised
+# by volatile-spin-loop.ll, which keeps its own multi-PTX RUN lines.
+TEST_SM_PTX_ARCHS = [(60, 50), (70, 63), (90, 87)]
+
+SCOPE_LLVM_TO_PTX = {"": "sys", "block": "cta", "cluster": "cluster", "device": "gpu"}
+
+ORDERINGS = ["notatomic", "unordered", "monotonic", "acquire", "seq_cst"]
+
+ADDRSPACE_NUM_TO_ADDRSPACE = {0: "generic", 1: "global", 3: "shared", 4: "const", 5: "local", 101: "param"}
+
+DATATYPE_SIZE_BITS = {
+ "i8": 8, "i16": 16, "i32": 32, "i64": 64, "i128": 128,
+ "half": 16, "bfloat": 16, "float": 32, "double": 64,
+}
+DATATYPES = list(DATATYPE_SIZE_BITS.keys())
+
+VOLATILES = [False, True]
+
+
+def get_addrspace_cast(addrspace):
+ if addrspace == 0:
+ return ""
+ return " addrspace({})".format(addrspace)
+
+
+load_atomic_func = Template(
+ """define ${datatype} @${func_name}(ptr${addrspace_cast} %addr) {
+ %retval = load atomic ${volatile_kw}${datatype}, ptr${addrspace_cast} %addr syncscope("${llvm_scope}") ${ordering}, align ${align}
+ ret ${datatype} %retval
+}
+"""
+)
+
+load_nonatomic_func = Template(
+ """define ${datatype} @${func_name}(ptr${addrspace_cast} %addr) {
+ %retval = load ${volatile_kw}${datatype}, ptr${addrspace_cast} %addr
+ ret ${datatype} %retval
+}
+"""
+)
+
+local_atomic_load_func = Template(
+ """define ${datatype} @${func_name}() {
+ %slot = alloca ${datatype}, align ${align}, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load atomic ${volatile_kw}${datatype}, ptr addrspace(5) %slot syncscope("${llvm_scope}") ${ordering}, align ${align}
+ ret ${datatype} %retval
+}
+"""
+)
+
+local_nonatomic_load_func = Template(
+ """define ${datatype} @${func_name}() {
+ %slot = alloca ${datatype}, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ %retval = load ${volatile_kw}${datatype}, ptr addrspace(5) %slot
+ ret ${datatype} %retval
+}
+"""
+)
+
+
+def get_load_func(datatype, addrspace, scope, ordering, volatile):
+ """Pick template + fill slots from (datatype, addrspace, scope, ordering, volatile).
+
+ - ordering == "notatomic" -> non-atomic (scope unused)
+ - addrspace == 5 -> local (alloca + inline-asm address capture)
+ - volatile -> emits `volatile` keyword and suffixes the name
+ """
+ size = DATATYPE_SIZE_BITS[datatype]
+ align = size // 8
+ addrspace_name = ADDRSPACE_NUM_TO_ADDRSPACE[addrspace]
+ is_atomic = ordering != "notatomic"
+ is_local = addrspace == 5
+ volatile_kw = "volatile " if volatile else ""
+ volatile_suffix = "_volatile" if volatile else ""
+
+ if is_atomic:
+ ptx_scope = SCOPE_LLVM_TO_PTX[scope]
+ func_name = "{}_{}_{}_{}{}".format(
+ ordering, datatype, addrspace_name, ptx_scope, volatile_suffix
+ )
+ if is_local:
+ return local_atomic_load_func.substitute(
+ datatype=datatype,
+ func_name=func_name,
+ llvm_scope=scope,
+ ordering=ordering,
+ align=align,
+ volatile_kw=volatile_kw,
+ )
+ return load_atomic_func.substitute(
+ datatype=datatype,
+ func_name=func_name,
+ addrspace_cast=get_addrspace_cast(addrspace),
+ llvm_scope=scope,
+ ordering=ordering,
+ align=align,
+ volatile_kw=volatile_kw,
+ )
+
+ func_name = "notatomic_{}_{}{}".format(datatype, addrspace_name, volatile_suffix)
+ if is_local:
+ return local_nonatomic_load_func.substitute(
+ datatype=datatype,
+ func_name=func_name,
+ volatile_kw=volatile_kw,
+ )
+ return load_nonatomic_func.substitute(
+ datatype=datatype,
+ func_name=func_name,
+ addrspace_cast=get_addrspace_cast(addrspace),
+ volatile_kw=volatile_kw,
+ )
+
+
+run_statement = Template(
+ """; RUN: llc < %s -march=nvptx64 -mcpu=sm_${sm} -mattr=+ptx${ptx} | FileCheck %s
+; RUN: %if ptxas-sm_${sm} && ptxas-isa-${ptxfp} %{ llc < %s -march=nvptx64 -mcpu=sm_${sm} -mattr=+ptx${ptx} | %ptxas-verify -arch=sm_${sm} %}
+"""
+)
+
+if __name__ == "__main__":
+ # NOTE: loads can't be `release` or `acq_rel` per LangRef. On sm < 70 the
+ # backend used to reject orderings other than notatomic/monotonic, but
+ # AtomicExpandPass now emulates them, so the same axes work uniformly.
+
+ for sm, ptx in TEST_SM_PTX_ARCHS:
+ with open("load-sm{}.ll".format(str(sm)), "w") as fp:
+ print(run_statement.substitute(sm=sm, ptx=ptx, ptxfp=ptx / 10.0), file=fp)
+
+ # All combinations of (ordering x addrspace x datatype x volatile x
+ # scope). Atomic loads on .const/.param are skipped (read-only
+ # spaces have no observable atomic semantic). i128 only at sm_90
+ # -- atom.b128 requires sm_90+/PTX 8.3+, and plain ld.b128 requires
+ # sm_70+/PTX 8.3+; neither holds at sm_60/70 with our test matrix.
+ # For notatomic, syncscope is illegal -- emit only at scope="".
+ for ordering, addrspace, datatype, volatile, scope in product(
+ ORDERINGS, ADDRSPACE_NUM_TO_ADDRSPACE, DATATYPES, VOLATILES,
+ SCOPE_LLVM_TO_PTX,
+ ):
+ if datatype == "i128" and sm < 90:
+ continue
+ is_atomic = ordering != "notatomic"
+ if is_atomic and addrspace in (4, 101):
+ continue
+ if not is_atomic and scope != "":
+ continue # scope irrelevant for notatomic
+ print(
+ get_load_func(datatype, addrspace, scope, ordering, volatile),
+ file=fp,
+ )
diff --git a/llvm/test/CodeGen/NVPTX/store-sm60.ll b/llvm/test/CodeGen/NVPTX/store-sm60.ll
new file mode 100644
index 0000000000000..dfea9409a916c
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/store-sm60.ll
@@ -0,0 +1,19360 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_60 -mattr=+ptx50 | FileCheck %s
+; RUN: %if ptxas-sm_60 && ptxas-isa-5.0 %{ llc < %s -march=nvptx64 -mcpu=sm_60 -mattr=+ptx50 | %ptxas-verify -arch=sm_60 %}
+
+define void @notatomic_i8_generic(i8 %val, ptr %addr) {
+; CHECK-LABEL: notatomic_i8_generic(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [notatomic_i8_generic_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i8_generic_param_1];
+; CHECK-NEXT: st.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store i8 %val, ptr %addr
+ ret void
+}
+
+define void @notatomic_i8_generic_volatile(i8 %val, ptr %addr) {
+; CHECK-LABEL: notatomic_i8_generic_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [notatomic_i8_generic_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i8_generic_volatile_param_1];
+; CHECK-NEXT: st.volatile.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store volatile i8 %val, ptr %addr
+ ret void
+}
+
+define void @notatomic_i16_generic(i16 %val, ptr %addr) {
+; CHECK-LABEL: notatomic_i16_generic(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [notatomic_i16_generic_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i16_generic_param_1];
+; CHECK-NEXT: st.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store i16 %val, ptr %addr
+ ret void
+}
+
+define void @notatomic_i16_generic_volatile(i16 %val, ptr %addr) {
+; CHECK-LABEL: notatomic_i16_generic_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [notatomic_i16_generic_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i16_generic_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store volatile i16 %val, ptr %addr
+ ret void
+}
+
+define void @notatomic_i32_generic(i32 %val, ptr %addr) {
+; CHECK-LABEL: notatomic_i32_generic(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [notatomic_i32_generic_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i32_generic_param_1];
+; CHECK-NEXT: st.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store i32 %val, ptr %addr
+ ret void
+}
+
+define void @notatomic_i32_generic_volatile(i32 %val, ptr %addr) {
+; CHECK-LABEL: notatomic_i32_generic_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [notatomic_i32_generic_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i32_generic_volatile_param_1];
+; CHECK-NEXT: st.volatile.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store volatile i32 %val, ptr %addr
+ ret void
+}
+
+define void @notatomic_i64_generic(i64 %val, ptr %addr) {
+; CHECK-LABEL: notatomic_i64_generic(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i64_generic_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [notatomic_i64_generic_param_1];
+; CHECK-NEXT: st.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store i64 %val, ptr %addr
+ ret void
+}
+
+define void @notatomic_i64_generic_volatile(i64 %val, ptr %addr) {
+; CHECK-LABEL: notatomic_i64_generic_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i64_generic_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [notatomic_i64_generic_volatile_param_1];
+; CHECK-NEXT: st.volatile.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store volatile i64 %val, ptr %addr
+ ret void
+}
+
+define void @notatomic_half_generic(half %val, ptr %addr) {
+; CHECK-LABEL: notatomic_half_generic(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [notatomic_half_generic_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_half_generic_param_1];
+; CHECK-NEXT: st.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store half %val, ptr %addr
+ ret void
+}
+
+define void @notatomic_half_generic_volatile(half %val, ptr %addr) {
+; CHECK-LABEL: notatomic_half_generic_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [notatomic_half_generic_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_half_generic_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store volatile half %val, ptr %addr
+ ret void
+}
+
+define void @notatomic_bfloat_generic(bfloat %val, ptr %addr) {
+; CHECK-LABEL: notatomic_bfloat_generic(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [notatomic_bfloat_generic_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_bfloat_generic_param_1];
+; CHECK-NEXT: st.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store bfloat %val, ptr %addr
+ ret void
+}
+
+define void @notatomic_bfloat_generic_volatile(bfloat %val, ptr %addr) {
+; CHECK-LABEL: notatomic_bfloat_generic_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [notatomic_bfloat_generic_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_bfloat_generic_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store volatile bfloat %val, ptr %addr
+ ret void
+}
+
+define void @notatomic_float_generic(float %val, ptr %addr) {
+; CHECK-LABEL: notatomic_float_generic(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [notatomic_float_generic_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_float_generic_param_1];
+; CHECK-NEXT: st.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store float %val, ptr %addr
+ ret void
+}
+
+define void @notatomic_float_generic_volatile(float %val, ptr %addr) {
+; CHECK-LABEL: notatomic_float_generic_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [notatomic_float_generic_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_float_generic_volatile_param_1];
+; CHECK-NEXT: st.volatile.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store volatile float %val, ptr %addr
+ ret void
+}
+
+define void @notatomic_double_generic(double %val, ptr %addr) {
+; CHECK-LABEL: notatomic_double_generic(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_double_generic_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [notatomic_double_generic_param_1];
+; CHECK-NEXT: st.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store double %val, ptr %addr
+ ret void
+}
+
+define void @notatomic_double_generic_volatile(double %val, ptr %addr) {
+; CHECK-LABEL: notatomic_double_generic_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_double_generic_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [notatomic_double_generic_volatile_param_1];
+; CHECK-NEXT: st.volatile.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store volatile double %val, ptr %addr
+ ret void
+}
+
+define void @notatomic_i8_global(i8 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: notatomic_i8_global(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [notatomic_i8_global_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i8_global_param_1];
+; CHECK-NEXT: st.global.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store i8 %val, ptr addrspace(1) %addr
+ ret void
+}
+
+define void @notatomic_i8_global_volatile(i8 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: notatomic_i8_global_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [notatomic_i8_global_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i8_global_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store volatile i8 %val, ptr addrspace(1) %addr
+ ret void
+}
+
+define void @notatomic_i16_global(i16 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: notatomic_i16_global(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [notatomic_i16_global_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i16_global_param_1];
+; CHECK-NEXT: st.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store i16 %val, ptr addrspace(1) %addr
+ ret void
+}
+
+define void @notatomic_i16_global_volatile(i16 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: notatomic_i16_global_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [notatomic_i16_global_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i16_global_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store volatile i16 %val, ptr addrspace(1) %addr
+ ret void
+}
+
+define void @notatomic_i32_global(i32 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: notatomic_i32_global(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [notatomic_i32_global_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i32_global_param_1];
+; CHECK-NEXT: st.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store i32 %val, ptr addrspace(1) %addr
+ ret void
+}
+
+define void @notatomic_i32_global_volatile(i32 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: notatomic_i32_global_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [notatomic_i32_global_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i32_global_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store volatile i32 %val, ptr addrspace(1) %addr
+ ret void
+}
+
+define void @notatomic_i64_global(i64 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: notatomic_i64_global(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i64_global_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [notatomic_i64_global_param_1];
+; CHECK-NEXT: st.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store i64 %val, ptr addrspace(1) %addr
+ ret void
+}
+
+define void @notatomic_i64_global_volatile(i64 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: notatomic_i64_global_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i64_global_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [notatomic_i64_global_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store volatile i64 %val, ptr addrspace(1) %addr
+ ret void
+}
+
+define void @notatomic_half_global(half %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: notatomic_half_global(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [notatomic_half_global_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_half_global_param_1];
+; CHECK-NEXT: st.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store half %val, ptr addrspace(1) %addr
+ ret void
+}
+
+define void @notatomic_half_global_volatile(half %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: notatomic_half_global_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [notatomic_half_global_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_half_global_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store volatile half %val, ptr addrspace(1) %addr
+ ret void
+}
+
+define void @notatomic_bfloat_global(bfloat %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: notatomic_bfloat_global(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [notatomic_bfloat_global_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_bfloat_global_param_1];
+; CHECK-NEXT: st.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store bfloat %val, ptr addrspace(1) %addr
+ ret void
+}
+
+define void @notatomic_bfloat_global_volatile(bfloat %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: notatomic_bfloat_global_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [notatomic_bfloat_global_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_bfloat_global_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store volatile bfloat %val, ptr addrspace(1) %addr
+ ret void
+}
+
+define void @notatomic_float_global(float %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: notatomic_float_global(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [notatomic_float_global_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_float_global_param_1];
+; CHECK-NEXT: st.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store float %val, ptr addrspace(1) %addr
+ ret void
+}
+
+define void @notatomic_float_global_volatile(float %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: notatomic_float_global_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [notatomic_float_global_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_float_global_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store volatile float %val, ptr addrspace(1) %addr
+ ret void
+}
+
+define void @notatomic_double_global(double %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: notatomic_double_global(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_double_global_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [notatomic_double_global_param_1];
+; CHECK-NEXT: st.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store double %val, ptr addrspace(1) %addr
+ ret void
+}
+
+define void @notatomic_double_global_volatile(double %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: notatomic_double_global_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_double_global_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [notatomic_double_global_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store volatile double %val, ptr addrspace(1) %addr
+ ret void
+}
+
+define void @notatomic_i8_shared(i8 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: notatomic_i8_shared(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [notatomic_i8_shared_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i8_shared_param_1];
+; CHECK-NEXT: st.shared.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store i8 %val, ptr addrspace(3) %addr
+ ret void
+}
+
+define void @notatomic_i8_shared_volatile(i8 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: notatomic_i8_shared_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [notatomic_i8_shared_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i8_shared_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store volatile i8 %val, ptr addrspace(3) %addr
+ ret void
+}
+
+define void @notatomic_i16_shared(i16 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: notatomic_i16_shared(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [notatomic_i16_shared_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i16_shared_param_1];
+; CHECK-NEXT: st.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store i16 %val, ptr addrspace(3) %addr
+ ret void
+}
+
+define void @notatomic_i16_shared_volatile(i16 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: notatomic_i16_shared_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [notatomic_i16_shared_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i16_shared_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store volatile i16 %val, ptr addrspace(3) %addr
+ ret void
+}
+
+define void @notatomic_i32_shared(i32 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: notatomic_i32_shared(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [notatomic_i32_shared_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i32_shared_param_1];
+; CHECK-NEXT: st.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store i32 %val, ptr addrspace(3) %addr
+ ret void
+}
+
+define void @notatomic_i32_shared_volatile(i32 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: notatomic_i32_shared_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [notatomic_i32_shared_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i32_shared_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store volatile i32 %val, ptr addrspace(3) %addr
+ ret void
+}
+
+define void @notatomic_i64_shared(i64 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: notatomic_i64_shared(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i64_shared_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [notatomic_i64_shared_param_1];
+; CHECK-NEXT: st.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store i64 %val, ptr addrspace(3) %addr
+ ret void
+}
+
+define void @notatomic_i64_shared_volatile(i64 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: notatomic_i64_shared_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i64_shared_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [notatomic_i64_shared_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store volatile i64 %val, ptr addrspace(3) %addr
+ ret void
+}
+
+define void @notatomic_half_shared(half %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: notatomic_half_shared(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [notatomic_half_shared_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_half_shared_param_1];
+; CHECK-NEXT: st.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store half %val, ptr addrspace(3) %addr
+ ret void
+}
+
+define void @notatomic_half_shared_volatile(half %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: notatomic_half_shared_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [notatomic_half_shared_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_half_shared_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store volatile half %val, ptr addrspace(3) %addr
+ ret void
+}
+
+define void @notatomic_bfloat_shared(bfloat %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: notatomic_bfloat_shared(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [notatomic_bfloat_shared_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_bfloat_shared_param_1];
+; CHECK-NEXT: st.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store bfloat %val, ptr addrspace(3) %addr
+ ret void
+}
+
+define void @notatomic_bfloat_shared_volatile(bfloat %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: notatomic_bfloat_shared_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [notatomic_bfloat_shared_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_bfloat_shared_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store volatile bfloat %val, ptr addrspace(3) %addr
+ ret void
+}
+
+define void @notatomic_float_shared(float %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: notatomic_float_shared(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [notatomic_float_shared_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_float_shared_param_1];
+; CHECK-NEXT: st.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store float %val, ptr addrspace(3) %addr
+ ret void
+}
+
+define void @notatomic_float_shared_volatile(float %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: notatomic_float_shared_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [notatomic_float_shared_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_float_shared_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store volatile float %val, ptr addrspace(3) %addr
+ ret void
+}
+
+define void @notatomic_double_shared(double %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: notatomic_double_shared(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_double_shared_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [notatomic_double_shared_param_1];
+; CHECK-NEXT: st.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store double %val, ptr addrspace(3) %addr
+ ret void
+}
+
+define void @notatomic_double_shared_volatile(double %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: notatomic_double_shared_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_double_shared_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [notatomic_double_shared_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store volatile double %val, ptr addrspace(3) %addr
+ ret void
+}
+
+define void @notatomic_i8_local(i8 %val) {
+; CHECK-LABEL: notatomic_i8_local(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot48[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot48;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b8 %rs1, [notatomic_i8_local_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b8 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store i8 %val, ptr addrspace(5) %slot
+ ret void
+}
+
+define void @notatomic_i8_local_volatile(i8 %val) {
+; CHECK-LABEL: notatomic_i8_local_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot49[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot49;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b8 %rs1, [notatomic_i8_local_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b8 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store volatile i8 %val, ptr addrspace(5) %slot
+ ret void
+}
+
+define void @notatomic_i16_local(i16 %val) {
+; CHECK-LABEL: notatomic_i16_local(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot50[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot50;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [notatomic_i16_local_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store i16 %val, ptr addrspace(5) %slot
+ ret void
+}
+
+define void @notatomic_i16_local_volatile(i16 %val) {
+; CHECK-LABEL: notatomic_i16_local_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot51[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot51;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [notatomic_i16_local_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store volatile i16 %val, ptr addrspace(5) %slot
+ ret void
+}
+
+define void @notatomic_i32_local(i32 %val) {
+; CHECK-LABEL: notatomic_i32_local(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot52[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot52;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [notatomic_i32_local_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store i32 %val, ptr addrspace(5) %slot
+ ret void
+}
+
+define void @notatomic_i32_local_volatile(i32 %val) {
+; CHECK-LABEL: notatomic_i32_local_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot53[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot53;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [notatomic_i32_local_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store volatile i32 %val, ptr addrspace(5) %slot
+ ret void
+}
+
+define void @notatomic_i64_local(i64 %val) {
+; CHECK-LABEL: notatomic_i64_local(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot54[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot54;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i64_local_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store i64 %val, ptr addrspace(5) %slot
+ ret void
+}
+
+define void @notatomic_i64_local_volatile(i64 %val) {
+; CHECK-LABEL: notatomic_i64_local_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot55[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot55;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i64_local_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store volatile i64 %val, ptr addrspace(5) %slot
+ ret void
+}
+
+define void @notatomic_half_local(half %val) {
+; CHECK-LABEL: notatomic_half_local(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot56[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot56;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [notatomic_half_local_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store half %val, ptr addrspace(5) %slot
+ ret void
+}
+
+define void @notatomic_half_local_volatile(half %val) {
+; CHECK-LABEL: notatomic_half_local_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot57[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot57;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [notatomic_half_local_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store volatile half %val, ptr addrspace(5) %slot
+ ret void
+}
+
+define void @notatomic_bfloat_local(bfloat %val) {
+; CHECK-LABEL: notatomic_bfloat_local(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot58[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot58;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [notatomic_bfloat_local_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store bfloat %val, ptr addrspace(5) %slot
+ ret void
+}
+
+define void @notatomic_bfloat_local_volatile(bfloat %val) {
+; CHECK-LABEL: notatomic_bfloat_local_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot59[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot59;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [notatomic_bfloat_local_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store volatile bfloat %val, ptr addrspace(5) %slot
+ ret void
+}
+
+define void @notatomic_float_local(float %val) {
+; CHECK-LABEL: notatomic_float_local(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot60[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot60;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [notatomic_float_local_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store float %val, ptr addrspace(5) %slot
+ ret void
+}
+
+define void @notatomic_float_local_volatile(float %val) {
+; CHECK-LABEL: notatomic_float_local_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot61[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot61;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [notatomic_float_local_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store volatile float %val, ptr addrspace(5) %slot
+ ret void
+}
+
+define void @notatomic_double_local(double %val) {
+; CHECK-LABEL: notatomic_double_local(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot62[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot62;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_double_local_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca double, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store double %val, ptr addrspace(5) %slot
+ ret void
+}
+
+define void @notatomic_double_local_volatile(double %val) {
+; CHECK-LABEL: notatomic_double_local_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot63[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot63;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_double_local_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca double, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store volatile double %val, ptr addrspace(5) %slot
+ ret void
+}
+
+define void @unordered_i8_generic_sys(i8 %val, ptr %addr) {
+; CHECK-LABEL: unordered_i8_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [unordered_i8_generic_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_generic_sys_param_1];
+; CHECK-NEXT: st.volatile.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr %addr syncscope("") unordered, align 1
+ ret void
+}
+
+define void @unordered_i8_generic_cta(i8 %val, ptr %addr) {
+; CHECK-LABEL: unordered_i8_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [unordered_i8_generic_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_generic_cta_param_1];
+; CHECK-NEXT: st.volatile.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr %addr syncscope("block") unordered, align 1
+ ret void
+}
+
+define void @unordered_i8_generic_cluster(i8 %val, ptr %addr) {
+; CHECK-LABEL: unordered_i8_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [unordered_i8_generic_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_generic_cluster_param_1];
+; CHECK-NEXT: st.volatile.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr %addr syncscope("cluster") unordered, align 1
+ ret void
+}
+
+define void @unordered_i8_generic_gpu(i8 %val, ptr %addr) {
+; CHECK-LABEL: unordered_i8_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [unordered_i8_generic_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_generic_gpu_param_1];
+; CHECK-NEXT: st.volatile.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr %addr syncscope("device") unordered, align 1
+ ret void
+}
+
+define void @unordered_i8_generic_sys_volatile(i8 %val, ptr %addr) {
+; CHECK-LABEL: unordered_i8_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [unordered_i8_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_generic_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr %addr syncscope("") unordered, align 1
+ ret void
+}
+
+define void @unordered_i8_generic_cta_volatile(i8 %val, ptr %addr) {
+; CHECK-LABEL: unordered_i8_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [unordered_i8_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_generic_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr %addr syncscope("block") unordered, align 1
+ ret void
+}
+
+define void @unordered_i8_generic_cluster_volatile(i8 %val, ptr %addr) {
+; CHECK-LABEL: unordered_i8_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [unordered_i8_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_generic_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr %addr syncscope("cluster") unordered, align 1
+ ret void
+}
+
+define void @unordered_i8_generic_gpu_volatile(i8 %val, ptr %addr) {
+; CHECK-LABEL: unordered_i8_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [unordered_i8_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_generic_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr %addr syncscope("device") unordered, align 1
+ ret void
+}
+
+define void @unordered_i16_generic_sys(i16 %val, ptr %addr) {
+; CHECK-LABEL: unordered_i16_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_i16_generic_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_generic_sys_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr %addr syncscope("") unordered, align 2
+ ret void
+}
+
+define void @unordered_i16_generic_cta(i16 %val, ptr %addr) {
+; CHECK-LABEL: unordered_i16_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_i16_generic_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_generic_cta_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr %addr syncscope("block") unordered, align 2
+ ret void
+}
+
+define void @unordered_i16_generic_cluster(i16 %val, ptr %addr) {
+; CHECK-LABEL: unordered_i16_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_i16_generic_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_generic_cluster_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr %addr syncscope("cluster") unordered, align 2
+ ret void
+}
+
+define void @unordered_i16_generic_gpu(i16 %val, ptr %addr) {
+; CHECK-LABEL: unordered_i16_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_i16_generic_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_generic_gpu_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr %addr syncscope("device") unordered, align 2
+ ret void
+}
+
+define void @unordered_i16_generic_sys_volatile(i16 %val, ptr %addr) {
+; CHECK-LABEL: unordered_i16_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_i16_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_generic_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr %addr syncscope("") unordered, align 2
+ ret void
+}
+
+define void @unordered_i16_generic_cta_volatile(i16 %val, ptr %addr) {
+; CHECK-LABEL: unordered_i16_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_i16_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_generic_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr %addr syncscope("block") unordered, align 2
+ ret void
+}
+
+define void @unordered_i16_generic_cluster_volatile(i16 %val, ptr %addr) {
+; CHECK-LABEL: unordered_i16_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_i16_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_generic_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr %addr syncscope("cluster") unordered, align 2
+ ret void
+}
+
+define void @unordered_i16_generic_gpu_volatile(i16 %val, ptr %addr) {
+; CHECK-LABEL: unordered_i16_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_i16_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_generic_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr %addr syncscope("device") unordered, align 2
+ ret void
+}
+
+define void @unordered_i32_generic_sys(i32 %val, ptr %addr) {
+; CHECK-LABEL: unordered_i32_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_i32_generic_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_generic_sys_param_1];
+; CHECK-NEXT: st.volatile.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr %addr syncscope("") unordered, align 4
+ ret void
+}
+
+define void @unordered_i32_generic_cta(i32 %val, ptr %addr) {
+; CHECK-LABEL: unordered_i32_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_i32_generic_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_generic_cta_param_1];
+; CHECK-NEXT: st.volatile.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr %addr syncscope("block") unordered, align 4
+ ret void
+}
+
+define void @unordered_i32_generic_cluster(i32 %val, ptr %addr) {
+; CHECK-LABEL: unordered_i32_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_i32_generic_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_generic_cluster_param_1];
+; CHECK-NEXT: st.volatile.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr %addr syncscope("cluster") unordered, align 4
+ ret void
+}
+
+define void @unordered_i32_generic_gpu(i32 %val, ptr %addr) {
+; CHECK-LABEL: unordered_i32_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_i32_generic_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_generic_gpu_param_1];
+; CHECK-NEXT: st.volatile.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr %addr syncscope("device") unordered, align 4
+ ret void
+}
+
+define void @unordered_i32_generic_sys_volatile(i32 %val, ptr %addr) {
+; CHECK-LABEL: unordered_i32_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_i32_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_generic_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr %addr syncscope("") unordered, align 4
+ ret void
+}
+
+define void @unordered_i32_generic_cta_volatile(i32 %val, ptr %addr) {
+; CHECK-LABEL: unordered_i32_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_i32_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_generic_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr %addr syncscope("block") unordered, align 4
+ ret void
+}
+
+define void @unordered_i32_generic_cluster_volatile(i32 %val, ptr %addr) {
+; CHECK-LABEL: unordered_i32_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_i32_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_generic_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr %addr syncscope("cluster") unordered, align 4
+ ret void
+}
+
+define void @unordered_i32_generic_gpu_volatile(i32 %val, ptr %addr) {
+; CHECK-LABEL: unordered_i32_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_i32_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_generic_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr %addr syncscope("device") unordered, align 4
+ ret void
+}
+
+define void @unordered_i64_generic_sys(i64 %val, ptr %addr) {
+; CHECK-LABEL: unordered_i64_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_generic_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_i64_generic_sys_param_1];
+; CHECK-NEXT: st.volatile.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr %addr syncscope("") unordered, align 8
+ ret void
+}
+
+define void @unordered_i64_generic_cta(i64 %val, ptr %addr) {
+; CHECK-LABEL: unordered_i64_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_generic_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_i64_generic_cta_param_1];
+; CHECK-NEXT: st.volatile.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr %addr syncscope("block") unordered, align 8
+ ret void
+}
+
+define void @unordered_i64_generic_cluster(i64 %val, ptr %addr) {
+; CHECK-LABEL: unordered_i64_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_generic_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_i64_generic_cluster_param_1];
+; CHECK-NEXT: st.volatile.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr %addr syncscope("cluster") unordered, align 8
+ ret void
+}
+
+define void @unordered_i64_generic_gpu(i64 %val, ptr %addr) {
+; CHECK-LABEL: unordered_i64_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_generic_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_i64_generic_gpu_param_1];
+; CHECK-NEXT: st.volatile.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr %addr syncscope("device") unordered, align 8
+ ret void
+}
+
+define void @unordered_i64_generic_sys_volatile(i64 %val, ptr %addr) {
+; CHECK-LABEL: unordered_i64_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_i64_generic_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr %addr syncscope("") unordered, align 8
+ ret void
+}
+
+define void @unordered_i64_generic_cta_volatile(i64 %val, ptr %addr) {
+; CHECK-LABEL: unordered_i64_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_i64_generic_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr %addr syncscope("block") unordered, align 8
+ ret void
+}
+
+define void @unordered_i64_generic_cluster_volatile(i64 %val, ptr %addr) {
+; CHECK-LABEL: unordered_i64_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_i64_generic_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr %addr syncscope("cluster") unordered, align 8
+ ret void
+}
+
+define void @unordered_i64_generic_gpu_volatile(i64 %val, ptr %addr) {
+; CHECK-LABEL: unordered_i64_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_i64_generic_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr %addr syncscope("device") unordered, align 8
+ ret void
+}
+
+define void @unordered_half_generic_sys(half %val, ptr %addr) {
+; CHECK-LABEL: unordered_half_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_half_generic_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_generic_sys_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr %addr syncscope("") unordered, align 2
+ ret void
+}
+
+define void @unordered_half_generic_cta(half %val, ptr %addr) {
+; CHECK-LABEL: unordered_half_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_half_generic_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_generic_cta_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr %addr syncscope("block") unordered, align 2
+ ret void
+}
+
+define void @unordered_half_generic_cluster(half %val, ptr %addr) {
+; CHECK-LABEL: unordered_half_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_half_generic_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_generic_cluster_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr %addr syncscope("cluster") unordered, align 2
+ ret void
+}
+
+define void @unordered_half_generic_gpu(half %val, ptr %addr) {
+; CHECK-LABEL: unordered_half_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_half_generic_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_generic_gpu_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr %addr syncscope("device") unordered, align 2
+ ret void
+}
+
+define void @unordered_half_generic_sys_volatile(half %val, ptr %addr) {
+; CHECK-LABEL: unordered_half_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_half_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_generic_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr %addr syncscope("") unordered, align 2
+ ret void
+}
+
+define void @unordered_half_generic_cta_volatile(half %val, ptr %addr) {
+; CHECK-LABEL: unordered_half_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_half_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_generic_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr %addr syncscope("block") unordered, align 2
+ ret void
+}
+
+define void @unordered_half_generic_cluster_volatile(half %val, ptr %addr) {
+; CHECK-LABEL: unordered_half_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_half_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_generic_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr %addr syncscope("cluster") unordered, align 2
+ ret void
+}
+
+define void @unordered_half_generic_gpu_volatile(half %val, ptr %addr) {
+; CHECK-LABEL: unordered_half_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_half_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_generic_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr %addr syncscope("device") unordered, align 2
+ ret void
+}
+
+define void @unordered_bfloat_generic_sys(bfloat %val, ptr %addr) {
+; CHECK-LABEL: unordered_bfloat_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_bfloat_generic_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_generic_sys_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr %addr syncscope("") unordered, align 2
+ ret void
+}
+
+define void @unordered_bfloat_generic_cta(bfloat %val, ptr %addr) {
+; CHECK-LABEL: unordered_bfloat_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_bfloat_generic_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_generic_cta_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr %addr syncscope("block") unordered, align 2
+ ret void
+}
+
+define void @unordered_bfloat_generic_cluster(bfloat %val, ptr %addr) {
+; CHECK-LABEL: unordered_bfloat_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_bfloat_generic_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_generic_cluster_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr %addr syncscope("cluster") unordered, align 2
+ ret void
+}
+
+define void @unordered_bfloat_generic_gpu(bfloat %val, ptr %addr) {
+; CHECK-LABEL: unordered_bfloat_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_bfloat_generic_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_generic_gpu_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr %addr syncscope("device") unordered, align 2
+ ret void
+}
+
+define void @unordered_bfloat_generic_sys_volatile(bfloat %val, ptr %addr) {
+; CHECK-LABEL: unordered_bfloat_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_bfloat_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_generic_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr %addr syncscope("") unordered, align 2
+ ret void
+}
+
+define void @unordered_bfloat_generic_cta_volatile(bfloat %val, ptr %addr) {
+; CHECK-LABEL: unordered_bfloat_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_bfloat_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_generic_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr %addr syncscope("block") unordered, align 2
+ ret void
+}
+
+define void @unordered_bfloat_generic_cluster_volatile(bfloat %val, ptr %addr) {
+; CHECK-LABEL: unordered_bfloat_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_bfloat_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_generic_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr %addr syncscope("cluster") unordered, align 2
+ ret void
+}
+
+define void @unordered_bfloat_generic_gpu_volatile(bfloat %val, ptr %addr) {
+; CHECK-LABEL: unordered_bfloat_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_bfloat_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_generic_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr %addr syncscope("device") unordered, align 2
+ ret void
+}
+
+define void @unordered_float_generic_sys(float %val, ptr %addr) {
+; CHECK-LABEL: unordered_float_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_float_generic_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_generic_sys_param_1];
+; CHECK-NEXT: st.volatile.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr %addr syncscope("") unordered, align 4
+ ret void
+}
+
+define void @unordered_float_generic_cta(float %val, ptr %addr) {
+; CHECK-LABEL: unordered_float_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_float_generic_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_generic_cta_param_1];
+; CHECK-NEXT: st.volatile.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr %addr syncscope("block") unordered, align 4
+ ret void
+}
+
+define void @unordered_float_generic_cluster(float %val, ptr %addr) {
+; CHECK-LABEL: unordered_float_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_float_generic_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_generic_cluster_param_1];
+; CHECK-NEXT: st.volatile.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr %addr syncscope("cluster") unordered, align 4
+ ret void
+}
+
+define void @unordered_float_generic_gpu(float %val, ptr %addr) {
+; CHECK-LABEL: unordered_float_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_float_generic_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_generic_gpu_param_1];
+; CHECK-NEXT: st.volatile.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr %addr syncscope("device") unordered, align 4
+ ret void
+}
+
+define void @unordered_float_generic_sys_volatile(float %val, ptr %addr) {
+; CHECK-LABEL: unordered_float_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_float_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_generic_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr %addr syncscope("") unordered, align 4
+ ret void
+}
+
+define void @unordered_float_generic_cta_volatile(float %val, ptr %addr) {
+; CHECK-LABEL: unordered_float_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_float_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_generic_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr %addr syncscope("block") unordered, align 4
+ ret void
+}
+
+define void @unordered_float_generic_cluster_volatile(float %val, ptr %addr) {
+; CHECK-LABEL: unordered_float_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_float_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_generic_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr %addr syncscope("cluster") unordered, align 4
+ ret void
+}
+
+define void @unordered_float_generic_gpu_volatile(float %val, ptr %addr) {
+; CHECK-LABEL: unordered_float_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_float_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_generic_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr %addr syncscope("device") unordered, align 4
+ ret void
+}
+
+define void @unordered_double_generic_sys(double %val, ptr %addr) {
+; CHECK-LABEL: unordered_double_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_generic_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_double_generic_sys_param_1];
+; CHECK-NEXT: st.volatile.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr %addr syncscope("") unordered, align 8
+ ret void
+}
+
+define void @unordered_double_generic_cta(double %val, ptr %addr) {
+; CHECK-LABEL: unordered_double_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_generic_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_double_generic_cta_param_1];
+; CHECK-NEXT: st.volatile.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr %addr syncscope("block") unordered, align 8
+ ret void
+}
+
+define void @unordered_double_generic_cluster(double %val, ptr %addr) {
+; CHECK-LABEL: unordered_double_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_generic_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_double_generic_cluster_param_1];
+; CHECK-NEXT: st.volatile.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr %addr syncscope("cluster") unordered, align 8
+ ret void
+}
+
+define void @unordered_double_generic_gpu(double %val, ptr %addr) {
+; CHECK-LABEL: unordered_double_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_generic_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_double_generic_gpu_param_1];
+; CHECK-NEXT: st.volatile.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr %addr syncscope("device") unordered, align 8
+ ret void
+}
+
+define void @unordered_double_generic_sys_volatile(double %val, ptr %addr) {
+; CHECK-LABEL: unordered_double_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_double_generic_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr %addr syncscope("") unordered, align 8
+ ret void
+}
+
+define void @unordered_double_generic_cta_volatile(double %val, ptr %addr) {
+; CHECK-LABEL: unordered_double_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_double_generic_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr %addr syncscope("block") unordered, align 8
+ ret void
+}
+
+define void @unordered_double_generic_cluster_volatile(double %val, ptr %addr) {
+; CHECK-LABEL: unordered_double_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_double_generic_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr %addr syncscope("cluster") unordered, align 8
+ ret void
+}
+
+define void @unordered_double_generic_gpu_volatile(double %val, ptr %addr) {
+; CHECK-LABEL: unordered_double_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_double_generic_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr %addr syncscope("device") unordered, align 8
+ ret void
+}
+
+define void @unordered_i8_global_sys(i8 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i8_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [unordered_i8_global_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_global_sys_param_1];
+; CHECK-NEXT: st.volatile.global.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr addrspace(1) %addr syncscope("") unordered, align 1
+ ret void
+}
+
+define void @unordered_i8_global_cta(i8 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i8_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [unordered_i8_global_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_global_cta_param_1];
+; CHECK-NEXT: st.volatile.global.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr addrspace(1) %addr syncscope("block") unordered, align 1
+ ret void
+}
+
+define void @unordered_i8_global_cluster(i8 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i8_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [unordered_i8_global_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_global_cluster_param_1];
+; CHECK-NEXT: st.volatile.global.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr addrspace(1) %addr syncscope("cluster") unordered, align 1
+ ret void
+}
+
+define void @unordered_i8_global_gpu(i8 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i8_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [unordered_i8_global_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_global_gpu_param_1];
+; CHECK-NEXT: st.volatile.global.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr addrspace(1) %addr syncscope("device") unordered, align 1
+ ret void
+}
+
+define void @unordered_i8_global_sys_volatile(i8 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i8_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [unordered_i8_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_global_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr addrspace(1) %addr syncscope("") unordered, align 1
+ ret void
+}
+
+define void @unordered_i8_global_cta_volatile(i8 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i8_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [unordered_i8_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_global_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr addrspace(1) %addr syncscope("block") unordered, align 1
+ ret void
+}
+
+define void @unordered_i8_global_cluster_volatile(i8 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i8_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [unordered_i8_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_global_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr addrspace(1) %addr syncscope("cluster") unordered, align 1
+ ret void
+}
+
+define void @unordered_i8_global_gpu_volatile(i8 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i8_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [unordered_i8_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_global_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr addrspace(1) %addr syncscope("device") unordered, align 1
+ ret void
+}
+
+define void @unordered_i16_global_sys(i16 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i16_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_i16_global_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_global_sys_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr addrspace(1) %addr syncscope("") unordered, align 2
+ ret void
+}
+
+define void @unordered_i16_global_cta(i16 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i16_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_i16_global_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_global_cta_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr addrspace(1) %addr syncscope("block") unordered, align 2
+ ret void
+}
+
+define void @unordered_i16_global_cluster(i16 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i16_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_i16_global_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_global_cluster_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr addrspace(1) %addr syncscope("cluster") unordered, align 2
+ ret void
+}
+
+define void @unordered_i16_global_gpu(i16 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i16_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_i16_global_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_global_gpu_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr addrspace(1) %addr syncscope("device") unordered, align 2
+ ret void
+}
+
+define void @unordered_i16_global_sys_volatile(i16 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i16_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_i16_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_global_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr addrspace(1) %addr syncscope("") unordered, align 2
+ ret void
+}
+
+define void @unordered_i16_global_cta_volatile(i16 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i16_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_i16_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_global_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr addrspace(1) %addr syncscope("block") unordered, align 2
+ ret void
+}
+
+define void @unordered_i16_global_cluster_volatile(i16 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i16_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_i16_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_global_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr addrspace(1) %addr syncscope("cluster") unordered, align 2
+ ret void
+}
+
+define void @unordered_i16_global_gpu_volatile(i16 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i16_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_i16_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_global_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr addrspace(1) %addr syncscope("device") unordered, align 2
+ ret void
+}
+
+define void @unordered_i32_global_sys(i32 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i32_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_i32_global_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_global_sys_param_1];
+; CHECK-NEXT: st.volatile.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr addrspace(1) %addr syncscope("") unordered, align 4
+ ret void
+}
+
+define void @unordered_i32_global_cta(i32 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i32_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_i32_global_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_global_cta_param_1];
+; CHECK-NEXT: st.volatile.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr addrspace(1) %addr syncscope("block") unordered, align 4
+ ret void
+}
+
+define void @unordered_i32_global_cluster(i32 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i32_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_i32_global_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_global_cluster_param_1];
+; CHECK-NEXT: st.volatile.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr addrspace(1) %addr syncscope("cluster") unordered, align 4
+ ret void
+}
+
+define void @unordered_i32_global_gpu(i32 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i32_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_i32_global_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_global_gpu_param_1];
+; CHECK-NEXT: st.volatile.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr addrspace(1) %addr syncscope("device") unordered, align 4
+ ret void
+}
+
+define void @unordered_i32_global_sys_volatile(i32 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i32_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_i32_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_global_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr addrspace(1) %addr syncscope("") unordered, align 4
+ ret void
+}
+
+define void @unordered_i32_global_cta_volatile(i32 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i32_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_i32_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_global_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr addrspace(1) %addr syncscope("block") unordered, align 4
+ ret void
+}
+
+define void @unordered_i32_global_cluster_volatile(i32 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i32_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_i32_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_global_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr addrspace(1) %addr syncscope("cluster") unordered, align 4
+ ret void
+}
+
+define void @unordered_i32_global_gpu_volatile(i32 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i32_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_i32_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_global_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr addrspace(1) %addr syncscope("device") unordered, align 4
+ ret void
+}
+
+define void @unordered_i64_global_sys(i64 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i64_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_global_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_i64_global_sys_param_1];
+; CHECK-NEXT: st.volatile.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr addrspace(1) %addr syncscope("") unordered, align 8
+ ret void
+}
+
+define void @unordered_i64_global_cta(i64 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i64_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_global_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_i64_global_cta_param_1];
+; CHECK-NEXT: st.volatile.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr addrspace(1) %addr syncscope("block") unordered, align 8
+ ret void
+}
+
+define void @unordered_i64_global_cluster(i64 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i64_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_global_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_i64_global_cluster_param_1];
+; CHECK-NEXT: st.volatile.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr addrspace(1) %addr syncscope("cluster") unordered, align 8
+ ret void
+}
+
+define void @unordered_i64_global_gpu(i64 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i64_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_global_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_i64_global_gpu_param_1];
+; CHECK-NEXT: st.volatile.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr addrspace(1) %addr syncscope("device") unordered, align 8
+ ret void
+}
+
+define void @unordered_i64_global_sys_volatile(i64 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i64_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_i64_global_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr addrspace(1) %addr syncscope("") unordered, align 8
+ ret void
+}
+
+define void @unordered_i64_global_cta_volatile(i64 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i64_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_i64_global_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr addrspace(1) %addr syncscope("block") unordered, align 8
+ ret void
+}
+
+define void @unordered_i64_global_cluster_volatile(i64 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i64_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_i64_global_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr addrspace(1) %addr syncscope("cluster") unordered, align 8
+ ret void
+}
+
+define void @unordered_i64_global_gpu_volatile(i64 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i64_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_i64_global_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr addrspace(1) %addr syncscope("device") unordered, align 8
+ ret void
+}
+
+define void @unordered_half_global_sys(half %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_half_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_half_global_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_global_sys_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr addrspace(1) %addr syncscope("") unordered, align 2
+ ret void
+}
+
+define void @unordered_half_global_cta(half %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_half_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_half_global_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_global_cta_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr addrspace(1) %addr syncscope("block") unordered, align 2
+ ret void
+}
+
+define void @unordered_half_global_cluster(half %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_half_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_half_global_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_global_cluster_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr addrspace(1) %addr syncscope("cluster") unordered, align 2
+ ret void
+}
+
+define void @unordered_half_global_gpu(half %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_half_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_half_global_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_global_gpu_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr addrspace(1) %addr syncscope("device") unordered, align 2
+ ret void
+}
+
+define void @unordered_half_global_sys_volatile(half %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_half_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_half_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_global_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr addrspace(1) %addr syncscope("") unordered, align 2
+ ret void
+}
+
+define void @unordered_half_global_cta_volatile(half %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_half_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_half_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_global_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr addrspace(1) %addr syncscope("block") unordered, align 2
+ ret void
+}
+
+define void @unordered_half_global_cluster_volatile(half %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_half_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_half_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_global_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr addrspace(1) %addr syncscope("cluster") unordered, align 2
+ ret void
+}
+
+define void @unordered_half_global_gpu_volatile(half %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_half_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_half_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_global_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr addrspace(1) %addr syncscope("device") unordered, align 2
+ ret void
+}
+
+define void @unordered_bfloat_global_sys(bfloat %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_bfloat_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_bfloat_global_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_global_sys_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr addrspace(1) %addr syncscope("") unordered, align 2
+ ret void
+}
+
+define void @unordered_bfloat_global_cta(bfloat %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_bfloat_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_bfloat_global_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_global_cta_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr addrspace(1) %addr syncscope("block") unordered, align 2
+ ret void
+}
+
+define void @unordered_bfloat_global_cluster(bfloat %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_bfloat_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_bfloat_global_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_global_cluster_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr addrspace(1) %addr syncscope("cluster") unordered, align 2
+ ret void
+}
+
+define void @unordered_bfloat_global_gpu(bfloat %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_bfloat_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_bfloat_global_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_global_gpu_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr addrspace(1) %addr syncscope("device") unordered, align 2
+ ret void
+}
+
+define void @unordered_bfloat_global_sys_volatile(bfloat %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_bfloat_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_bfloat_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_global_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr addrspace(1) %addr syncscope("") unordered, align 2
+ ret void
+}
+
+define void @unordered_bfloat_global_cta_volatile(bfloat %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_bfloat_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_bfloat_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_global_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr addrspace(1) %addr syncscope("block") unordered, align 2
+ ret void
+}
+
+define void @unordered_bfloat_global_cluster_volatile(bfloat %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_bfloat_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_bfloat_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_global_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr addrspace(1) %addr syncscope("cluster") unordered, align 2
+ ret void
+}
+
+define void @unordered_bfloat_global_gpu_volatile(bfloat %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_bfloat_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_bfloat_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_global_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr addrspace(1) %addr syncscope("device") unordered, align 2
+ ret void
+}
+
+define void @unordered_float_global_sys(float %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_float_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_float_global_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_global_sys_param_1];
+; CHECK-NEXT: st.volatile.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr addrspace(1) %addr syncscope("") unordered, align 4
+ ret void
+}
+
+define void @unordered_float_global_cta(float %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_float_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_float_global_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_global_cta_param_1];
+; CHECK-NEXT: st.volatile.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr addrspace(1) %addr syncscope("block") unordered, align 4
+ ret void
+}
+
+define void @unordered_float_global_cluster(float %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_float_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_float_global_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_global_cluster_param_1];
+; CHECK-NEXT: st.volatile.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr addrspace(1) %addr syncscope("cluster") unordered, align 4
+ ret void
+}
+
+define void @unordered_float_global_gpu(float %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_float_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_float_global_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_global_gpu_param_1];
+; CHECK-NEXT: st.volatile.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr addrspace(1) %addr syncscope("device") unordered, align 4
+ ret void
+}
+
+define void @unordered_float_global_sys_volatile(float %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_float_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_float_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_global_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr addrspace(1) %addr syncscope("") unordered, align 4
+ ret void
+}
+
+define void @unordered_float_global_cta_volatile(float %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_float_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_float_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_global_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr addrspace(1) %addr syncscope("block") unordered, align 4
+ ret void
+}
+
+define void @unordered_float_global_cluster_volatile(float %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_float_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_float_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_global_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr addrspace(1) %addr syncscope("cluster") unordered, align 4
+ ret void
+}
+
+define void @unordered_float_global_gpu_volatile(float %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_float_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_float_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_global_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr addrspace(1) %addr syncscope("device") unordered, align 4
+ ret void
+}
+
+define void @unordered_double_global_sys(double %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_double_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_global_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_double_global_sys_param_1];
+; CHECK-NEXT: st.volatile.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr addrspace(1) %addr syncscope("") unordered, align 8
+ ret void
+}
+
+define void @unordered_double_global_cta(double %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_double_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_global_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_double_global_cta_param_1];
+; CHECK-NEXT: st.volatile.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr addrspace(1) %addr syncscope("block") unordered, align 8
+ ret void
+}
+
+define void @unordered_double_global_cluster(double %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_double_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_global_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_double_global_cluster_param_1];
+; CHECK-NEXT: st.volatile.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr addrspace(1) %addr syncscope("cluster") unordered, align 8
+ ret void
+}
+
+define void @unordered_double_global_gpu(double %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_double_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_global_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_double_global_gpu_param_1];
+; CHECK-NEXT: st.volatile.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr addrspace(1) %addr syncscope("device") unordered, align 8
+ ret void
+}
+
+define void @unordered_double_global_sys_volatile(double %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_double_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_double_global_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr addrspace(1) %addr syncscope("") unordered, align 8
+ ret void
+}
+
+define void @unordered_double_global_cta_volatile(double %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_double_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_double_global_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr addrspace(1) %addr syncscope("block") unordered, align 8
+ ret void
+}
+
+define void @unordered_double_global_cluster_volatile(double %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_double_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_double_global_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr addrspace(1) %addr syncscope("cluster") unordered, align 8
+ ret void
+}
+
+define void @unordered_double_global_gpu_volatile(double %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_double_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_double_global_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr addrspace(1) %addr syncscope("device") unordered, align 8
+ ret void
+}
+
+define void @unordered_i8_shared_sys(i8 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i8_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [unordered_i8_shared_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_shared_sys_param_1];
+; CHECK-NEXT: st.volatile.shared.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr addrspace(3) %addr syncscope("") unordered, align 1
+ ret void
+}
+
+define void @unordered_i8_shared_cta(i8 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i8_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [unordered_i8_shared_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_shared_cta_param_1];
+; CHECK-NEXT: st.volatile.shared.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr addrspace(3) %addr syncscope("block") unordered, align 1
+ ret void
+}
+
+define void @unordered_i8_shared_cluster(i8 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i8_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [unordered_i8_shared_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_shared_cluster_param_1];
+; CHECK-NEXT: st.volatile.shared.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr addrspace(3) %addr syncscope("cluster") unordered, align 1
+ ret void
+}
+
+define void @unordered_i8_shared_gpu(i8 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i8_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [unordered_i8_shared_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_shared_gpu_param_1];
+; CHECK-NEXT: st.volatile.shared.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr addrspace(3) %addr syncscope("device") unordered, align 1
+ ret void
+}
+
+define void @unordered_i8_shared_sys_volatile(i8 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i8_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [unordered_i8_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_shared_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr addrspace(3) %addr syncscope("") unordered, align 1
+ ret void
+}
+
+define void @unordered_i8_shared_cta_volatile(i8 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i8_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [unordered_i8_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_shared_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr addrspace(3) %addr syncscope("block") unordered, align 1
+ ret void
+}
+
+define void @unordered_i8_shared_cluster_volatile(i8 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i8_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [unordered_i8_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_shared_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr addrspace(3) %addr syncscope("cluster") unordered, align 1
+ ret void
+}
+
+define void @unordered_i8_shared_gpu_volatile(i8 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i8_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [unordered_i8_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_shared_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr addrspace(3) %addr syncscope("device") unordered, align 1
+ ret void
+}
+
+define void @unordered_i16_shared_sys(i16 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i16_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_i16_shared_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_shared_sys_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr addrspace(3) %addr syncscope("") unordered, align 2
+ ret void
+}
+
+define void @unordered_i16_shared_cta(i16 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i16_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_i16_shared_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_shared_cta_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr addrspace(3) %addr syncscope("block") unordered, align 2
+ ret void
+}
+
+define void @unordered_i16_shared_cluster(i16 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i16_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_i16_shared_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_shared_cluster_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr addrspace(3) %addr syncscope("cluster") unordered, align 2
+ ret void
+}
+
+define void @unordered_i16_shared_gpu(i16 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i16_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_i16_shared_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_shared_gpu_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr addrspace(3) %addr syncscope("device") unordered, align 2
+ ret void
+}
+
+define void @unordered_i16_shared_sys_volatile(i16 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i16_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_i16_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_shared_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr addrspace(3) %addr syncscope("") unordered, align 2
+ ret void
+}
+
+define void @unordered_i16_shared_cta_volatile(i16 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i16_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_i16_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_shared_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr addrspace(3) %addr syncscope("block") unordered, align 2
+ ret void
+}
+
+define void @unordered_i16_shared_cluster_volatile(i16 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i16_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_i16_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_shared_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr addrspace(3) %addr syncscope("cluster") unordered, align 2
+ ret void
+}
+
+define void @unordered_i16_shared_gpu_volatile(i16 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i16_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_i16_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_shared_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr addrspace(3) %addr syncscope("device") unordered, align 2
+ ret void
+}
+
+define void @unordered_i32_shared_sys(i32 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i32_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_i32_shared_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_shared_sys_param_1];
+; CHECK-NEXT: st.volatile.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr addrspace(3) %addr syncscope("") unordered, align 4
+ ret void
+}
+
+define void @unordered_i32_shared_cta(i32 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i32_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_i32_shared_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_shared_cta_param_1];
+; CHECK-NEXT: st.volatile.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr addrspace(3) %addr syncscope("block") unordered, align 4
+ ret void
+}
+
+define void @unordered_i32_shared_cluster(i32 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i32_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_i32_shared_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_shared_cluster_param_1];
+; CHECK-NEXT: st.volatile.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr addrspace(3) %addr syncscope("cluster") unordered, align 4
+ ret void
+}
+
+define void @unordered_i32_shared_gpu(i32 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i32_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_i32_shared_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_shared_gpu_param_1];
+; CHECK-NEXT: st.volatile.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr addrspace(3) %addr syncscope("device") unordered, align 4
+ ret void
+}
+
+define void @unordered_i32_shared_sys_volatile(i32 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i32_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_i32_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_shared_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr addrspace(3) %addr syncscope("") unordered, align 4
+ ret void
+}
+
+define void @unordered_i32_shared_cta_volatile(i32 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i32_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_i32_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_shared_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr addrspace(3) %addr syncscope("block") unordered, align 4
+ ret void
+}
+
+define void @unordered_i32_shared_cluster_volatile(i32 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i32_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_i32_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_shared_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr addrspace(3) %addr syncscope("cluster") unordered, align 4
+ ret void
+}
+
+define void @unordered_i32_shared_gpu_volatile(i32 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i32_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_i32_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_shared_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr addrspace(3) %addr syncscope("device") unordered, align 4
+ ret void
+}
+
+define void @unordered_i64_shared_sys(i64 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i64_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_shared_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_i64_shared_sys_param_1];
+; CHECK-NEXT: st.volatile.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr addrspace(3) %addr syncscope("") unordered, align 8
+ ret void
+}
+
+define void @unordered_i64_shared_cta(i64 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i64_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_shared_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_i64_shared_cta_param_1];
+; CHECK-NEXT: st.volatile.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr addrspace(3) %addr syncscope("block") unordered, align 8
+ ret void
+}
+
+define void @unordered_i64_shared_cluster(i64 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i64_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_shared_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_i64_shared_cluster_param_1];
+; CHECK-NEXT: st.volatile.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr addrspace(3) %addr syncscope("cluster") unordered, align 8
+ ret void
+}
+
+define void @unordered_i64_shared_gpu(i64 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i64_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_shared_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_i64_shared_gpu_param_1];
+; CHECK-NEXT: st.volatile.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr addrspace(3) %addr syncscope("device") unordered, align 8
+ ret void
+}
+
+define void @unordered_i64_shared_sys_volatile(i64 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i64_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_i64_shared_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr addrspace(3) %addr syncscope("") unordered, align 8
+ ret void
+}
+
+define void @unordered_i64_shared_cta_volatile(i64 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i64_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_i64_shared_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr addrspace(3) %addr syncscope("block") unordered, align 8
+ ret void
+}
+
+define void @unordered_i64_shared_cluster_volatile(i64 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i64_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_i64_shared_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr addrspace(3) %addr syncscope("cluster") unordered, align 8
+ ret void
+}
+
+define void @unordered_i64_shared_gpu_volatile(i64 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i64_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_i64_shared_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr addrspace(3) %addr syncscope("device") unordered, align 8
+ ret void
+}
+
+define void @unordered_half_shared_sys(half %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_half_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_half_shared_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_shared_sys_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr addrspace(3) %addr syncscope("") unordered, align 2
+ ret void
+}
+
+define void @unordered_half_shared_cta(half %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_half_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_half_shared_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_shared_cta_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr addrspace(3) %addr syncscope("block") unordered, align 2
+ ret void
+}
+
+define void @unordered_half_shared_cluster(half %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_half_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_half_shared_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_shared_cluster_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr addrspace(3) %addr syncscope("cluster") unordered, align 2
+ ret void
+}
+
+define void @unordered_half_shared_gpu(half %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_half_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_half_shared_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_shared_gpu_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr addrspace(3) %addr syncscope("device") unordered, align 2
+ ret void
+}
+
+define void @unordered_half_shared_sys_volatile(half %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_half_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_half_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_shared_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr addrspace(3) %addr syncscope("") unordered, align 2
+ ret void
+}
+
+define void @unordered_half_shared_cta_volatile(half %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_half_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_half_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_shared_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr addrspace(3) %addr syncscope("block") unordered, align 2
+ ret void
+}
+
+define void @unordered_half_shared_cluster_volatile(half %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_half_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_half_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_shared_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr addrspace(3) %addr syncscope("cluster") unordered, align 2
+ ret void
+}
+
+define void @unordered_half_shared_gpu_volatile(half %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_half_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_half_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_shared_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr addrspace(3) %addr syncscope("device") unordered, align 2
+ ret void
+}
+
+define void @unordered_bfloat_shared_sys(bfloat %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_bfloat_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_bfloat_shared_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_shared_sys_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr addrspace(3) %addr syncscope("") unordered, align 2
+ ret void
+}
+
+define void @unordered_bfloat_shared_cta(bfloat %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_bfloat_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_bfloat_shared_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_shared_cta_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr addrspace(3) %addr syncscope("block") unordered, align 2
+ ret void
+}
+
+define void @unordered_bfloat_shared_cluster(bfloat %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_bfloat_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_bfloat_shared_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_shared_cluster_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr addrspace(3) %addr syncscope("cluster") unordered, align 2
+ ret void
+}
+
+define void @unordered_bfloat_shared_gpu(bfloat %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_bfloat_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_bfloat_shared_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_shared_gpu_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr addrspace(3) %addr syncscope("device") unordered, align 2
+ ret void
+}
+
+define void @unordered_bfloat_shared_sys_volatile(bfloat %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_bfloat_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_bfloat_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_shared_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr addrspace(3) %addr syncscope("") unordered, align 2
+ ret void
+}
+
+define void @unordered_bfloat_shared_cta_volatile(bfloat %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_bfloat_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_bfloat_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_shared_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr addrspace(3) %addr syncscope("block") unordered, align 2
+ ret void
+}
+
+define void @unordered_bfloat_shared_cluster_volatile(bfloat %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_bfloat_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_bfloat_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_shared_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr addrspace(3) %addr syncscope("cluster") unordered, align 2
+ ret void
+}
+
+define void @unordered_bfloat_shared_gpu_volatile(bfloat %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_bfloat_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_bfloat_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_shared_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr addrspace(3) %addr syncscope("device") unordered, align 2
+ ret void
+}
+
+define void @unordered_float_shared_sys(float %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_float_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_float_shared_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_shared_sys_param_1];
+; CHECK-NEXT: st.volatile.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr addrspace(3) %addr syncscope("") unordered, align 4
+ ret void
+}
+
+define void @unordered_float_shared_cta(float %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_float_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_float_shared_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_shared_cta_param_1];
+; CHECK-NEXT: st.volatile.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr addrspace(3) %addr syncscope("block") unordered, align 4
+ ret void
+}
+
+define void @unordered_float_shared_cluster(float %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_float_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_float_shared_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_shared_cluster_param_1];
+; CHECK-NEXT: st.volatile.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr addrspace(3) %addr syncscope("cluster") unordered, align 4
+ ret void
+}
+
+define void @unordered_float_shared_gpu(float %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_float_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_float_shared_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_shared_gpu_param_1];
+; CHECK-NEXT: st.volatile.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr addrspace(3) %addr syncscope("device") unordered, align 4
+ ret void
+}
+
+define void @unordered_float_shared_sys_volatile(float %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_float_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_float_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_shared_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr addrspace(3) %addr syncscope("") unordered, align 4
+ ret void
+}
+
+define void @unordered_float_shared_cta_volatile(float %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_float_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_float_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_shared_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr addrspace(3) %addr syncscope("block") unordered, align 4
+ ret void
+}
+
+define void @unordered_float_shared_cluster_volatile(float %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_float_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_float_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_shared_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr addrspace(3) %addr syncscope("cluster") unordered, align 4
+ ret void
+}
+
+define void @unordered_float_shared_gpu_volatile(float %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_float_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_float_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_shared_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr addrspace(3) %addr syncscope("device") unordered, align 4
+ ret void
+}
+
+define void @unordered_double_shared_sys(double %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_double_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_shared_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_double_shared_sys_param_1];
+; CHECK-NEXT: st.volatile.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr addrspace(3) %addr syncscope("") unordered, align 8
+ ret void
+}
+
+define void @unordered_double_shared_cta(double %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_double_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_shared_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_double_shared_cta_param_1];
+; CHECK-NEXT: st.volatile.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr addrspace(3) %addr syncscope("block") unordered, align 8
+ ret void
+}
+
+define void @unordered_double_shared_cluster(double %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_double_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_shared_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_double_shared_cluster_param_1];
+; CHECK-NEXT: st.volatile.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr addrspace(3) %addr syncscope("cluster") unordered, align 8
+ ret void
+}
+
+define void @unordered_double_shared_gpu(double %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_double_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_shared_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_double_shared_gpu_param_1];
+; CHECK-NEXT: st.volatile.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr addrspace(3) %addr syncscope("device") unordered, align 8
+ ret void
+}
+
+define void @unordered_double_shared_sys_volatile(double %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_double_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_double_shared_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr addrspace(3) %addr syncscope("") unordered, align 8
+ ret void
+}
+
+define void @unordered_double_shared_cta_volatile(double %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_double_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_double_shared_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr addrspace(3) %addr syncscope("block") unordered, align 8
+ ret void
+}
+
+define void @unordered_double_shared_cluster_volatile(double %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_double_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_double_shared_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr addrspace(3) %addr syncscope("cluster") unordered, align 8
+ ret void
+}
+
+define void @unordered_double_shared_gpu_volatile(double %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_double_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_double_shared_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr addrspace(3) %addr syncscope("device") unordered, align 8
+ ret void
+}
+
+define void @unordered_i8_local_sys(i8 %val) {
+; CHECK-LABEL: unordered_i8_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot256[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot256;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b8 %rs1, [unordered_i8_local_sys_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b8 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i8 %val, ptr addrspace(5) %slot syncscope("") unordered, align 1
+ ret void
+}
+
+define void @unordered_i8_local_cta(i8 %val) {
+; CHECK-LABEL: unordered_i8_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot257[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot257;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b8 %rs1, [unordered_i8_local_cta_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b8 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i8 %val, ptr addrspace(5) %slot syncscope("block") unordered, align 1
+ ret void
+}
+
+define void @unordered_i8_local_cluster(i8 %val) {
+; CHECK-LABEL: unordered_i8_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot258[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot258;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b8 %rs1, [unordered_i8_local_cluster_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b8 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i8 %val, ptr addrspace(5) %slot syncscope("cluster") unordered, align 1
+ ret void
+}
+
+define void @unordered_i8_local_gpu(i8 %val) {
+; CHECK-LABEL: unordered_i8_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot259[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot259;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b8 %rs1, [unordered_i8_local_gpu_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b8 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i8 %val, ptr addrspace(5) %slot syncscope("device") unordered, align 1
+ ret void
+}
+
+define void @unordered_i8_local_sys_volatile(i8 %val) {
+; CHECK-LABEL: unordered_i8_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot260[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot260;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b8 %rs1, [unordered_i8_local_sys_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b8 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i8 %val, ptr addrspace(5) %slot syncscope("") unordered, align 1
+ ret void
+}
+
+define void @unordered_i8_local_cta_volatile(i8 %val) {
+; CHECK-LABEL: unordered_i8_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot261[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot261;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b8 %rs1, [unordered_i8_local_cta_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b8 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i8 %val, ptr addrspace(5) %slot syncscope("block") unordered, align 1
+ ret void
+}
+
+define void @unordered_i8_local_cluster_volatile(i8 %val) {
+; CHECK-LABEL: unordered_i8_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot262[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot262;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b8 %rs1, [unordered_i8_local_cluster_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b8 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i8 %val, ptr addrspace(5) %slot syncscope("cluster") unordered, align 1
+ ret void
+}
+
+define void @unordered_i8_local_gpu_volatile(i8 %val) {
+; CHECK-LABEL: unordered_i8_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot263[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot263;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b8 %rs1, [unordered_i8_local_gpu_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b8 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i8 %val, ptr addrspace(5) %slot syncscope("device") unordered, align 1
+ ret void
+}
+
+define void @unordered_i16_local_sys(i16 %val) {
+; CHECK-LABEL: unordered_i16_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot264[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot264;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_i16_local_sys_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i16 %val, ptr addrspace(5) %slot syncscope("") unordered, align 2
+ ret void
+}
+
+define void @unordered_i16_local_cta(i16 %val) {
+; CHECK-LABEL: unordered_i16_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot265[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot265;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_i16_local_cta_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i16 %val, ptr addrspace(5) %slot syncscope("block") unordered, align 2
+ ret void
+}
+
+define void @unordered_i16_local_cluster(i16 %val) {
+; CHECK-LABEL: unordered_i16_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot266[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot266;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_i16_local_cluster_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i16 %val, ptr addrspace(5) %slot syncscope("cluster") unordered, align 2
+ ret void
+}
+
+define void @unordered_i16_local_gpu(i16 %val) {
+; CHECK-LABEL: unordered_i16_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot267[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot267;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_i16_local_gpu_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i16 %val, ptr addrspace(5) %slot syncscope("device") unordered, align 2
+ ret void
+}
+
+define void @unordered_i16_local_sys_volatile(i16 %val) {
+; CHECK-LABEL: unordered_i16_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot268[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot268;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_i16_local_sys_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i16 %val, ptr addrspace(5) %slot syncscope("") unordered, align 2
+ ret void
+}
+
+define void @unordered_i16_local_cta_volatile(i16 %val) {
+; CHECK-LABEL: unordered_i16_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot269[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot269;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_i16_local_cta_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i16 %val, ptr addrspace(5) %slot syncscope("block") unordered, align 2
+ ret void
+}
+
+define void @unordered_i16_local_cluster_volatile(i16 %val) {
+; CHECK-LABEL: unordered_i16_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot270[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot270;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_i16_local_cluster_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i16 %val, ptr addrspace(5) %slot syncscope("cluster") unordered, align 2
+ ret void
+}
+
+define void @unordered_i16_local_gpu_volatile(i16 %val) {
+; CHECK-LABEL: unordered_i16_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot271[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot271;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_i16_local_gpu_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i16 %val, ptr addrspace(5) %slot syncscope("device") unordered, align 2
+ ret void
+}
+
+define void @unordered_i32_local_sys(i32 %val) {
+; CHECK-LABEL: unordered_i32_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot272[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot272;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [unordered_i32_local_sys_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i32 %val, ptr addrspace(5) %slot syncscope("") unordered, align 4
+ ret void
+}
+
+define void @unordered_i32_local_cta(i32 %val) {
+; CHECK-LABEL: unordered_i32_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot273[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot273;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [unordered_i32_local_cta_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i32 %val, ptr addrspace(5) %slot syncscope("block") unordered, align 4
+ ret void
+}
+
+define void @unordered_i32_local_cluster(i32 %val) {
+; CHECK-LABEL: unordered_i32_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot274[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot274;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [unordered_i32_local_cluster_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i32 %val, ptr addrspace(5) %slot syncscope("cluster") unordered, align 4
+ ret void
+}
+
+define void @unordered_i32_local_gpu(i32 %val) {
+; CHECK-LABEL: unordered_i32_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot275[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot275;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [unordered_i32_local_gpu_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i32 %val, ptr addrspace(5) %slot syncscope("device") unordered, align 4
+ ret void
+}
+
+define void @unordered_i32_local_sys_volatile(i32 %val) {
+; CHECK-LABEL: unordered_i32_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot276[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot276;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [unordered_i32_local_sys_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i32 %val, ptr addrspace(5) %slot syncscope("") unordered, align 4
+ ret void
+}
+
+define void @unordered_i32_local_cta_volatile(i32 %val) {
+; CHECK-LABEL: unordered_i32_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot277[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot277;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [unordered_i32_local_cta_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i32 %val, ptr addrspace(5) %slot syncscope("block") unordered, align 4
+ ret void
+}
+
+define void @unordered_i32_local_cluster_volatile(i32 %val) {
+; CHECK-LABEL: unordered_i32_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot278[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot278;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [unordered_i32_local_cluster_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i32 %val, ptr addrspace(5) %slot syncscope("cluster") unordered, align 4
+ ret void
+}
+
+define void @unordered_i32_local_gpu_volatile(i32 %val) {
+; CHECK-LABEL: unordered_i32_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot279[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot279;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [unordered_i32_local_gpu_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i32 %val, ptr addrspace(5) %slot syncscope("device") unordered, align 4
+ ret void
+}
+
+define void @unordered_i64_local_sys(i64 %val) {
+; CHECK-LABEL: unordered_i64_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot280[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot280;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_local_sys_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i64 %val, ptr addrspace(5) %slot syncscope("") unordered, align 8
+ ret void
+}
+
+define void @unordered_i64_local_cta(i64 %val) {
+; CHECK-LABEL: unordered_i64_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot281[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot281;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_local_cta_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i64 %val, ptr addrspace(5) %slot syncscope("block") unordered, align 8
+ ret void
+}
+
+define void @unordered_i64_local_cluster(i64 %val) {
+; CHECK-LABEL: unordered_i64_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot282[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot282;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_local_cluster_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i64 %val, ptr addrspace(5) %slot syncscope("cluster") unordered, align 8
+ ret void
+}
+
+define void @unordered_i64_local_gpu(i64 %val) {
+; CHECK-LABEL: unordered_i64_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot283[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot283;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_local_gpu_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i64 %val, ptr addrspace(5) %slot syncscope("device") unordered, align 8
+ ret void
+}
+
+define void @unordered_i64_local_sys_volatile(i64 %val) {
+; CHECK-LABEL: unordered_i64_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot284[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot284;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_local_sys_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i64 %val, ptr addrspace(5) %slot syncscope("") unordered, align 8
+ ret void
+}
+
+define void @unordered_i64_local_cta_volatile(i64 %val) {
+; CHECK-LABEL: unordered_i64_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot285[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot285;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_local_cta_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i64 %val, ptr addrspace(5) %slot syncscope("block") unordered, align 8
+ ret void
+}
+
+define void @unordered_i64_local_cluster_volatile(i64 %val) {
+; CHECK-LABEL: unordered_i64_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot286[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot286;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_local_cluster_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i64 %val, ptr addrspace(5) %slot syncscope("cluster") unordered, align 8
+ ret void
+}
+
+define void @unordered_i64_local_gpu_volatile(i64 %val) {
+; CHECK-LABEL: unordered_i64_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot287[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot287;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_local_gpu_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i64 %val, ptr addrspace(5) %slot syncscope("device") unordered, align 8
+ ret void
+}
+
+define void @unordered_half_local_sys(half %val) {
+; CHECK-LABEL: unordered_half_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot288[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot288;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_half_local_sys_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic half %val, ptr addrspace(5) %slot syncscope("") unordered, align 2
+ ret void
+}
+
+define void @unordered_half_local_cta(half %val) {
+; CHECK-LABEL: unordered_half_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot289[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot289;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_half_local_cta_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic half %val, ptr addrspace(5) %slot syncscope("block") unordered, align 2
+ ret void
+}
+
+define void @unordered_half_local_cluster(half %val) {
+; CHECK-LABEL: unordered_half_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot290[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot290;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_half_local_cluster_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic half %val, ptr addrspace(5) %slot syncscope("cluster") unordered, align 2
+ ret void
+}
+
+define void @unordered_half_local_gpu(half %val) {
+; CHECK-LABEL: unordered_half_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot291[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot291;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_half_local_gpu_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic half %val, ptr addrspace(5) %slot syncscope("device") unordered, align 2
+ ret void
+}
+
+define void @unordered_half_local_sys_volatile(half %val) {
+; CHECK-LABEL: unordered_half_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot292[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot292;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_half_local_sys_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile half %val, ptr addrspace(5) %slot syncscope("") unordered, align 2
+ ret void
+}
+
+define void @unordered_half_local_cta_volatile(half %val) {
+; CHECK-LABEL: unordered_half_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot293[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot293;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_half_local_cta_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile half %val, ptr addrspace(5) %slot syncscope("block") unordered, align 2
+ ret void
+}
+
+define void @unordered_half_local_cluster_volatile(half %val) {
+; CHECK-LABEL: unordered_half_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot294[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot294;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_half_local_cluster_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile half %val, ptr addrspace(5) %slot syncscope("cluster") unordered, align 2
+ ret void
+}
+
+define void @unordered_half_local_gpu_volatile(half %val) {
+; CHECK-LABEL: unordered_half_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot295[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot295;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_half_local_gpu_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile half %val, ptr addrspace(5) %slot syncscope("device") unordered, align 2
+ ret void
+}
+
+define void @unordered_bfloat_local_sys(bfloat %val) {
+; CHECK-LABEL: unordered_bfloat_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot296[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot296;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_bfloat_local_sys_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic bfloat %val, ptr addrspace(5) %slot syncscope("") unordered, align 2
+ ret void
+}
+
+define void @unordered_bfloat_local_cta(bfloat %val) {
+; CHECK-LABEL: unordered_bfloat_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot297[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot297;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_bfloat_local_cta_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic bfloat %val, ptr addrspace(5) %slot syncscope("block") unordered, align 2
+ ret void
+}
+
+define void @unordered_bfloat_local_cluster(bfloat %val) {
+; CHECK-LABEL: unordered_bfloat_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot298[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot298;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_bfloat_local_cluster_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic bfloat %val, ptr addrspace(5) %slot syncscope("cluster") unordered, align 2
+ ret void
+}
+
+define void @unordered_bfloat_local_gpu(bfloat %val) {
+; CHECK-LABEL: unordered_bfloat_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot299[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot299;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_bfloat_local_gpu_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic bfloat %val, ptr addrspace(5) %slot syncscope("device") unordered, align 2
+ ret void
+}
+
+define void @unordered_bfloat_local_sys_volatile(bfloat %val) {
+; CHECK-LABEL: unordered_bfloat_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot300[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot300;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_bfloat_local_sys_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile bfloat %val, ptr addrspace(5) %slot syncscope("") unordered, align 2
+ ret void
+}
+
+define void @unordered_bfloat_local_cta_volatile(bfloat %val) {
+; CHECK-LABEL: unordered_bfloat_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot301[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot301;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_bfloat_local_cta_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile bfloat %val, ptr addrspace(5) %slot syncscope("block") unordered, align 2
+ ret void
+}
+
+define void @unordered_bfloat_local_cluster_volatile(bfloat %val) {
+; CHECK-LABEL: unordered_bfloat_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot302[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot302;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_bfloat_local_cluster_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile bfloat %val, ptr addrspace(5) %slot syncscope("cluster") unordered, align 2
+ ret void
+}
+
+define void @unordered_bfloat_local_gpu_volatile(bfloat %val) {
+; CHECK-LABEL: unordered_bfloat_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot303[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot303;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_bfloat_local_gpu_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile bfloat %val, ptr addrspace(5) %slot syncscope("device") unordered, align 2
+ ret void
+}
+
+define void @unordered_float_local_sys(float %val) {
+; CHECK-LABEL: unordered_float_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot304[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot304;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [unordered_float_local_sys_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic float %val, ptr addrspace(5) %slot syncscope("") unordered, align 4
+ ret void
+}
+
+define void @unordered_float_local_cta(float %val) {
+; CHECK-LABEL: unordered_float_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot305[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot305;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [unordered_float_local_cta_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic float %val, ptr addrspace(5) %slot syncscope("block") unordered, align 4
+ ret void
+}
+
+define void @unordered_float_local_cluster(float %val) {
+; CHECK-LABEL: unordered_float_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot306[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot306;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [unordered_float_local_cluster_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic float %val, ptr addrspace(5) %slot syncscope("cluster") unordered, align 4
+ ret void
+}
+
+define void @unordered_float_local_gpu(float %val) {
+; CHECK-LABEL: unordered_float_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot307[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot307;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [unordered_float_local_gpu_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic float %val, ptr addrspace(5) %slot syncscope("device") unordered, align 4
+ ret void
+}
+
+define void @unordered_float_local_sys_volatile(float %val) {
+; CHECK-LABEL: unordered_float_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot308[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot308;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [unordered_float_local_sys_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile float %val, ptr addrspace(5) %slot syncscope("") unordered, align 4
+ ret void
+}
+
+define void @unordered_float_local_cta_volatile(float %val) {
+; CHECK-LABEL: unordered_float_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot309[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot309;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [unordered_float_local_cta_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile float %val, ptr addrspace(5) %slot syncscope("block") unordered, align 4
+ ret void
+}
+
+define void @unordered_float_local_cluster_volatile(float %val) {
+; CHECK-LABEL: unordered_float_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot310[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot310;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [unordered_float_local_cluster_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile float %val, ptr addrspace(5) %slot syncscope("cluster") unordered, align 4
+ ret void
+}
+
+define void @unordered_float_local_gpu_volatile(float %val) {
+; CHECK-LABEL: unordered_float_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot311[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot311;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [unordered_float_local_gpu_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile float %val, ptr addrspace(5) %slot syncscope("device") unordered, align 4
+ ret void
+}
+
+define void @unordered_double_local_sys(double %val) {
+; CHECK-LABEL: unordered_double_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot312[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot312;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_local_sys_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic double %val, ptr addrspace(5) %slot syncscope("") unordered, align 8
+ ret void
+}
+
+define void @unordered_double_local_cta(double %val) {
+; CHECK-LABEL: unordered_double_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot313[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot313;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_local_cta_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic double %val, ptr addrspace(5) %slot syncscope("block") unordered, align 8
+ ret void
+}
+
+define void @unordered_double_local_cluster(double %val) {
+; CHECK-LABEL: unordered_double_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot314[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot314;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_local_cluster_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic double %val, ptr addrspace(5) %slot syncscope("cluster") unordered, align 8
+ ret void
+}
+
+define void @unordered_double_local_gpu(double %val) {
+; CHECK-LABEL: unordered_double_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot315[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot315;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_local_gpu_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic double %val, ptr addrspace(5) %slot syncscope("device") unordered, align 8
+ ret void
+}
+
+define void @unordered_double_local_sys_volatile(double %val) {
+; CHECK-LABEL: unordered_double_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot316[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot316;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_local_sys_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile double %val, ptr addrspace(5) %slot syncscope("") unordered, align 8
+ ret void
+}
+
+define void @unordered_double_local_cta_volatile(double %val) {
+; CHECK-LABEL: unordered_double_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot317[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot317;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_local_cta_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile double %val, ptr addrspace(5) %slot syncscope("block") unordered, align 8
+ ret void
+}
+
+define void @unordered_double_local_cluster_volatile(double %val) {
+; CHECK-LABEL: unordered_double_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot318[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot318;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_local_cluster_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile double %val, ptr addrspace(5) %slot syncscope("cluster") unordered, align 8
+ ret void
+}
+
+define void @unordered_double_local_gpu_volatile(double %val) {
+; CHECK-LABEL: unordered_double_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot319[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot319;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_local_gpu_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile double %val, ptr addrspace(5) %slot syncscope("device") unordered, align 8
+ ret void
+}
+
+define void @monotonic_i8_generic_sys(i8 %val, ptr %addr) {
+; CHECK-LABEL: monotonic_i8_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [monotonic_i8_generic_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_generic_sys_param_1];
+; CHECK-NEXT: st.volatile.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr %addr syncscope("") monotonic, align 1
+ ret void
+}
+
+define void @monotonic_i8_generic_cta(i8 %val, ptr %addr) {
+; CHECK-LABEL: monotonic_i8_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [monotonic_i8_generic_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_generic_cta_param_1];
+; CHECK-NEXT: st.volatile.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr %addr syncscope("block") monotonic, align 1
+ ret void
+}
+
+define void @monotonic_i8_generic_cluster(i8 %val, ptr %addr) {
+; CHECK-LABEL: monotonic_i8_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [monotonic_i8_generic_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_generic_cluster_param_1];
+; CHECK-NEXT: st.volatile.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr %addr syncscope("cluster") monotonic, align 1
+ ret void
+}
+
+define void @monotonic_i8_generic_gpu(i8 %val, ptr %addr) {
+; CHECK-LABEL: monotonic_i8_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [monotonic_i8_generic_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_generic_gpu_param_1];
+; CHECK-NEXT: st.volatile.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr %addr syncscope("device") monotonic, align 1
+ ret void
+}
+
+define void @monotonic_i8_generic_sys_volatile(i8 %val, ptr %addr) {
+; CHECK-LABEL: monotonic_i8_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [monotonic_i8_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_generic_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr %addr syncscope("") monotonic, align 1
+ ret void
+}
+
+define void @monotonic_i8_generic_cta_volatile(i8 %val, ptr %addr) {
+; CHECK-LABEL: monotonic_i8_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [monotonic_i8_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_generic_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr %addr syncscope("block") monotonic, align 1
+ ret void
+}
+
+define void @monotonic_i8_generic_cluster_volatile(i8 %val, ptr %addr) {
+; CHECK-LABEL: monotonic_i8_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [monotonic_i8_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_generic_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr %addr syncscope("cluster") monotonic, align 1
+ ret void
+}
+
+define void @monotonic_i8_generic_gpu_volatile(i8 %val, ptr %addr) {
+; CHECK-LABEL: monotonic_i8_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [monotonic_i8_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_generic_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr %addr syncscope("device") monotonic, align 1
+ ret void
+}
+
+define void @monotonic_i16_generic_sys(i16 %val, ptr %addr) {
+; CHECK-LABEL: monotonic_i16_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_i16_generic_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_generic_sys_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr %addr syncscope("") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_i16_generic_cta(i16 %val, ptr %addr) {
+; CHECK-LABEL: monotonic_i16_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_i16_generic_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_generic_cta_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr %addr syncscope("block") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_i16_generic_cluster(i16 %val, ptr %addr) {
+; CHECK-LABEL: monotonic_i16_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_i16_generic_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_generic_cluster_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr %addr syncscope("cluster") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_i16_generic_gpu(i16 %val, ptr %addr) {
+; CHECK-LABEL: monotonic_i16_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_i16_generic_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_generic_gpu_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr %addr syncscope("device") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_i16_generic_sys_volatile(i16 %val, ptr %addr) {
+; CHECK-LABEL: monotonic_i16_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_i16_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_generic_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr %addr syncscope("") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_i16_generic_cta_volatile(i16 %val, ptr %addr) {
+; CHECK-LABEL: monotonic_i16_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_i16_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_generic_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr %addr syncscope("block") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_i16_generic_cluster_volatile(i16 %val, ptr %addr) {
+; CHECK-LABEL: monotonic_i16_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_i16_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_generic_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr %addr syncscope("cluster") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_i16_generic_gpu_volatile(i16 %val, ptr %addr) {
+; CHECK-LABEL: monotonic_i16_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_i16_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_generic_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr %addr syncscope("device") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_i32_generic_sys(i32 %val, ptr %addr) {
+; CHECK-LABEL: monotonic_i32_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_i32_generic_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_generic_sys_param_1];
+; CHECK-NEXT: st.volatile.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr %addr syncscope("") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_i32_generic_cta(i32 %val, ptr %addr) {
+; CHECK-LABEL: monotonic_i32_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_i32_generic_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_generic_cta_param_1];
+; CHECK-NEXT: st.volatile.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr %addr syncscope("block") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_i32_generic_cluster(i32 %val, ptr %addr) {
+; CHECK-LABEL: monotonic_i32_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_i32_generic_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_generic_cluster_param_1];
+; CHECK-NEXT: st.volatile.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr %addr syncscope("cluster") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_i32_generic_gpu(i32 %val, ptr %addr) {
+; CHECK-LABEL: monotonic_i32_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_i32_generic_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_generic_gpu_param_1];
+; CHECK-NEXT: st.volatile.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr %addr syncscope("device") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_i32_generic_sys_volatile(i32 %val, ptr %addr) {
+; CHECK-LABEL: monotonic_i32_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_i32_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_generic_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr %addr syncscope("") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_i32_generic_cta_volatile(i32 %val, ptr %addr) {
+; CHECK-LABEL: monotonic_i32_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_i32_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_generic_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr %addr syncscope("block") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_i32_generic_cluster_volatile(i32 %val, ptr %addr) {
+; CHECK-LABEL: monotonic_i32_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_i32_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_generic_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr %addr syncscope("cluster") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_i32_generic_gpu_volatile(i32 %val, ptr %addr) {
+; CHECK-LABEL: monotonic_i32_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_i32_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_generic_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr %addr syncscope("device") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_i64_generic_sys(i64 %val, ptr %addr) {
+; CHECK-LABEL: monotonic_i64_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_generic_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_i64_generic_sys_param_1];
+; CHECK-NEXT: st.volatile.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr %addr syncscope("") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_i64_generic_cta(i64 %val, ptr %addr) {
+; CHECK-LABEL: monotonic_i64_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_generic_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_i64_generic_cta_param_1];
+; CHECK-NEXT: st.volatile.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr %addr syncscope("block") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_i64_generic_cluster(i64 %val, ptr %addr) {
+; CHECK-LABEL: monotonic_i64_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_generic_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_i64_generic_cluster_param_1];
+; CHECK-NEXT: st.volatile.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr %addr syncscope("cluster") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_i64_generic_gpu(i64 %val, ptr %addr) {
+; CHECK-LABEL: monotonic_i64_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_generic_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_i64_generic_gpu_param_1];
+; CHECK-NEXT: st.volatile.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr %addr syncscope("device") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_i64_generic_sys_volatile(i64 %val, ptr %addr) {
+; CHECK-LABEL: monotonic_i64_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_i64_generic_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr %addr syncscope("") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_i64_generic_cta_volatile(i64 %val, ptr %addr) {
+; CHECK-LABEL: monotonic_i64_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_i64_generic_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr %addr syncscope("block") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_i64_generic_cluster_volatile(i64 %val, ptr %addr) {
+; CHECK-LABEL: monotonic_i64_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_i64_generic_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr %addr syncscope("cluster") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_i64_generic_gpu_volatile(i64 %val, ptr %addr) {
+; CHECK-LABEL: monotonic_i64_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_i64_generic_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr %addr syncscope("device") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_half_generic_sys(half %val, ptr %addr) {
+; CHECK-LABEL: monotonic_half_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_half_generic_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_generic_sys_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr %addr syncscope("") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_half_generic_cta(half %val, ptr %addr) {
+; CHECK-LABEL: monotonic_half_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_half_generic_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_generic_cta_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr %addr syncscope("block") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_half_generic_cluster(half %val, ptr %addr) {
+; CHECK-LABEL: monotonic_half_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_half_generic_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_generic_cluster_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr %addr syncscope("cluster") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_half_generic_gpu(half %val, ptr %addr) {
+; CHECK-LABEL: monotonic_half_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_half_generic_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_generic_gpu_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr %addr syncscope("device") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_half_generic_sys_volatile(half %val, ptr %addr) {
+; CHECK-LABEL: monotonic_half_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_half_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_generic_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr %addr syncscope("") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_half_generic_cta_volatile(half %val, ptr %addr) {
+; CHECK-LABEL: monotonic_half_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_half_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_generic_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr %addr syncscope("block") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_half_generic_cluster_volatile(half %val, ptr %addr) {
+; CHECK-LABEL: monotonic_half_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_half_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_generic_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr %addr syncscope("cluster") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_half_generic_gpu_volatile(half %val, ptr %addr) {
+; CHECK-LABEL: monotonic_half_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_half_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_generic_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr %addr syncscope("device") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_bfloat_generic_sys(bfloat %val, ptr %addr) {
+; CHECK-LABEL: monotonic_bfloat_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_bfloat_generic_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_generic_sys_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr %addr syncscope("") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_bfloat_generic_cta(bfloat %val, ptr %addr) {
+; CHECK-LABEL: monotonic_bfloat_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_bfloat_generic_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_generic_cta_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr %addr syncscope("block") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_bfloat_generic_cluster(bfloat %val, ptr %addr) {
+; CHECK-LABEL: monotonic_bfloat_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_bfloat_generic_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_generic_cluster_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr %addr syncscope("cluster") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_bfloat_generic_gpu(bfloat %val, ptr %addr) {
+; CHECK-LABEL: monotonic_bfloat_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_bfloat_generic_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_generic_gpu_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr %addr syncscope("device") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_bfloat_generic_sys_volatile(bfloat %val, ptr %addr) {
+; CHECK-LABEL: monotonic_bfloat_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_bfloat_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_generic_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr %addr syncscope("") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_bfloat_generic_cta_volatile(bfloat %val, ptr %addr) {
+; CHECK-LABEL: monotonic_bfloat_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_bfloat_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_generic_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr %addr syncscope("block") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_bfloat_generic_cluster_volatile(bfloat %val, ptr %addr) {
+; CHECK-LABEL: monotonic_bfloat_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_bfloat_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_generic_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr %addr syncscope("cluster") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_bfloat_generic_gpu_volatile(bfloat %val, ptr %addr) {
+; CHECK-LABEL: monotonic_bfloat_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_bfloat_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_generic_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr %addr syncscope("device") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_float_generic_sys(float %val, ptr %addr) {
+; CHECK-LABEL: monotonic_float_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_float_generic_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_generic_sys_param_1];
+; CHECK-NEXT: st.volatile.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr %addr syncscope("") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_float_generic_cta(float %val, ptr %addr) {
+; CHECK-LABEL: monotonic_float_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_float_generic_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_generic_cta_param_1];
+; CHECK-NEXT: st.volatile.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr %addr syncscope("block") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_float_generic_cluster(float %val, ptr %addr) {
+; CHECK-LABEL: monotonic_float_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_float_generic_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_generic_cluster_param_1];
+; CHECK-NEXT: st.volatile.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr %addr syncscope("cluster") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_float_generic_gpu(float %val, ptr %addr) {
+; CHECK-LABEL: monotonic_float_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_float_generic_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_generic_gpu_param_1];
+; CHECK-NEXT: st.volatile.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr %addr syncscope("device") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_float_generic_sys_volatile(float %val, ptr %addr) {
+; CHECK-LABEL: monotonic_float_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_float_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_generic_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr %addr syncscope("") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_float_generic_cta_volatile(float %val, ptr %addr) {
+; CHECK-LABEL: monotonic_float_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_float_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_generic_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr %addr syncscope("block") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_float_generic_cluster_volatile(float %val, ptr %addr) {
+; CHECK-LABEL: monotonic_float_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_float_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_generic_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr %addr syncscope("cluster") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_float_generic_gpu_volatile(float %val, ptr %addr) {
+; CHECK-LABEL: monotonic_float_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_float_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_generic_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr %addr syncscope("device") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_double_generic_sys(double %val, ptr %addr) {
+; CHECK-LABEL: monotonic_double_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_generic_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_double_generic_sys_param_1];
+; CHECK-NEXT: st.volatile.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr %addr syncscope("") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_double_generic_cta(double %val, ptr %addr) {
+; CHECK-LABEL: monotonic_double_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_generic_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_double_generic_cta_param_1];
+; CHECK-NEXT: st.volatile.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr %addr syncscope("block") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_double_generic_cluster(double %val, ptr %addr) {
+; CHECK-LABEL: monotonic_double_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_generic_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_double_generic_cluster_param_1];
+; CHECK-NEXT: st.volatile.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr %addr syncscope("cluster") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_double_generic_gpu(double %val, ptr %addr) {
+; CHECK-LABEL: monotonic_double_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_generic_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_double_generic_gpu_param_1];
+; CHECK-NEXT: st.volatile.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr %addr syncscope("device") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_double_generic_sys_volatile(double %val, ptr %addr) {
+; CHECK-LABEL: monotonic_double_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_double_generic_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr %addr syncscope("") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_double_generic_cta_volatile(double %val, ptr %addr) {
+; CHECK-LABEL: monotonic_double_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_double_generic_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr %addr syncscope("block") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_double_generic_cluster_volatile(double %val, ptr %addr) {
+; CHECK-LABEL: monotonic_double_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_double_generic_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr %addr syncscope("cluster") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_double_generic_gpu_volatile(double %val, ptr %addr) {
+; CHECK-LABEL: monotonic_double_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_double_generic_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr %addr syncscope("device") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_i8_global_sys(i8 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i8_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [monotonic_i8_global_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_global_sys_param_1];
+; CHECK-NEXT: st.volatile.global.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr addrspace(1) %addr syncscope("") monotonic, align 1
+ ret void
+}
+
+define void @monotonic_i8_global_cta(i8 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i8_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [monotonic_i8_global_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_global_cta_param_1];
+; CHECK-NEXT: st.volatile.global.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr addrspace(1) %addr syncscope("block") monotonic, align 1
+ ret void
+}
+
+define void @monotonic_i8_global_cluster(i8 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i8_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [monotonic_i8_global_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_global_cluster_param_1];
+; CHECK-NEXT: st.volatile.global.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr addrspace(1) %addr syncscope("cluster") monotonic, align 1
+ ret void
+}
+
+define void @monotonic_i8_global_gpu(i8 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i8_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [monotonic_i8_global_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_global_gpu_param_1];
+; CHECK-NEXT: st.volatile.global.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr addrspace(1) %addr syncscope("device") monotonic, align 1
+ ret void
+}
+
+define void @monotonic_i8_global_sys_volatile(i8 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i8_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [monotonic_i8_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_global_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr addrspace(1) %addr syncscope("") monotonic, align 1
+ ret void
+}
+
+define void @monotonic_i8_global_cta_volatile(i8 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i8_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [monotonic_i8_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_global_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr addrspace(1) %addr syncscope("block") monotonic, align 1
+ ret void
+}
+
+define void @monotonic_i8_global_cluster_volatile(i8 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i8_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [monotonic_i8_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_global_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr addrspace(1) %addr syncscope("cluster") monotonic, align 1
+ ret void
+}
+
+define void @monotonic_i8_global_gpu_volatile(i8 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i8_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [monotonic_i8_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_global_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr addrspace(1) %addr syncscope("device") monotonic, align 1
+ ret void
+}
+
+define void @monotonic_i16_global_sys(i16 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i16_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_i16_global_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_global_sys_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr addrspace(1) %addr syncscope("") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_i16_global_cta(i16 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i16_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_i16_global_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_global_cta_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr addrspace(1) %addr syncscope("block") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_i16_global_cluster(i16 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i16_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_i16_global_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_global_cluster_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr addrspace(1) %addr syncscope("cluster") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_i16_global_gpu(i16 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i16_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_i16_global_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_global_gpu_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr addrspace(1) %addr syncscope("device") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_i16_global_sys_volatile(i16 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i16_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_i16_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_global_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr addrspace(1) %addr syncscope("") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_i16_global_cta_volatile(i16 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i16_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_i16_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_global_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr addrspace(1) %addr syncscope("block") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_i16_global_cluster_volatile(i16 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i16_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_i16_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_global_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr addrspace(1) %addr syncscope("cluster") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_i16_global_gpu_volatile(i16 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i16_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_i16_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_global_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr addrspace(1) %addr syncscope("device") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_i32_global_sys(i32 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i32_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_i32_global_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_global_sys_param_1];
+; CHECK-NEXT: st.volatile.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr addrspace(1) %addr syncscope("") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_i32_global_cta(i32 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i32_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_i32_global_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_global_cta_param_1];
+; CHECK-NEXT: st.volatile.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr addrspace(1) %addr syncscope("block") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_i32_global_cluster(i32 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i32_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_i32_global_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_global_cluster_param_1];
+; CHECK-NEXT: st.volatile.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr addrspace(1) %addr syncscope("cluster") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_i32_global_gpu(i32 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i32_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_i32_global_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_global_gpu_param_1];
+; CHECK-NEXT: st.volatile.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr addrspace(1) %addr syncscope("device") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_i32_global_sys_volatile(i32 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i32_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_i32_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_global_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr addrspace(1) %addr syncscope("") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_i32_global_cta_volatile(i32 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i32_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_i32_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_global_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr addrspace(1) %addr syncscope("block") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_i32_global_cluster_volatile(i32 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i32_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_i32_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_global_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr addrspace(1) %addr syncscope("cluster") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_i32_global_gpu_volatile(i32 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i32_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_i32_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_global_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr addrspace(1) %addr syncscope("device") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_i64_global_sys(i64 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i64_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_global_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_i64_global_sys_param_1];
+; CHECK-NEXT: st.volatile.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr addrspace(1) %addr syncscope("") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_i64_global_cta(i64 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i64_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_global_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_i64_global_cta_param_1];
+; CHECK-NEXT: st.volatile.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr addrspace(1) %addr syncscope("block") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_i64_global_cluster(i64 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i64_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_global_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_i64_global_cluster_param_1];
+; CHECK-NEXT: st.volatile.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr addrspace(1) %addr syncscope("cluster") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_i64_global_gpu(i64 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i64_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_global_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_i64_global_gpu_param_1];
+; CHECK-NEXT: st.volatile.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr addrspace(1) %addr syncscope("device") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_i64_global_sys_volatile(i64 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i64_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_i64_global_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr addrspace(1) %addr syncscope("") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_i64_global_cta_volatile(i64 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i64_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_i64_global_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr addrspace(1) %addr syncscope("block") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_i64_global_cluster_volatile(i64 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i64_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_i64_global_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr addrspace(1) %addr syncscope("cluster") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_i64_global_gpu_volatile(i64 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i64_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_i64_global_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr addrspace(1) %addr syncscope("device") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_half_global_sys(half %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_half_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_half_global_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_global_sys_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr addrspace(1) %addr syncscope("") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_half_global_cta(half %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_half_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_half_global_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_global_cta_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr addrspace(1) %addr syncscope("block") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_half_global_cluster(half %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_half_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_half_global_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_global_cluster_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr addrspace(1) %addr syncscope("cluster") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_half_global_gpu(half %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_half_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_half_global_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_global_gpu_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr addrspace(1) %addr syncscope("device") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_half_global_sys_volatile(half %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_half_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_half_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_global_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr addrspace(1) %addr syncscope("") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_half_global_cta_volatile(half %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_half_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_half_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_global_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr addrspace(1) %addr syncscope("block") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_half_global_cluster_volatile(half %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_half_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_half_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_global_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr addrspace(1) %addr syncscope("cluster") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_half_global_gpu_volatile(half %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_half_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_half_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_global_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr addrspace(1) %addr syncscope("device") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_bfloat_global_sys(bfloat %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_bfloat_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_bfloat_global_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_global_sys_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr addrspace(1) %addr syncscope("") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_bfloat_global_cta(bfloat %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_bfloat_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_bfloat_global_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_global_cta_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr addrspace(1) %addr syncscope("block") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_bfloat_global_cluster(bfloat %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_bfloat_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_bfloat_global_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_global_cluster_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr addrspace(1) %addr syncscope("cluster") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_bfloat_global_gpu(bfloat %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_bfloat_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_bfloat_global_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_global_gpu_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr addrspace(1) %addr syncscope("device") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_bfloat_global_sys_volatile(bfloat %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_bfloat_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_bfloat_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_global_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr addrspace(1) %addr syncscope("") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_bfloat_global_cta_volatile(bfloat %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_bfloat_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_bfloat_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_global_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr addrspace(1) %addr syncscope("block") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_bfloat_global_cluster_volatile(bfloat %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_bfloat_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_bfloat_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_global_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr addrspace(1) %addr syncscope("cluster") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_bfloat_global_gpu_volatile(bfloat %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_bfloat_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_bfloat_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_global_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr addrspace(1) %addr syncscope("device") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_float_global_sys(float %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_float_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_float_global_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_global_sys_param_1];
+; CHECK-NEXT: st.volatile.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr addrspace(1) %addr syncscope("") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_float_global_cta(float %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_float_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_float_global_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_global_cta_param_1];
+; CHECK-NEXT: st.volatile.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr addrspace(1) %addr syncscope("block") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_float_global_cluster(float %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_float_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_float_global_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_global_cluster_param_1];
+; CHECK-NEXT: st.volatile.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr addrspace(1) %addr syncscope("cluster") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_float_global_gpu(float %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_float_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_float_global_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_global_gpu_param_1];
+; CHECK-NEXT: st.volatile.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr addrspace(1) %addr syncscope("device") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_float_global_sys_volatile(float %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_float_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_float_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_global_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr addrspace(1) %addr syncscope("") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_float_global_cta_volatile(float %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_float_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_float_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_global_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr addrspace(1) %addr syncscope("block") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_float_global_cluster_volatile(float %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_float_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_float_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_global_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr addrspace(1) %addr syncscope("cluster") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_float_global_gpu_volatile(float %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_float_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_float_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_global_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr addrspace(1) %addr syncscope("device") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_double_global_sys(double %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_double_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_global_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_double_global_sys_param_1];
+; CHECK-NEXT: st.volatile.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr addrspace(1) %addr syncscope("") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_double_global_cta(double %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_double_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_global_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_double_global_cta_param_1];
+; CHECK-NEXT: st.volatile.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr addrspace(1) %addr syncscope("block") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_double_global_cluster(double %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_double_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_global_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_double_global_cluster_param_1];
+; CHECK-NEXT: st.volatile.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr addrspace(1) %addr syncscope("cluster") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_double_global_gpu(double %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_double_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_global_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_double_global_gpu_param_1];
+; CHECK-NEXT: st.volatile.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr addrspace(1) %addr syncscope("device") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_double_global_sys_volatile(double %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_double_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_double_global_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr addrspace(1) %addr syncscope("") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_double_global_cta_volatile(double %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_double_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_double_global_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr addrspace(1) %addr syncscope("block") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_double_global_cluster_volatile(double %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_double_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_double_global_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr addrspace(1) %addr syncscope("cluster") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_double_global_gpu_volatile(double %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_double_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_double_global_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr addrspace(1) %addr syncscope("device") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_i8_shared_sys(i8 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i8_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [monotonic_i8_shared_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_shared_sys_param_1];
+; CHECK-NEXT: st.volatile.shared.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr addrspace(3) %addr syncscope("") monotonic, align 1
+ ret void
+}
+
+define void @monotonic_i8_shared_cta(i8 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i8_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [monotonic_i8_shared_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_shared_cta_param_1];
+; CHECK-NEXT: st.volatile.shared.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr addrspace(3) %addr syncscope("block") monotonic, align 1
+ ret void
+}
+
+define void @monotonic_i8_shared_cluster(i8 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i8_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [monotonic_i8_shared_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_shared_cluster_param_1];
+; CHECK-NEXT: st.volatile.shared.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr addrspace(3) %addr syncscope("cluster") monotonic, align 1
+ ret void
+}
+
+define void @monotonic_i8_shared_gpu(i8 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i8_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [monotonic_i8_shared_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_shared_gpu_param_1];
+; CHECK-NEXT: st.volatile.shared.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr addrspace(3) %addr syncscope("device") monotonic, align 1
+ ret void
+}
+
+define void @monotonic_i8_shared_sys_volatile(i8 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i8_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [monotonic_i8_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_shared_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr addrspace(3) %addr syncscope("") monotonic, align 1
+ ret void
+}
+
+define void @monotonic_i8_shared_cta_volatile(i8 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i8_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [monotonic_i8_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_shared_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr addrspace(3) %addr syncscope("block") monotonic, align 1
+ ret void
+}
+
+define void @monotonic_i8_shared_cluster_volatile(i8 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i8_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [monotonic_i8_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_shared_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr addrspace(3) %addr syncscope("cluster") monotonic, align 1
+ ret void
+}
+
+define void @monotonic_i8_shared_gpu_volatile(i8 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i8_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [monotonic_i8_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_shared_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr addrspace(3) %addr syncscope("device") monotonic, align 1
+ ret void
+}
+
+define void @monotonic_i16_shared_sys(i16 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i16_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_i16_shared_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_shared_sys_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr addrspace(3) %addr syncscope("") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_i16_shared_cta(i16 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i16_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_i16_shared_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_shared_cta_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr addrspace(3) %addr syncscope("block") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_i16_shared_cluster(i16 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i16_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_i16_shared_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_shared_cluster_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr addrspace(3) %addr syncscope("cluster") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_i16_shared_gpu(i16 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i16_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_i16_shared_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_shared_gpu_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr addrspace(3) %addr syncscope("device") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_i16_shared_sys_volatile(i16 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i16_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_i16_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_shared_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr addrspace(3) %addr syncscope("") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_i16_shared_cta_volatile(i16 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i16_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_i16_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_shared_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr addrspace(3) %addr syncscope("block") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_i16_shared_cluster_volatile(i16 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i16_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_i16_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_shared_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr addrspace(3) %addr syncscope("cluster") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_i16_shared_gpu_volatile(i16 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i16_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_i16_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_shared_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr addrspace(3) %addr syncscope("device") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_i32_shared_sys(i32 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i32_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_i32_shared_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_shared_sys_param_1];
+; CHECK-NEXT: st.volatile.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr addrspace(3) %addr syncscope("") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_i32_shared_cta(i32 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i32_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_i32_shared_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_shared_cta_param_1];
+; CHECK-NEXT: st.volatile.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr addrspace(3) %addr syncscope("block") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_i32_shared_cluster(i32 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i32_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_i32_shared_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_shared_cluster_param_1];
+; CHECK-NEXT: st.volatile.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr addrspace(3) %addr syncscope("cluster") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_i32_shared_gpu(i32 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i32_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_i32_shared_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_shared_gpu_param_1];
+; CHECK-NEXT: st.volatile.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr addrspace(3) %addr syncscope("device") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_i32_shared_sys_volatile(i32 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i32_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_i32_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_shared_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr addrspace(3) %addr syncscope("") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_i32_shared_cta_volatile(i32 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i32_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_i32_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_shared_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr addrspace(3) %addr syncscope("block") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_i32_shared_cluster_volatile(i32 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i32_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_i32_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_shared_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr addrspace(3) %addr syncscope("cluster") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_i32_shared_gpu_volatile(i32 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i32_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_i32_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_shared_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr addrspace(3) %addr syncscope("device") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_i64_shared_sys(i64 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i64_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_shared_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_i64_shared_sys_param_1];
+; CHECK-NEXT: st.volatile.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr addrspace(3) %addr syncscope("") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_i64_shared_cta(i64 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i64_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_shared_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_i64_shared_cta_param_1];
+; CHECK-NEXT: st.volatile.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr addrspace(3) %addr syncscope("block") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_i64_shared_cluster(i64 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i64_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_shared_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_i64_shared_cluster_param_1];
+; CHECK-NEXT: st.volatile.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr addrspace(3) %addr syncscope("cluster") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_i64_shared_gpu(i64 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i64_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_shared_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_i64_shared_gpu_param_1];
+; CHECK-NEXT: st.volatile.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr addrspace(3) %addr syncscope("device") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_i64_shared_sys_volatile(i64 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i64_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_i64_shared_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr addrspace(3) %addr syncscope("") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_i64_shared_cta_volatile(i64 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i64_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_i64_shared_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr addrspace(3) %addr syncscope("block") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_i64_shared_cluster_volatile(i64 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i64_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_i64_shared_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr addrspace(3) %addr syncscope("cluster") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_i64_shared_gpu_volatile(i64 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i64_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_i64_shared_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr addrspace(3) %addr syncscope("device") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_half_shared_sys(half %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_half_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_half_shared_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_shared_sys_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr addrspace(3) %addr syncscope("") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_half_shared_cta(half %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_half_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_half_shared_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_shared_cta_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr addrspace(3) %addr syncscope("block") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_half_shared_cluster(half %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_half_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_half_shared_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_shared_cluster_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr addrspace(3) %addr syncscope("cluster") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_half_shared_gpu(half %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_half_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_half_shared_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_shared_gpu_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr addrspace(3) %addr syncscope("device") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_half_shared_sys_volatile(half %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_half_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_half_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_shared_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr addrspace(3) %addr syncscope("") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_half_shared_cta_volatile(half %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_half_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_half_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_shared_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr addrspace(3) %addr syncscope("block") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_half_shared_cluster_volatile(half %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_half_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_half_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_shared_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr addrspace(3) %addr syncscope("cluster") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_half_shared_gpu_volatile(half %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_half_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_half_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_shared_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr addrspace(3) %addr syncscope("device") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_bfloat_shared_sys(bfloat %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_bfloat_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_bfloat_shared_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_shared_sys_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr addrspace(3) %addr syncscope("") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_bfloat_shared_cta(bfloat %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_bfloat_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_bfloat_shared_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_shared_cta_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr addrspace(3) %addr syncscope("block") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_bfloat_shared_cluster(bfloat %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_bfloat_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_bfloat_shared_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_shared_cluster_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr addrspace(3) %addr syncscope("cluster") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_bfloat_shared_gpu(bfloat %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_bfloat_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_bfloat_shared_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_shared_gpu_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr addrspace(3) %addr syncscope("device") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_bfloat_shared_sys_volatile(bfloat %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_bfloat_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_bfloat_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_shared_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr addrspace(3) %addr syncscope("") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_bfloat_shared_cta_volatile(bfloat %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_bfloat_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_bfloat_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_shared_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr addrspace(3) %addr syncscope("block") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_bfloat_shared_cluster_volatile(bfloat %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_bfloat_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_bfloat_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_shared_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr addrspace(3) %addr syncscope("cluster") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_bfloat_shared_gpu_volatile(bfloat %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_bfloat_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_bfloat_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_shared_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr addrspace(3) %addr syncscope("device") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_float_shared_sys(float %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_float_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_float_shared_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_shared_sys_param_1];
+; CHECK-NEXT: st.volatile.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr addrspace(3) %addr syncscope("") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_float_shared_cta(float %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_float_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_float_shared_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_shared_cta_param_1];
+; CHECK-NEXT: st.volatile.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr addrspace(3) %addr syncscope("block") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_float_shared_cluster(float %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_float_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_float_shared_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_shared_cluster_param_1];
+; CHECK-NEXT: st.volatile.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr addrspace(3) %addr syncscope("cluster") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_float_shared_gpu(float %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_float_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_float_shared_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_shared_gpu_param_1];
+; CHECK-NEXT: st.volatile.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr addrspace(3) %addr syncscope("device") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_float_shared_sys_volatile(float %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_float_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_float_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_shared_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr addrspace(3) %addr syncscope("") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_float_shared_cta_volatile(float %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_float_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_float_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_shared_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr addrspace(3) %addr syncscope("block") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_float_shared_cluster_volatile(float %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_float_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_float_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_shared_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr addrspace(3) %addr syncscope("cluster") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_float_shared_gpu_volatile(float %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_float_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_float_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_shared_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr addrspace(3) %addr syncscope("device") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_double_shared_sys(double %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_double_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_shared_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_double_shared_sys_param_1];
+; CHECK-NEXT: st.volatile.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr addrspace(3) %addr syncscope("") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_double_shared_cta(double %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_double_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_shared_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_double_shared_cta_param_1];
+; CHECK-NEXT: st.volatile.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr addrspace(3) %addr syncscope("block") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_double_shared_cluster(double %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_double_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_shared_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_double_shared_cluster_param_1];
+; CHECK-NEXT: st.volatile.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr addrspace(3) %addr syncscope("cluster") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_double_shared_gpu(double %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_double_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_shared_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_double_shared_gpu_param_1];
+; CHECK-NEXT: st.volatile.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr addrspace(3) %addr syncscope("device") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_double_shared_sys_volatile(double %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_double_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_double_shared_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr addrspace(3) %addr syncscope("") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_double_shared_cta_volatile(double %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_double_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_double_shared_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr addrspace(3) %addr syncscope("block") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_double_shared_cluster_volatile(double %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_double_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_double_shared_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr addrspace(3) %addr syncscope("cluster") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_double_shared_gpu_volatile(double %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_double_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_double_shared_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr addrspace(3) %addr syncscope("device") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_i8_local_sys(i8 %val) {
+; CHECK-LABEL: monotonic_i8_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot512[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot512;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b8 %rs1, [monotonic_i8_local_sys_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b8 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i8 %val, ptr addrspace(5) %slot syncscope("") monotonic, align 1
+ ret void
+}
+
+define void @monotonic_i8_local_cta(i8 %val) {
+; CHECK-LABEL: monotonic_i8_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot513[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot513;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b8 %rs1, [monotonic_i8_local_cta_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b8 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i8 %val, ptr addrspace(5) %slot syncscope("block") monotonic, align 1
+ ret void
+}
+
+define void @monotonic_i8_local_cluster(i8 %val) {
+; CHECK-LABEL: monotonic_i8_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot514[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot514;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b8 %rs1, [monotonic_i8_local_cluster_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b8 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i8 %val, ptr addrspace(5) %slot syncscope("cluster") monotonic, align 1
+ ret void
+}
+
+define void @monotonic_i8_local_gpu(i8 %val) {
+; CHECK-LABEL: monotonic_i8_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot515[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot515;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b8 %rs1, [monotonic_i8_local_gpu_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b8 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i8 %val, ptr addrspace(5) %slot syncscope("device") monotonic, align 1
+ ret void
+}
+
+define void @monotonic_i8_local_sys_volatile(i8 %val) {
+; CHECK-LABEL: monotonic_i8_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot516[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot516;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b8 %rs1, [monotonic_i8_local_sys_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b8 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i8 %val, ptr addrspace(5) %slot syncscope("") monotonic, align 1
+ ret void
+}
+
+define void @monotonic_i8_local_cta_volatile(i8 %val) {
+; CHECK-LABEL: monotonic_i8_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot517[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot517;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b8 %rs1, [monotonic_i8_local_cta_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b8 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i8 %val, ptr addrspace(5) %slot syncscope("block") monotonic, align 1
+ ret void
+}
+
+define void @monotonic_i8_local_cluster_volatile(i8 %val) {
+; CHECK-LABEL: monotonic_i8_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot518[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot518;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b8 %rs1, [monotonic_i8_local_cluster_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b8 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i8 %val, ptr addrspace(5) %slot syncscope("cluster") monotonic, align 1
+ ret void
+}
+
+define void @monotonic_i8_local_gpu_volatile(i8 %val) {
+; CHECK-LABEL: monotonic_i8_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot519[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot519;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b8 %rs1, [monotonic_i8_local_gpu_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b8 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i8 %val, ptr addrspace(5) %slot syncscope("device") monotonic, align 1
+ ret void
+}
+
+define void @monotonic_i16_local_sys(i16 %val) {
+; CHECK-LABEL: monotonic_i16_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot520[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot520;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_i16_local_sys_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i16 %val, ptr addrspace(5) %slot syncscope("") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_i16_local_cta(i16 %val) {
+; CHECK-LABEL: monotonic_i16_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot521[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot521;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_i16_local_cta_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i16 %val, ptr addrspace(5) %slot syncscope("block") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_i16_local_cluster(i16 %val) {
+; CHECK-LABEL: monotonic_i16_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot522[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot522;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_i16_local_cluster_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i16 %val, ptr addrspace(5) %slot syncscope("cluster") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_i16_local_gpu(i16 %val) {
+; CHECK-LABEL: monotonic_i16_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot523[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot523;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_i16_local_gpu_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i16 %val, ptr addrspace(5) %slot syncscope("device") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_i16_local_sys_volatile(i16 %val) {
+; CHECK-LABEL: monotonic_i16_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot524[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot524;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_i16_local_sys_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i16 %val, ptr addrspace(5) %slot syncscope("") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_i16_local_cta_volatile(i16 %val) {
+; CHECK-LABEL: monotonic_i16_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot525[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot525;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_i16_local_cta_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i16 %val, ptr addrspace(5) %slot syncscope("block") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_i16_local_cluster_volatile(i16 %val) {
+; CHECK-LABEL: monotonic_i16_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot526[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot526;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_i16_local_cluster_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i16 %val, ptr addrspace(5) %slot syncscope("cluster") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_i16_local_gpu_volatile(i16 %val) {
+; CHECK-LABEL: monotonic_i16_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot527[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot527;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_i16_local_gpu_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i16 %val, ptr addrspace(5) %slot syncscope("device") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_i32_local_sys(i32 %val) {
+; CHECK-LABEL: monotonic_i32_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot528[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot528;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [monotonic_i32_local_sys_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i32 %val, ptr addrspace(5) %slot syncscope("") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_i32_local_cta(i32 %val) {
+; CHECK-LABEL: monotonic_i32_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot529[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot529;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [monotonic_i32_local_cta_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i32 %val, ptr addrspace(5) %slot syncscope("block") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_i32_local_cluster(i32 %val) {
+; CHECK-LABEL: monotonic_i32_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot530[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot530;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [monotonic_i32_local_cluster_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i32 %val, ptr addrspace(5) %slot syncscope("cluster") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_i32_local_gpu(i32 %val) {
+; CHECK-LABEL: monotonic_i32_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot531[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot531;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [monotonic_i32_local_gpu_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i32 %val, ptr addrspace(5) %slot syncscope("device") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_i32_local_sys_volatile(i32 %val) {
+; CHECK-LABEL: monotonic_i32_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot532[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot532;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [monotonic_i32_local_sys_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i32 %val, ptr addrspace(5) %slot syncscope("") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_i32_local_cta_volatile(i32 %val) {
+; CHECK-LABEL: monotonic_i32_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot533[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot533;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [monotonic_i32_local_cta_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i32 %val, ptr addrspace(5) %slot syncscope("block") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_i32_local_cluster_volatile(i32 %val) {
+; CHECK-LABEL: monotonic_i32_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot534[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot534;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [monotonic_i32_local_cluster_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i32 %val, ptr addrspace(5) %slot syncscope("cluster") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_i32_local_gpu_volatile(i32 %val) {
+; CHECK-LABEL: monotonic_i32_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot535[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot535;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [monotonic_i32_local_gpu_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i32 %val, ptr addrspace(5) %slot syncscope("device") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_i64_local_sys(i64 %val) {
+; CHECK-LABEL: monotonic_i64_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot536[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot536;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_local_sys_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i64 %val, ptr addrspace(5) %slot syncscope("") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_i64_local_cta(i64 %val) {
+; CHECK-LABEL: monotonic_i64_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot537[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot537;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_local_cta_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i64 %val, ptr addrspace(5) %slot syncscope("block") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_i64_local_cluster(i64 %val) {
+; CHECK-LABEL: monotonic_i64_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot538[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot538;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_local_cluster_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i64 %val, ptr addrspace(5) %slot syncscope("cluster") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_i64_local_gpu(i64 %val) {
+; CHECK-LABEL: monotonic_i64_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot539[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot539;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_local_gpu_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i64 %val, ptr addrspace(5) %slot syncscope("device") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_i64_local_sys_volatile(i64 %val) {
+; CHECK-LABEL: monotonic_i64_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot540[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot540;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_local_sys_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i64 %val, ptr addrspace(5) %slot syncscope("") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_i64_local_cta_volatile(i64 %val) {
+; CHECK-LABEL: monotonic_i64_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot541[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot541;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_local_cta_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i64 %val, ptr addrspace(5) %slot syncscope("block") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_i64_local_cluster_volatile(i64 %val) {
+; CHECK-LABEL: monotonic_i64_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot542[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot542;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_local_cluster_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i64 %val, ptr addrspace(5) %slot syncscope("cluster") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_i64_local_gpu_volatile(i64 %val) {
+; CHECK-LABEL: monotonic_i64_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot543[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot543;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_local_gpu_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i64 %val, ptr addrspace(5) %slot syncscope("device") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_half_local_sys(half %val) {
+; CHECK-LABEL: monotonic_half_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot544[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot544;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_half_local_sys_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic half %val, ptr addrspace(5) %slot syncscope("") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_half_local_cta(half %val) {
+; CHECK-LABEL: monotonic_half_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot545[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot545;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_half_local_cta_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic half %val, ptr addrspace(5) %slot syncscope("block") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_half_local_cluster(half %val) {
+; CHECK-LABEL: monotonic_half_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot546[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot546;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_half_local_cluster_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic half %val, ptr addrspace(5) %slot syncscope("cluster") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_half_local_gpu(half %val) {
+; CHECK-LABEL: monotonic_half_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot547[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot547;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_half_local_gpu_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic half %val, ptr addrspace(5) %slot syncscope("device") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_half_local_sys_volatile(half %val) {
+; CHECK-LABEL: monotonic_half_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot548[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot548;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_half_local_sys_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile half %val, ptr addrspace(5) %slot syncscope("") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_half_local_cta_volatile(half %val) {
+; CHECK-LABEL: monotonic_half_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot549[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot549;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_half_local_cta_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile half %val, ptr addrspace(5) %slot syncscope("block") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_half_local_cluster_volatile(half %val) {
+; CHECK-LABEL: monotonic_half_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot550[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot550;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_half_local_cluster_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile half %val, ptr addrspace(5) %slot syncscope("cluster") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_half_local_gpu_volatile(half %val) {
+; CHECK-LABEL: monotonic_half_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot551[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot551;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_half_local_gpu_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile half %val, ptr addrspace(5) %slot syncscope("device") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_bfloat_local_sys(bfloat %val) {
+; CHECK-LABEL: monotonic_bfloat_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot552[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot552;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_bfloat_local_sys_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic bfloat %val, ptr addrspace(5) %slot syncscope("") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_bfloat_local_cta(bfloat %val) {
+; CHECK-LABEL: monotonic_bfloat_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot553[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot553;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_bfloat_local_cta_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic bfloat %val, ptr addrspace(5) %slot syncscope("block") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_bfloat_local_cluster(bfloat %val) {
+; CHECK-LABEL: monotonic_bfloat_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot554[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot554;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_bfloat_local_cluster_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic bfloat %val, ptr addrspace(5) %slot syncscope("cluster") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_bfloat_local_gpu(bfloat %val) {
+; CHECK-LABEL: monotonic_bfloat_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot555[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot555;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_bfloat_local_gpu_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic bfloat %val, ptr addrspace(5) %slot syncscope("device") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_bfloat_local_sys_volatile(bfloat %val) {
+; CHECK-LABEL: monotonic_bfloat_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot556[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot556;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_bfloat_local_sys_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile bfloat %val, ptr addrspace(5) %slot syncscope("") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_bfloat_local_cta_volatile(bfloat %val) {
+; CHECK-LABEL: monotonic_bfloat_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot557[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot557;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_bfloat_local_cta_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile bfloat %val, ptr addrspace(5) %slot syncscope("block") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_bfloat_local_cluster_volatile(bfloat %val) {
+; CHECK-LABEL: monotonic_bfloat_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot558[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot558;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_bfloat_local_cluster_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile bfloat %val, ptr addrspace(5) %slot syncscope("cluster") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_bfloat_local_gpu_volatile(bfloat %val) {
+; CHECK-LABEL: monotonic_bfloat_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot559[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot559;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_bfloat_local_gpu_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile bfloat %val, ptr addrspace(5) %slot syncscope("device") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_float_local_sys(float %val) {
+; CHECK-LABEL: monotonic_float_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot560[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot560;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [monotonic_float_local_sys_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic float %val, ptr addrspace(5) %slot syncscope("") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_float_local_cta(float %val) {
+; CHECK-LABEL: monotonic_float_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot561[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot561;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [monotonic_float_local_cta_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic float %val, ptr addrspace(5) %slot syncscope("block") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_float_local_cluster(float %val) {
+; CHECK-LABEL: monotonic_float_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot562[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot562;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [monotonic_float_local_cluster_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic float %val, ptr addrspace(5) %slot syncscope("cluster") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_float_local_gpu(float %val) {
+; CHECK-LABEL: monotonic_float_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot563[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot563;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [monotonic_float_local_gpu_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic float %val, ptr addrspace(5) %slot syncscope("device") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_float_local_sys_volatile(float %val) {
+; CHECK-LABEL: monotonic_float_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot564[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot564;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [monotonic_float_local_sys_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile float %val, ptr addrspace(5) %slot syncscope("") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_float_local_cta_volatile(float %val) {
+; CHECK-LABEL: monotonic_float_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot565[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot565;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [monotonic_float_local_cta_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile float %val, ptr addrspace(5) %slot syncscope("block") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_float_local_cluster_volatile(float %val) {
+; CHECK-LABEL: monotonic_float_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot566[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot566;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [monotonic_float_local_cluster_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile float %val, ptr addrspace(5) %slot syncscope("cluster") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_float_local_gpu_volatile(float %val) {
+; CHECK-LABEL: monotonic_float_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot567[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot567;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [monotonic_float_local_gpu_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile float %val, ptr addrspace(5) %slot syncscope("device") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_double_local_sys(double %val) {
+; CHECK-LABEL: monotonic_double_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot568[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot568;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_local_sys_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic double %val, ptr addrspace(5) %slot syncscope("") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_double_local_cta(double %val) {
+; CHECK-LABEL: monotonic_double_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot569[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot569;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_local_cta_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic double %val, ptr addrspace(5) %slot syncscope("block") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_double_local_cluster(double %val) {
+; CHECK-LABEL: monotonic_double_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot570[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot570;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_local_cluster_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic double %val, ptr addrspace(5) %slot syncscope("cluster") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_double_local_gpu(double %val) {
+; CHECK-LABEL: monotonic_double_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot571[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot571;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_local_gpu_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic double %val, ptr addrspace(5) %slot syncscope("device") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_double_local_sys_volatile(double %val) {
+; CHECK-LABEL: monotonic_double_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot572[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot572;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_local_sys_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile double %val, ptr addrspace(5) %slot syncscope("") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_double_local_cta_volatile(double %val) {
+; CHECK-LABEL: monotonic_double_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot573[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot573;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_local_cta_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile double %val, ptr addrspace(5) %slot syncscope("block") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_double_local_cluster_volatile(double %val) {
+; CHECK-LABEL: monotonic_double_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot574[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot574;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_local_cluster_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile double %val, ptr addrspace(5) %slot syncscope("cluster") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_double_local_gpu_volatile(double %val) {
+; CHECK-LABEL: monotonic_double_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot575[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot575;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_local_gpu_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile double %val, ptr addrspace(5) %slot syncscope("device") monotonic, align 8
+ ret void
+}
+
+define void @release_i8_generic_sys(i8 %val, ptr %addr) {
+; CHECK-LABEL: release_i8_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [release_i8_generic_sys_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i8_generic_sys_param_1];
+; CHECK-NEXT: st.volatile.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr %addr syncscope("") release, align 1
+ ret void
+}
+
+define void @release_i8_generic_cta(i8 %val, ptr %addr) {
+; CHECK-LABEL: release_i8_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [release_i8_generic_cta_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i8_generic_cta_param_1];
+; CHECK-NEXT: st.volatile.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr %addr syncscope("block") release, align 1
+ ret void
+}
+
+define void @release_i8_generic_cluster(i8 %val, ptr %addr) {
+; CHECK-LABEL: release_i8_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [release_i8_generic_cluster_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i8_generic_cluster_param_1];
+; CHECK-NEXT: st.volatile.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr %addr syncscope("cluster") release, align 1
+ ret void
+}
+
+define void @release_i8_generic_gpu(i8 %val, ptr %addr) {
+; CHECK-LABEL: release_i8_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [release_i8_generic_gpu_param_0];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i8_generic_gpu_param_1];
+; CHECK-NEXT: st.volatile.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr %addr syncscope("device") release, align 1
+ ret void
+}
+
+define void @release_i8_generic_sys_volatile(i8 %val, ptr %addr) {
+; CHECK-LABEL: release_i8_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [release_i8_generic_sys_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i8_generic_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr %addr syncscope("") release, align 1
+ ret void
+}
+
+define void @release_i8_generic_cta_volatile(i8 %val, ptr %addr) {
+; CHECK-LABEL: release_i8_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [release_i8_generic_cta_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i8_generic_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr %addr syncscope("block") release, align 1
+ ret void
+}
+
+define void @release_i8_generic_cluster_volatile(i8 %val, ptr %addr) {
+; CHECK-LABEL: release_i8_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [release_i8_generic_cluster_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i8_generic_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr %addr syncscope("cluster") release, align 1
+ ret void
+}
+
+define void @release_i8_generic_gpu_volatile(i8 %val, ptr %addr) {
+; CHECK-LABEL: release_i8_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [release_i8_generic_gpu_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i8_generic_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr %addr syncscope("device") release, align 1
+ ret void
+}
+
+define void @release_i16_generic_sys(i16 %val, ptr %addr) {
+; CHECK-LABEL: release_i16_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_i16_generic_sys_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i16_generic_sys_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr %addr syncscope("") release, align 2
+ ret void
+}
+
+define void @release_i16_generic_cta(i16 %val, ptr %addr) {
+; CHECK-LABEL: release_i16_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_i16_generic_cta_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i16_generic_cta_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr %addr syncscope("block") release, align 2
+ ret void
+}
+
+define void @release_i16_generic_cluster(i16 %val, ptr %addr) {
+; CHECK-LABEL: release_i16_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_i16_generic_cluster_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i16_generic_cluster_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr %addr syncscope("cluster") release, align 2
+ ret void
+}
+
+define void @release_i16_generic_gpu(i16 %val, ptr %addr) {
+; CHECK-LABEL: release_i16_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_i16_generic_gpu_param_0];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i16_generic_gpu_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr %addr syncscope("device") release, align 2
+ ret void
+}
+
+define void @release_i16_generic_sys_volatile(i16 %val, ptr %addr) {
+; CHECK-LABEL: release_i16_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_i16_generic_sys_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i16_generic_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr %addr syncscope("") release, align 2
+ ret void
+}
+
+define void @release_i16_generic_cta_volatile(i16 %val, ptr %addr) {
+; CHECK-LABEL: release_i16_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_i16_generic_cta_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i16_generic_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr %addr syncscope("block") release, align 2
+ ret void
+}
+
+define void @release_i16_generic_cluster_volatile(i16 %val, ptr %addr) {
+; CHECK-LABEL: release_i16_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_i16_generic_cluster_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i16_generic_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr %addr syncscope("cluster") release, align 2
+ ret void
+}
+
+define void @release_i16_generic_gpu_volatile(i16 %val, ptr %addr) {
+; CHECK-LABEL: release_i16_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_i16_generic_gpu_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i16_generic_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr %addr syncscope("device") release, align 2
+ ret void
+}
+
+define void @release_i32_generic_sys(i32 %val, ptr %addr) {
+; CHECK-LABEL: release_i32_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_i32_generic_sys_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i32_generic_sys_param_1];
+; CHECK-NEXT: st.volatile.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr %addr syncscope("") release, align 4
+ ret void
+}
+
+define void @release_i32_generic_cta(i32 %val, ptr %addr) {
+; CHECK-LABEL: release_i32_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_i32_generic_cta_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i32_generic_cta_param_1];
+; CHECK-NEXT: st.volatile.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr %addr syncscope("block") release, align 4
+ ret void
+}
+
+define void @release_i32_generic_cluster(i32 %val, ptr %addr) {
+; CHECK-LABEL: release_i32_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_i32_generic_cluster_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i32_generic_cluster_param_1];
+; CHECK-NEXT: st.volatile.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr %addr syncscope("cluster") release, align 4
+ ret void
+}
+
+define void @release_i32_generic_gpu(i32 %val, ptr %addr) {
+; CHECK-LABEL: release_i32_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_i32_generic_gpu_param_0];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i32_generic_gpu_param_1];
+; CHECK-NEXT: st.volatile.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr %addr syncscope("device") release, align 4
+ ret void
+}
+
+define void @release_i32_generic_sys_volatile(i32 %val, ptr %addr) {
+; CHECK-LABEL: release_i32_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_i32_generic_sys_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i32_generic_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr %addr syncscope("") release, align 4
+ ret void
+}
+
+define void @release_i32_generic_cta_volatile(i32 %val, ptr %addr) {
+; CHECK-LABEL: release_i32_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_i32_generic_cta_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i32_generic_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr %addr syncscope("block") release, align 4
+ ret void
+}
+
+define void @release_i32_generic_cluster_volatile(i32 %val, ptr %addr) {
+; CHECK-LABEL: release_i32_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_i32_generic_cluster_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i32_generic_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr %addr syncscope("cluster") release, align 4
+ ret void
+}
+
+define void @release_i32_generic_gpu_volatile(i32 %val, ptr %addr) {
+; CHECK-LABEL: release_i32_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_i32_generic_gpu_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i32_generic_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr %addr syncscope("device") release, align 4
+ ret void
+}
+
+define void @release_i64_generic_sys(i64 %val, ptr %addr) {
+; CHECK-LABEL: release_i64_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i64_generic_sys_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd2, [release_i64_generic_sys_param_1];
+; CHECK-NEXT: st.volatile.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr %addr syncscope("") release, align 8
+ ret void
+}
+
+define void @release_i64_generic_cta(i64 %val, ptr %addr) {
+; CHECK-LABEL: release_i64_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i64_generic_cta_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.param.b64 %rd2, [release_i64_generic_cta_param_1];
+; CHECK-NEXT: st.volatile.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr %addr syncscope("block") release, align 8
+ ret void
+}
+
+define void @release_i64_generic_cluster(i64 %val, ptr %addr) {
+; CHECK-LABEL: release_i64_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i64_generic_cluster_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.param.b64 %rd2, [release_i64_generic_cluster_param_1];
+; CHECK-NEXT: st.volatile.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr %addr syncscope("cluster") release, align 8
+ ret void
+}
+
+define void @release_i64_generic_gpu(i64 %val, ptr %addr) {
+; CHECK-LABEL: release_i64_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i64_generic_gpu_param_0];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: ld.param.b64 %rd2, [release_i64_generic_gpu_param_1];
+; CHECK-NEXT: st.volatile.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr %addr syncscope("device") release, align 8
+ ret void
+}
+
+define void @release_i64_generic_sys_volatile(i64 %val, ptr %addr) {
+; CHECK-LABEL: release_i64_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i64_generic_sys_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd2, [release_i64_generic_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr %addr syncscope("") release, align 8
+ ret void
+}
+
+define void @release_i64_generic_cta_volatile(i64 %val, ptr %addr) {
+; CHECK-LABEL: release_i64_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i64_generic_cta_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd2, [release_i64_generic_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr %addr syncscope("block") release, align 8
+ ret void
+}
+
+define void @release_i64_generic_cluster_volatile(i64 %val, ptr %addr) {
+; CHECK-LABEL: release_i64_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i64_generic_cluster_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd2, [release_i64_generic_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr %addr syncscope("cluster") release, align 8
+ ret void
+}
+
+define void @release_i64_generic_gpu_volatile(i64 %val, ptr %addr) {
+; CHECK-LABEL: release_i64_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i64_generic_gpu_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd2, [release_i64_generic_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr %addr syncscope("device") release, align 8
+ ret void
+}
+
+define void @release_half_generic_sys(half %val, ptr %addr) {
+; CHECK-LABEL: release_half_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_half_generic_sys_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_half_generic_sys_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr %addr syncscope("") release, align 2
+ ret void
+}
+
+define void @release_half_generic_cta(half %val, ptr %addr) {
+; CHECK-LABEL: release_half_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_half_generic_cta_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_half_generic_cta_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr %addr syncscope("block") release, align 2
+ ret void
+}
+
+define void @release_half_generic_cluster(half %val, ptr %addr) {
+; CHECK-LABEL: release_half_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_half_generic_cluster_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_half_generic_cluster_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr %addr syncscope("cluster") release, align 2
+ ret void
+}
+
+define void @release_half_generic_gpu(half %val, ptr %addr) {
+; CHECK-LABEL: release_half_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_half_generic_gpu_param_0];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_half_generic_gpu_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr %addr syncscope("device") release, align 2
+ ret void
+}
+
+define void @release_half_generic_sys_volatile(half %val, ptr %addr) {
+; CHECK-LABEL: release_half_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_half_generic_sys_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_half_generic_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr %addr syncscope("") release, align 2
+ ret void
+}
+
+define void @release_half_generic_cta_volatile(half %val, ptr %addr) {
+; CHECK-LABEL: release_half_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_half_generic_cta_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_half_generic_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr %addr syncscope("block") release, align 2
+ ret void
+}
+
+define void @release_half_generic_cluster_volatile(half %val, ptr %addr) {
+; CHECK-LABEL: release_half_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_half_generic_cluster_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_half_generic_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr %addr syncscope("cluster") release, align 2
+ ret void
+}
+
+define void @release_half_generic_gpu_volatile(half %val, ptr %addr) {
+; CHECK-LABEL: release_half_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_half_generic_gpu_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_half_generic_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr %addr syncscope("device") release, align 2
+ ret void
+}
+
+define void @release_bfloat_generic_sys(bfloat %val, ptr %addr) {
+; CHECK-LABEL: release_bfloat_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_bfloat_generic_sys_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_bfloat_generic_sys_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr %addr syncscope("") release, align 2
+ ret void
+}
+
+define void @release_bfloat_generic_cta(bfloat %val, ptr %addr) {
+; CHECK-LABEL: release_bfloat_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_bfloat_generic_cta_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_bfloat_generic_cta_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr %addr syncscope("block") release, align 2
+ ret void
+}
+
+define void @release_bfloat_generic_cluster(bfloat %val, ptr %addr) {
+; CHECK-LABEL: release_bfloat_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_bfloat_generic_cluster_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_bfloat_generic_cluster_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr %addr syncscope("cluster") release, align 2
+ ret void
+}
+
+define void @release_bfloat_generic_gpu(bfloat %val, ptr %addr) {
+; CHECK-LABEL: release_bfloat_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_bfloat_generic_gpu_param_0];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_bfloat_generic_gpu_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr %addr syncscope("device") release, align 2
+ ret void
+}
+
+define void @release_bfloat_generic_sys_volatile(bfloat %val, ptr %addr) {
+; CHECK-LABEL: release_bfloat_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_bfloat_generic_sys_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_bfloat_generic_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr %addr syncscope("") release, align 2
+ ret void
+}
+
+define void @release_bfloat_generic_cta_volatile(bfloat %val, ptr %addr) {
+; CHECK-LABEL: release_bfloat_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_bfloat_generic_cta_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_bfloat_generic_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr %addr syncscope("block") release, align 2
+ ret void
+}
+
+define void @release_bfloat_generic_cluster_volatile(bfloat %val, ptr %addr) {
+; CHECK-LABEL: release_bfloat_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_bfloat_generic_cluster_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_bfloat_generic_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr %addr syncscope("cluster") release, align 2
+ ret void
+}
+
+define void @release_bfloat_generic_gpu_volatile(bfloat %val, ptr %addr) {
+; CHECK-LABEL: release_bfloat_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_bfloat_generic_gpu_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_bfloat_generic_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr %addr syncscope("device") release, align 2
+ ret void
+}
+
+define void @release_float_generic_sys(float %val, ptr %addr) {
+; CHECK-LABEL: release_float_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_float_generic_sys_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_float_generic_sys_param_1];
+; CHECK-NEXT: st.volatile.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr %addr syncscope("") release, align 4
+ ret void
+}
+
+define void @release_float_generic_cta(float %val, ptr %addr) {
+; CHECK-LABEL: release_float_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_float_generic_cta_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_float_generic_cta_param_1];
+; CHECK-NEXT: st.volatile.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr %addr syncscope("block") release, align 4
+ ret void
+}
+
+define void @release_float_generic_cluster(float %val, ptr %addr) {
+; CHECK-LABEL: release_float_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_float_generic_cluster_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_float_generic_cluster_param_1];
+; CHECK-NEXT: st.volatile.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr %addr syncscope("cluster") release, align 4
+ ret void
+}
+
+define void @release_float_generic_gpu(float %val, ptr %addr) {
+; CHECK-LABEL: release_float_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_float_generic_gpu_param_0];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_float_generic_gpu_param_1];
+; CHECK-NEXT: st.volatile.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr %addr syncscope("device") release, align 4
+ ret void
+}
+
+define void @release_float_generic_sys_volatile(float %val, ptr %addr) {
+; CHECK-LABEL: release_float_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_float_generic_sys_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_float_generic_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr %addr syncscope("") release, align 4
+ ret void
+}
+
+define void @release_float_generic_cta_volatile(float %val, ptr %addr) {
+; CHECK-LABEL: release_float_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_float_generic_cta_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_float_generic_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr %addr syncscope("block") release, align 4
+ ret void
+}
+
+define void @release_float_generic_cluster_volatile(float %val, ptr %addr) {
+; CHECK-LABEL: release_float_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_float_generic_cluster_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_float_generic_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr %addr syncscope("cluster") release, align 4
+ ret void
+}
+
+define void @release_float_generic_gpu_volatile(float %val, ptr %addr) {
+; CHECK-LABEL: release_float_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_float_generic_gpu_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_float_generic_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr %addr syncscope("device") release, align 4
+ ret void
+}
+
+define void @release_double_generic_sys(double %val, ptr %addr) {
+; CHECK-LABEL: release_double_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_double_generic_sys_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd2, [release_double_generic_sys_param_1];
+; CHECK-NEXT: st.volatile.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr %addr syncscope("") release, align 8
+ ret void
+}
+
+define void @release_double_generic_cta(double %val, ptr %addr) {
+; CHECK-LABEL: release_double_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_double_generic_cta_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.param.b64 %rd2, [release_double_generic_cta_param_1];
+; CHECK-NEXT: st.volatile.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr %addr syncscope("block") release, align 8
+ ret void
+}
+
+define void @release_double_generic_cluster(double %val, ptr %addr) {
+; CHECK-LABEL: release_double_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_double_generic_cluster_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.param.b64 %rd2, [release_double_generic_cluster_param_1];
+; CHECK-NEXT: st.volatile.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr %addr syncscope("cluster") release, align 8
+ ret void
+}
+
+define void @release_double_generic_gpu(double %val, ptr %addr) {
+; CHECK-LABEL: release_double_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_double_generic_gpu_param_0];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: ld.param.b64 %rd2, [release_double_generic_gpu_param_1];
+; CHECK-NEXT: st.volatile.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr %addr syncscope("device") release, align 8
+ ret void
+}
+
+define void @release_double_generic_sys_volatile(double %val, ptr %addr) {
+; CHECK-LABEL: release_double_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_double_generic_sys_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd2, [release_double_generic_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr %addr syncscope("") release, align 8
+ ret void
+}
+
+define void @release_double_generic_cta_volatile(double %val, ptr %addr) {
+; CHECK-LABEL: release_double_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_double_generic_cta_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd2, [release_double_generic_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr %addr syncscope("block") release, align 8
+ ret void
+}
+
+define void @release_double_generic_cluster_volatile(double %val, ptr %addr) {
+; CHECK-LABEL: release_double_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_double_generic_cluster_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd2, [release_double_generic_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr %addr syncscope("cluster") release, align 8
+ ret void
+}
+
+define void @release_double_generic_gpu_volatile(double %val, ptr %addr) {
+; CHECK-LABEL: release_double_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_double_generic_gpu_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd2, [release_double_generic_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr %addr syncscope("device") release, align 8
+ ret void
+}
+
+define void @release_i8_global_sys(i8 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_i8_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [release_i8_global_sys_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i8_global_sys_param_1];
+; CHECK-NEXT: st.volatile.global.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr addrspace(1) %addr syncscope("") release, align 1
+ ret void
+}
+
+define void @release_i8_global_cta(i8 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_i8_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [release_i8_global_cta_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i8_global_cta_param_1];
+; CHECK-NEXT: st.volatile.global.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr addrspace(1) %addr syncscope("block") release, align 1
+ ret void
+}
+
+define void @release_i8_global_cluster(i8 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_i8_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [release_i8_global_cluster_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i8_global_cluster_param_1];
+; CHECK-NEXT: st.volatile.global.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr addrspace(1) %addr syncscope("cluster") release, align 1
+ ret void
+}
+
+define void @release_i8_global_gpu(i8 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_i8_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [release_i8_global_gpu_param_0];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i8_global_gpu_param_1];
+; CHECK-NEXT: st.volatile.global.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr addrspace(1) %addr syncscope("device") release, align 1
+ ret void
+}
+
+define void @release_i8_global_sys_volatile(i8 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_i8_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [release_i8_global_sys_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i8_global_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr addrspace(1) %addr syncscope("") release, align 1
+ ret void
+}
+
+define void @release_i8_global_cta_volatile(i8 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_i8_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [release_i8_global_cta_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i8_global_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr addrspace(1) %addr syncscope("block") release, align 1
+ ret void
+}
+
+define void @release_i8_global_cluster_volatile(i8 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_i8_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [release_i8_global_cluster_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i8_global_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr addrspace(1) %addr syncscope("cluster") release, align 1
+ ret void
+}
+
+define void @release_i8_global_gpu_volatile(i8 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_i8_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [release_i8_global_gpu_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i8_global_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr addrspace(1) %addr syncscope("device") release, align 1
+ ret void
+}
+
+define void @release_i16_global_sys(i16 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_i16_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_i16_global_sys_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i16_global_sys_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr addrspace(1) %addr syncscope("") release, align 2
+ ret void
+}
+
+define void @release_i16_global_cta(i16 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_i16_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_i16_global_cta_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i16_global_cta_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr addrspace(1) %addr syncscope("block") release, align 2
+ ret void
+}
+
+define void @release_i16_global_cluster(i16 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_i16_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_i16_global_cluster_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i16_global_cluster_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr addrspace(1) %addr syncscope("cluster") release, align 2
+ ret void
+}
+
+define void @release_i16_global_gpu(i16 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_i16_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_i16_global_gpu_param_0];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i16_global_gpu_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr addrspace(1) %addr syncscope("device") release, align 2
+ ret void
+}
+
+define void @release_i16_global_sys_volatile(i16 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_i16_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_i16_global_sys_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i16_global_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr addrspace(1) %addr syncscope("") release, align 2
+ ret void
+}
+
+define void @release_i16_global_cta_volatile(i16 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_i16_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_i16_global_cta_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i16_global_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr addrspace(1) %addr syncscope("block") release, align 2
+ ret void
+}
+
+define void @release_i16_global_cluster_volatile(i16 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_i16_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_i16_global_cluster_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i16_global_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr addrspace(1) %addr syncscope("cluster") release, align 2
+ ret void
+}
+
+define void @release_i16_global_gpu_volatile(i16 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_i16_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_i16_global_gpu_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i16_global_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr addrspace(1) %addr syncscope("device") release, align 2
+ ret void
+}
+
+define void @release_i32_global_sys(i32 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_i32_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_i32_global_sys_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i32_global_sys_param_1];
+; CHECK-NEXT: st.volatile.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr addrspace(1) %addr syncscope("") release, align 4
+ ret void
+}
+
+define void @release_i32_global_cta(i32 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_i32_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_i32_global_cta_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i32_global_cta_param_1];
+; CHECK-NEXT: st.volatile.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr addrspace(1) %addr syncscope("block") release, align 4
+ ret void
+}
+
+define void @release_i32_global_cluster(i32 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_i32_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_i32_global_cluster_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i32_global_cluster_param_1];
+; CHECK-NEXT: st.volatile.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr addrspace(1) %addr syncscope("cluster") release, align 4
+ ret void
+}
+
+define void @release_i32_global_gpu(i32 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_i32_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_i32_global_gpu_param_0];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i32_global_gpu_param_1];
+; CHECK-NEXT: st.volatile.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr addrspace(1) %addr syncscope("device") release, align 4
+ ret void
+}
+
+define void @release_i32_global_sys_volatile(i32 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_i32_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_i32_global_sys_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i32_global_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr addrspace(1) %addr syncscope("") release, align 4
+ ret void
+}
+
+define void @release_i32_global_cta_volatile(i32 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_i32_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_i32_global_cta_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i32_global_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr addrspace(1) %addr syncscope("block") release, align 4
+ ret void
+}
+
+define void @release_i32_global_cluster_volatile(i32 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_i32_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_i32_global_cluster_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i32_global_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr addrspace(1) %addr syncscope("cluster") release, align 4
+ ret void
+}
+
+define void @release_i32_global_gpu_volatile(i32 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_i32_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_i32_global_gpu_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i32_global_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr addrspace(1) %addr syncscope("device") release, align 4
+ ret void
+}
+
+define void @release_i64_global_sys(i64 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_i64_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i64_global_sys_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd2, [release_i64_global_sys_param_1];
+; CHECK-NEXT: st.volatile.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr addrspace(1) %addr syncscope("") release, align 8
+ ret void
+}
+
+define void @release_i64_global_cta(i64 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_i64_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i64_global_cta_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.param.b64 %rd2, [release_i64_global_cta_param_1];
+; CHECK-NEXT: st.volatile.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr addrspace(1) %addr syncscope("block") release, align 8
+ ret void
+}
+
+define void @release_i64_global_cluster(i64 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_i64_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i64_global_cluster_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.param.b64 %rd2, [release_i64_global_cluster_param_1];
+; CHECK-NEXT: st.volatile.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr addrspace(1) %addr syncscope("cluster") release, align 8
+ ret void
+}
+
+define void @release_i64_global_gpu(i64 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_i64_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i64_global_gpu_param_0];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: ld.param.b64 %rd2, [release_i64_global_gpu_param_1];
+; CHECK-NEXT: st.volatile.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr addrspace(1) %addr syncscope("device") release, align 8
+ ret void
+}
+
+define void @release_i64_global_sys_volatile(i64 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_i64_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i64_global_sys_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd2, [release_i64_global_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr addrspace(1) %addr syncscope("") release, align 8
+ ret void
+}
+
+define void @release_i64_global_cta_volatile(i64 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_i64_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i64_global_cta_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd2, [release_i64_global_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr addrspace(1) %addr syncscope("block") release, align 8
+ ret void
+}
+
+define void @release_i64_global_cluster_volatile(i64 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_i64_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i64_global_cluster_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd2, [release_i64_global_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr addrspace(1) %addr syncscope("cluster") release, align 8
+ ret void
+}
+
+define void @release_i64_global_gpu_volatile(i64 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_i64_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i64_global_gpu_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd2, [release_i64_global_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr addrspace(1) %addr syncscope("device") release, align 8
+ ret void
+}
+
+define void @release_half_global_sys(half %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_half_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_half_global_sys_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_half_global_sys_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr addrspace(1) %addr syncscope("") release, align 2
+ ret void
+}
+
+define void @release_half_global_cta(half %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_half_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_half_global_cta_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_half_global_cta_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr addrspace(1) %addr syncscope("block") release, align 2
+ ret void
+}
+
+define void @release_half_global_cluster(half %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_half_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_half_global_cluster_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_half_global_cluster_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr addrspace(1) %addr syncscope("cluster") release, align 2
+ ret void
+}
+
+define void @release_half_global_gpu(half %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_half_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_half_global_gpu_param_0];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_half_global_gpu_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr addrspace(1) %addr syncscope("device") release, align 2
+ ret void
+}
+
+define void @release_half_global_sys_volatile(half %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_half_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_half_global_sys_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_half_global_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr addrspace(1) %addr syncscope("") release, align 2
+ ret void
+}
+
+define void @release_half_global_cta_volatile(half %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_half_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_half_global_cta_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_half_global_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr addrspace(1) %addr syncscope("block") release, align 2
+ ret void
+}
+
+define void @release_half_global_cluster_volatile(half %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_half_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_half_global_cluster_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_half_global_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr addrspace(1) %addr syncscope("cluster") release, align 2
+ ret void
+}
+
+define void @release_half_global_gpu_volatile(half %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_half_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_half_global_gpu_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_half_global_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr addrspace(1) %addr syncscope("device") release, align 2
+ ret void
+}
+
+define void @release_bfloat_global_sys(bfloat %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_bfloat_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_bfloat_global_sys_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_bfloat_global_sys_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr addrspace(1) %addr syncscope("") release, align 2
+ ret void
+}
+
+define void @release_bfloat_global_cta(bfloat %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_bfloat_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_bfloat_global_cta_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_bfloat_global_cta_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr addrspace(1) %addr syncscope("block") release, align 2
+ ret void
+}
+
+define void @release_bfloat_global_cluster(bfloat %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_bfloat_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_bfloat_global_cluster_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_bfloat_global_cluster_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr addrspace(1) %addr syncscope("cluster") release, align 2
+ ret void
+}
+
+define void @release_bfloat_global_gpu(bfloat %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_bfloat_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_bfloat_global_gpu_param_0];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_bfloat_global_gpu_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr addrspace(1) %addr syncscope("device") release, align 2
+ ret void
+}
+
+define void @release_bfloat_global_sys_volatile(bfloat %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_bfloat_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_bfloat_global_sys_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_bfloat_global_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr addrspace(1) %addr syncscope("") release, align 2
+ ret void
+}
+
+define void @release_bfloat_global_cta_volatile(bfloat %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_bfloat_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_bfloat_global_cta_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_bfloat_global_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr addrspace(1) %addr syncscope("block") release, align 2
+ ret void
+}
+
+define void @release_bfloat_global_cluster_volatile(bfloat %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_bfloat_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_bfloat_global_cluster_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_bfloat_global_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr addrspace(1) %addr syncscope("cluster") release, align 2
+ ret void
+}
+
+define void @release_bfloat_global_gpu_volatile(bfloat %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_bfloat_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_bfloat_global_gpu_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_bfloat_global_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr addrspace(1) %addr syncscope("device") release, align 2
+ ret void
+}
+
+define void @release_float_global_sys(float %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_float_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_float_global_sys_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_float_global_sys_param_1];
+; CHECK-NEXT: st.volatile.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr addrspace(1) %addr syncscope("") release, align 4
+ ret void
+}
+
+define void @release_float_global_cta(float %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_float_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_float_global_cta_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_float_global_cta_param_1];
+; CHECK-NEXT: st.volatile.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr addrspace(1) %addr syncscope("block") release, align 4
+ ret void
+}
+
+define void @release_float_global_cluster(float %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_float_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_float_global_cluster_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_float_global_cluster_param_1];
+; CHECK-NEXT: st.volatile.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr addrspace(1) %addr syncscope("cluster") release, align 4
+ ret void
+}
+
+define void @release_float_global_gpu(float %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_float_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_float_global_gpu_param_0];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_float_global_gpu_param_1];
+; CHECK-NEXT: st.volatile.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr addrspace(1) %addr syncscope("device") release, align 4
+ ret void
+}
+
+define void @release_float_global_sys_volatile(float %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_float_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_float_global_sys_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_float_global_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr addrspace(1) %addr syncscope("") release, align 4
+ ret void
+}
+
+define void @release_float_global_cta_volatile(float %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_float_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_float_global_cta_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_float_global_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr addrspace(1) %addr syncscope("block") release, align 4
+ ret void
+}
+
+define void @release_float_global_cluster_volatile(float %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_float_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_float_global_cluster_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_float_global_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr addrspace(1) %addr syncscope("cluster") release, align 4
+ ret void
+}
+
+define void @release_float_global_gpu_volatile(float %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_float_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_float_global_gpu_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_float_global_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr addrspace(1) %addr syncscope("device") release, align 4
+ ret void
+}
+
+define void @release_double_global_sys(double %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_double_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_double_global_sys_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd2, [release_double_global_sys_param_1];
+; CHECK-NEXT: st.volatile.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr addrspace(1) %addr syncscope("") release, align 8
+ ret void
+}
+
+define void @release_double_global_cta(double %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_double_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_double_global_cta_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.param.b64 %rd2, [release_double_global_cta_param_1];
+; CHECK-NEXT: st.volatile.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr addrspace(1) %addr syncscope("block") release, align 8
+ ret void
+}
+
+define void @release_double_global_cluster(double %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_double_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_double_global_cluster_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.param.b64 %rd2, [release_double_global_cluster_param_1];
+; CHECK-NEXT: st.volatile.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr addrspace(1) %addr syncscope("cluster") release, align 8
+ ret void
+}
+
+define void @release_double_global_gpu(double %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_double_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_double_global_gpu_param_0];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: ld.param.b64 %rd2, [release_double_global_gpu_param_1];
+; CHECK-NEXT: st.volatile.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr addrspace(1) %addr syncscope("device") release, align 8
+ ret void
+}
+
+define void @release_double_global_sys_volatile(double %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_double_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_double_global_sys_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd2, [release_double_global_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr addrspace(1) %addr syncscope("") release, align 8
+ ret void
+}
+
+define void @release_double_global_cta_volatile(double %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_double_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_double_global_cta_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd2, [release_double_global_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr addrspace(1) %addr syncscope("block") release, align 8
+ ret void
+}
+
+define void @release_double_global_cluster_volatile(double %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_double_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_double_global_cluster_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd2, [release_double_global_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr addrspace(1) %addr syncscope("cluster") release, align 8
+ ret void
+}
+
+define void @release_double_global_gpu_volatile(double %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_double_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_double_global_gpu_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd2, [release_double_global_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr addrspace(1) %addr syncscope("device") release, align 8
+ ret void
+}
+
+define void @release_i8_shared_sys(i8 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_i8_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [release_i8_shared_sys_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i8_shared_sys_param_1];
+; CHECK-NEXT: st.volatile.shared.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr addrspace(3) %addr syncscope("") release, align 1
+ ret void
+}
+
+define void @release_i8_shared_cta(i8 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_i8_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [release_i8_shared_cta_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i8_shared_cta_param_1];
+; CHECK-NEXT: st.volatile.shared.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr addrspace(3) %addr syncscope("block") release, align 1
+ ret void
+}
+
+define void @release_i8_shared_cluster(i8 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_i8_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [release_i8_shared_cluster_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i8_shared_cluster_param_1];
+; CHECK-NEXT: st.volatile.shared.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr addrspace(3) %addr syncscope("cluster") release, align 1
+ ret void
+}
+
+define void @release_i8_shared_gpu(i8 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_i8_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [release_i8_shared_gpu_param_0];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i8_shared_gpu_param_1];
+; CHECK-NEXT: st.volatile.shared.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr addrspace(3) %addr syncscope("device") release, align 1
+ ret void
+}
+
+define void @release_i8_shared_sys_volatile(i8 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_i8_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [release_i8_shared_sys_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i8_shared_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr addrspace(3) %addr syncscope("") release, align 1
+ ret void
+}
+
+define void @release_i8_shared_cta_volatile(i8 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_i8_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [release_i8_shared_cta_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i8_shared_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr addrspace(3) %addr syncscope("block") release, align 1
+ ret void
+}
+
+define void @release_i8_shared_cluster_volatile(i8 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_i8_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [release_i8_shared_cluster_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i8_shared_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr addrspace(3) %addr syncscope("cluster") release, align 1
+ ret void
+}
+
+define void @release_i8_shared_gpu_volatile(i8 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_i8_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [release_i8_shared_gpu_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i8_shared_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr addrspace(3) %addr syncscope("device") release, align 1
+ ret void
+}
+
+define void @release_i16_shared_sys(i16 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_i16_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_i16_shared_sys_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i16_shared_sys_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr addrspace(3) %addr syncscope("") release, align 2
+ ret void
+}
+
+define void @release_i16_shared_cta(i16 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_i16_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_i16_shared_cta_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i16_shared_cta_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr addrspace(3) %addr syncscope("block") release, align 2
+ ret void
+}
+
+define void @release_i16_shared_cluster(i16 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_i16_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_i16_shared_cluster_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i16_shared_cluster_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr addrspace(3) %addr syncscope("cluster") release, align 2
+ ret void
+}
+
+define void @release_i16_shared_gpu(i16 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_i16_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_i16_shared_gpu_param_0];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i16_shared_gpu_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr addrspace(3) %addr syncscope("device") release, align 2
+ ret void
+}
+
+define void @release_i16_shared_sys_volatile(i16 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_i16_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_i16_shared_sys_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i16_shared_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr addrspace(3) %addr syncscope("") release, align 2
+ ret void
+}
+
+define void @release_i16_shared_cta_volatile(i16 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_i16_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_i16_shared_cta_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i16_shared_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr addrspace(3) %addr syncscope("block") release, align 2
+ ret void
+}
+
+define void @release_i16_shared_cluster_volatile(i16 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_i16_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_i16_shared_cluster_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i16_shared_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr addrspace(3) %addr syncscope("cluster") release, align 2
+ ret void
+}
+
+define void @release_i16_shared_gpu_volatile(i16 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_i16_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_i16_shared_gpu_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i16_shared_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr addrspace(3) %addr syncscope("device") release, align 2
+ ret void
+}
+
+define void @release_i32_shared_sys(i32 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_i32_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_i32_shared_sys_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i32_shared_sys_param_1];
+; CHECK-NEXT: st.volatile.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr addrspace(3) %addr syncscope("") release, align 4
+ ret void
+}
+
+define void @release_i32_shared_cta(i32 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_i32_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_i32_shared_cta_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i32_shared_cta_param_1];
+; CHECK-NEXT: st.volatile.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr addrspace(3) %addr syncscope("block") release, align 4
+ ret void
+}
+
+define void @release_i32_shared_cluster(i32 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_i32_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_i32_shared_cluster_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i32_shared_cluster_param_1];
+; CHECK-NEXT: st.volatile.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr addrspace(3) %addr syncscope("cluster") release, align 4
+ ret void
+}
+
+define void @release_i32_shared_gpu(i32 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_i32_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_i32_shared_gpu_param_0];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i32_shared_gpu_param_1];
+; CHECK-NEXT: st.volatile.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr addrspace(3) %addr syncscope("device") release, align 4
+ ret void
+}
+
+define void @release_i32_shared_sys_volatile(i32 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_i32_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_i32_shared_sys_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i32_shared_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr addrspace(3) %addr syncscope("") release, align 4
+ ret void
+}
+
+define void @release_i32_shared_cta_volatile(i32 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_i32_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_i32_shared_cta_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i32_shared_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr addrspace(3) %addr syncscope("block") release, align 4
+ ret void
+}
+
+define void @release_i32_shared_cluster_volatile(i32 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_i32_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_i32_shared_cluster_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i32_shared_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr addrspace(3) %addr syncscope("cluster") release, align 4
+ ret void
+}
+
+define void @release_i32_shared_gpu_volatile(i32 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_i32_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_i32_shared_gpu_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i32_shared_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr addrspace(3) %addr syncscope("device") release, align 4
+ ret void
+}
+
+define void @release_i64_shared_sys(i64 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_i64_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i64_shared_sys_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd2, [release_i64_shared_sys_param_1];
+; CHECK-NEXT: st.volatile.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr addrspace(3) %addr syncscope("") release, align 8
+ ret void
+}
+
+define void @release_i64_shared_cta(i64 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_i64_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i64_shared_cta_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.param.b64 %rd2, [release_i64_shared_cta_param_1];
+; CHECK-NEXT: st.volatile.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr addrspace(3) %addr syncscope("block") release, align 8
+ ret void
+}
+
+define void @release_i64_shared_cluster(i64 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_i64_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i64_shared_cluster_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.param.b64 %rd2, [release_i64_shared_cluster_param_1];
+; CHECK-NEXT: st.volatile.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr addrspace(3) %addr syncscope("cluster") release, align 8
+ ret void
+}
+
+define void @release_i64_shared_gpu(i64 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_i64_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i64_shared_gpu_param_0];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: ld.param.b64 %rd2, [release_i64_shared_gpu_param_1];
+; CHECK-NEXT: st.volatile.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr addrspace(3) %addr syncscope("device") release, align 8
+ ret void
+}
+
+define void @release_i64_shared_sys_volatile(i64 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_i64_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i64_shared_sys_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd2, [release_i64_shared_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr addrspace(3) %addr syncscope("") release, align 8
+ ret void
+}
+
+define void @release_i64_shared_cta_volatile(i64 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_i64_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i64_shared_cta_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd2, [release_i64_shared_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr addrspace(3) %addr syncscope("block") release, align 8
+ ret void
+}
+
+define void @release_i64_shared_cluster_volatile(i64 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_i64_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i64_shared_cluster_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd2, [release_i64_shared_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr addrspace(3) %addr syncscope("cluster") release, align 8
+ ret void
+}
+
+define void @release_i64_shared_gpu_volatile(i64 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_i64_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i64_shared_gpu_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd2, [release_i64_shared_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr addrspace(3) %addr syncscope("device") release, align 8
+ ret void
+}
+
+define void @release_half_shared_sys(half %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_half_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_half_shared_sys_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_half_shared_sys_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr addrspace(3) %addr syncscope("") release, align 2
+ ret void
+}
+
+define void @release_half_shared_cta(half %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_half_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_half_shared_cta_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_half_shared_cta_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr addrspace(3) %addr syncscope("block") release, align 2
+ ret void
+}
+
+define void @release_half_shared_cluster(half %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_half_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_half_shared_cluster_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_half_shared_cluster_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr addrspace(3) %addr syncscope("cluster") release, align 2
+ ret void
+}
+
+define void @release_half_shared_gpu(half %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_half_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_half_shared_gpu_param_0];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_half_shared_gpu_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr addrspace(3) %addr syncscope("device") release, align 2
+ ret void
+}
+
+define void @release_half_shared_sys_volatile(half %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_half_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_half_shared_sys_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_half_shared_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr addrspace(3) %addr syncscope("") release, align 2
+ ret void
+}
+
+define void @release_half_shared_cta_volatile(half %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_half_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_half_shared_cta_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_half_shared_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr addrspace(3) %addr syncscope("block") release, align 2
+ ret void
+}
+
+define void @release_half_shared_cluster_volatile(half %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_half_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_half_shared_cluster_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_half_shared_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr addrspace(3) %addr syncscope("cluster") release, align 2
+ ret void
+}
+
+define void @release_half_shared_gpu_volatile(half %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_half_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_half_shared_gpu_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_half_shared_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr addrspace(3) %addr syncscope("device") release, align 2
+ ret void
+}
+
+define void @release_bfloat_shared_sys(bfloat %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_bfloat_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_bfloat_shared_sys_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_bfloat_shared_sys_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr addrspace(3) %addr syncscope("") release, align 2
+ ret void
+}
+
+define void @release_bfloat_shared_cta(bfloat %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_bfloat_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_bfloat_shared_cta_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_bfloat_shared_cta_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr addrspace(3) %addr syncscope("block") release, align 2
+ ret void
+}
+
+define void @release_bfloat_shared_cluster(bfloat %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_bfloat_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_bfloat_shared_cluster_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_bfloat_shared_cluster_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr addrspace(3) %addr syncscope("cluster") release, align 2
+ ret void
+}
+
+define void @release_bfloat_shared_gpu(bfloat %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_bfloat_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_bfloat_shared_gpu_param_0];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_bfloat_shared_gpu_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr addrspace(3) %addr syncscope("device") release, align 2
+ ret void
+}
+
+define void @release_bfloat_shared_sys_volatile(bfloat %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_bfloat_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_bfloat_shared_sys_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_bfloat_shared_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr addrspace(3) %addr syncscope("") release, align 2
+ ret void
+}
+
+define void @release_bfloat_shared_cta_volatile(bfloat %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_bfloat_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_bfloat_shared_cta_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_bfloat_shared_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr addrspace(3) %addr syncscope("block") release, align 2
+ ret void
+}
+
+define void @release_bfloat_shared_cluster_volatile(bfloat %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_bfloat_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_bfloat_shared_cluster_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_bfloat_shared_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr addrspace(3) %addr syncscope("cluster") release, align 2
+ ret void
+}
+
+define void @release_bfloat_shared_gpu_volatile(bfloat %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_bfloat_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_bfloat_shared_gpu_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_bfloat_shared_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr addrspace(3) %addr syncscope("device") release, align 2
+ ret void
+}
+
+define void @release_float_shared_sys(float %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_float_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_float_shared_sys_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_float_shared_sys_param_1];
+; CHECK-NEXT: st.volatile.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr addrspace(3) %addr syncscope("") release, align 4
+ ret void
+}
+
+define void @release_float_shared_cta(float %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_float_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_float_shared_cta_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_float_shared_cta_param_1];
+; CHECK-NEXT: st.volatile.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr addrspace(3) %addr syncscope("block") release, align 4
+ ret void
+}
+
+define void @release_float_shared_cluster(float %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_float_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_float_shared_cluster_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_float_shared_cluster_param_1];
+; CHECK-NEXT: st.volatile.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr addrspace(3) %addr syncscope("cluster") release, align 4
+ ret void
+}
+
+define void @release_float_shared_gpu(float %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_float_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_float_shared_gpu_param_0];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_float_shared_gpu_param_1];
+; CHECK-NEXT: st.volatile.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr addrspace(3) %addr syncscope("device") release, align 4
+ ret void
+}
+
+define void @release_float_shared_sys_volatile(float %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_float_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_float_shared_sys_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_float_shared_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr addrspace(3) %addr syncscope("") release, align 4
+ ret void
+}
+
+define void @release_float_shared_cta_volatile(float %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_float_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_float_shared_cta_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_float_shared_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr addrspace(3) %addr syncscope("block") release, align 4
+ ret void
+}
+
+define void @release_float_shared_cluster_volatile(float %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_float_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_float_shared_cluster_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_float_shared_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr addrspace(3) %addr syncscope("cluster") release, align 4
+ ret void
+}
+
+define void @release_float_shared_gpu_volatile(float %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_float_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_float_shared_gpu_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_float_shared_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr addrspace(3) %addr syncscope("device") release, align 4
+ ret void
+}
+
+define void @release_double_shared_sys(double %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_double_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_double_shared_sys_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd2, [release_double_shared_sys_param_1];
+; CHECK-NEXT: st.volatile.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr addrspace(3) %addr syncscope("") release, align 8
+ ret void
+}
+
+define void @release_double_shared_cta(double %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_double_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_double_shared_cta_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.param.b64 %rd2, [release_double_shared_cta_param_1];
+; CHECK-NEXT: st.volatile.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr addrspace(3) %addr syncscope("block") release, align 8
+ ret void
+}
+
+define void @release_double_shared_cluster(double %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_double_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_double_shared_cluster_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.param.b64 %rd2, [release_double_shared_cluster_param_1];
+; CHECK-NEXT: st.volatile.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr addrspace(3) %addr syncscope("cluster") release, align 8
+ ret void
+}
+
+define void @release_double_shared_gpu(double %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_double_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_double_shared_gpu_param_0];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: ld.param.b64 %rd2, [release_double_shared_gpu_param_1];
+; CHECK-NEXT: st.volatile.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr addrspace(3) %addr syncscope("device") release, align 8
+ ret void
+}
+
+define void @release_double_shared_sys_volatile(double %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_double_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_double_shared_sys_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd2, [release_double_shared_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr addrspace(3) %addr syncscope("") release, align 8
+ ret void
+}
+
+define void @release_double_shared_cta_volatile(double %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_double_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_double_shared_cta_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd2, [release_double_shared_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr addrspace(3) %addr syncscope("block") release, align 8
+ ret void
+}
+
+define void @release_double_shared_cluster_volatile(double %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_double_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_double_shared_cluster_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd2, [release_double_shared_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr addrspace(3) %addr syncscope("cluster") release, align 8
+ ret void
+}
+
+define void @release_double_shared_gpu_volatile(double %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_double_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_double_shared_gpu_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd2, [release_double_shared_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr addrspace(3) %addr syncscope("device") release, align 8
+ ret void
+}
+
+define void @release_i8_local_sys(i8 %val) {
+; CHECK-LABEL: release_i8_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot768[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot768;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b8 %rs1, [release_i8_local_sys_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b8 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i8 %val, ptr addrspace(5) %slot syncscope("") release, align 1
+ ret void
+}
+
+define void @release_i8_local_cta(i8 %val) {
+; CHECK-LABEL: release_i8_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot769[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot769;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b8 %rs1, [release_i8_local_cta_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b8 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i8 %val, ptr addrspace(5) %slot syncscope("block") release, align 1
+ ret void
+}
+
+define void @release_i8_local_cluster(i8 %val) {
+; CHECK-LABEL: release_i8_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot770[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot770;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b8 %rs1, [release_i8_local_cluster_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b8 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i8 %val, ptr addrspace(5) %slot syncscope("cluster") release, align 1
+ ret void
+}
+
+define void @release_i8_local_gpu(i8 %val) {
+; CHECK-LABEL: release_i8_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot771[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot771;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b8 %rs1, [release_i8_local_gpu_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b8 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i8 %val, ptr addrspace(5) %slot syncscope("device") release, align 1
+ ret void
+}
+
+define void @release_i8_local_sys_volatile(i8 %val) {
+; CHECK-LABEL: release_i8_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot772[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot772;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b8 %rs1, [release_i8_local_sys_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b8 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i8 %val, ptr addrspace(5) %slot syncscope("") release, align 1
+ ret void
+}
+
+define void @release_i8_local_cta_volatile(i8 %val) {
+; CHECK-LABEL: release_i8_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot773[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot773;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b8 %rs1, [release_i8_local_cta_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b8 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i8 %val, ptr addrspace(5) %slot syncscope("block") release, align 1
+ ret void
+}
+
+define void @release_i8_local_cluster_volatile(i8 %val) {
+; CHECK-LABEL: release_i8_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot774[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot774;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b8 %rs1, [release_i8_local_cluster_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b8 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i8 %val, ptr addrspace(5) %slot syncscope("cluster") release, align 1
+ ret void
+}
+
+define void @release_i8_local_gpu_volatile(i8 %val) {
+; CHECK-LABEL: release_i8_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot775[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot775;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b8 %rs1, [release_i8_local_gpu_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b8 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i8 %val, ptr addrspace(5) %slot syncscope("device") release, align 1
+ ret void
+}
+
+define void @release_i16_local_sys(i16 %val) {
+; CHECK-LABEL: release_i16_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot776[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot776;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [release_i16_local_sys_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i16 %val, ptr addrspace(5) %slot syncscope("") release, align 2
+ ret void
+}
+
+define void @release_i16_local_cta(i16 %val) {
+; CHECK-LABEL: release_i16_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot777[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot777;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [release_i16_local_cta_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i16 %val, ptr addrspace(5) %slot syncscope("block") release, align 2
+ ret void
+}
+
+define void @release_i16_local_cluster(i16 %val) {
+; CHECK-LABEL: release_i16_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot778[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot778;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [release_i16_local_cluster_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i16 %val, ptr addrspace(5) %slot syncscope("cluster") release, align 2
+ ret void
+}
+
+define void @release_i16_local_gpu(i16 %val) {
+; CHECK-LABEL: release_i16_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot779[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot779;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [release_i16_local_gpu_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i16 %val, ptr addrspace(5) %slot syncscope("device") release, align 2
+ ret void
+}
+
+define void @release_i16_local_sys_volatile(i16 %val) {
+; CHECK-LABEL: release_i16_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot780[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot780;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [release_i16_local_sys_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i16 %val, ptr addrspace(5) %slot syncscope("") release, align 2
+ ret void
+}
+
+define void @release_i16_local_cta_volatile(i16 %val) {
+; CHECK-LABEL: release_i16_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot781[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot781;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [release_i16_local_cta_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i16 %val, ptr addrspace(5) %slot syncscope("block") release, align 2
+ ret void
+}
+
+define void @release_i16_local_cluster_volatile(i16 %val) {
+; CHECK-LABEL: release_i16_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot782[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot782;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [release_i16_local_cluster_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i16 %val, ptr addrspace(5) %slot syncscope("cluster") release, align 2
+ ret void
+}
+
+define void @release_i16_local_gpu_volatile(i16 %val) {
+; CHECK-LABEL: release_i16_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot783[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot783;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [release_i16_local_gpu_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i16 %val, ptr addrspace(5) %slot syncscope("device") release, align 2
+ ret void
+}
+
+define void @release_i32_local_sys(i32 %val) {
+; CHECK-LABEL: release_i32_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot784[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot784;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [release_i32_local_sys_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i32 %val, ptr addrspace(5) %slot syncscope("") release, align 4
+ ret void
+}
+
+define void @release_i32_local_cta(i32 %val) {
+; CHECK-LABEL: release_i32_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot785[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot785;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [release_i32_local_cta_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i32 %val, ptr addrspace(5) %slot syncscope("block") release, align 4
+ ret void
+}
+
+define void @release_i32_local_cluster(i32 %val) {
+; CHECK-LABEL: release_i32_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot786[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot786;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [release_i32_local_cluster_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i32 %val, ptr addrspace(5) %slot syncscope("cluster") release, align 4
+ ret void
+}
+
+define void @release_i32_local_gpu(i32 %val) {
+; CHECK-LABEL: release_i32_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot787[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot787;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [release_i32_local_gpu_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i32 %val, ptr addrspace(5) %slot syncscope("device") release, align 4
+ ret void
+}
+
+define void @release_i32_local_sys_volatile(i32 %val) {
+; CHECK-LABEL: release_i32_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot788[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot788;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [release_i32_local_sys_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i32 %val, ptr addrspace(5) %slot syncscope("") release, align 4
+ ret void
+}
+
+define void @release_i32_local_cta_volatile(i32 %val) {
+; CHECK-LABEL: release_i32_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot789[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot789;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [release_i32_local_cta_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i32 %val, ptr addrspace(5) %slot syncscope("block") release, align 4
+ ret void
+}
+
+define void @release_i32_local_cluster_volatile(i32 %val) {
+; CHECK-LABEL: release_i32_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot790[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot790;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [release_i32_local_cluster_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i32 %val, ptr addrspace(5) %slot syncscope("cluster") release, align 4
+ ret void
+}
+
+define void @release_i32_local_gpu_volatile(i32 %val) {
+; CHECK-LABEL: release_i32_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot791[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot791;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [release_i32_local_gpu_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i32 %val, ptr addrspace(5) %slot syncscope("device") release, align 4
+ ret void
+}
+
+define void @release_i64_local_sys(i64 %val) {
+; CHECK-LABEL: release_i64_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot792[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot792;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i64_local_sys_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i64 %val, ptr addrspace(5) %slot syncscope("") release, align 8
+ ret void
+}
+
+define void @release_i64_local_cta(i64 %val) {
+; CHECK-LABEL: release_i64_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot793[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot793;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i64_local_cta_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i64 %val, ptr addrspace(5) %slot syncscope("block") release, align 8
+ ret void
+}
+
+define void @release_i64_local_cluster(i64 %val) {
+; CHECK-LABEL: release_i64_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot794[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot794;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i64_local_cluster_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i64 %val, ptr addrspace(5) %slot syncscope("cluster") release, align 8
+ ret void
+}
+
+define void @release_i64_local_gpu(i64 %val) {
+; CHECK-LABEL: release_i64_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot795[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot795;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i64_local_gpu_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i64 %val, ptr addrspace(5) %slot syncscope("device") release, align 8
+ ret void
+}
+
+define void @release_i64_local_sys_volatile(i64 %val) {
+; CHECK-LABEL: release_i64_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot796[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot796;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i64_local_sys_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i64 %val, ptr addrspace(5) %slot syncscope("") release, align 8
+ ret void
+}
+
+define void @release_i64_local_cta_volatile(i64 %val) {
+; CHECK-LABEL: release_i64_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot797[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot797;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i64_local_cta_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i64 %val, ptr addrspace(5) %slot syncscope("block") release, align 8
+ ret void
+}
+
+define void @release_i64_local_cluster_volatile(i64 %val) {
+; CHECK-LABEL: release_i64_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot798[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot798;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i64_local_cluster_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i64 %val, ptr addrspace(5) %slot syncscope("cluster") release, align 8
+ ret void
+}
+
+define void @release_i64_local_gpu_volatile(i64 %val) {
+; CHECK-LABEL: release_i64_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot799[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot799;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i64_local_gpu_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i64 %val, ptr addrspace(5) %slot syncscope("device") release, align 8
+ ret void
+}
+
+define void @release_half_local_sys(half %val) {
+; CHECK-LABEL: release_half_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot800[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot800;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [release_half_local_sys_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic half %val, ptr addrspace(5) %slot syncscope("") release, align 2
+ ret void
+}
+
+define void @release_half_local_cta(half %val) {
+; CHECK-LABEL: release_half_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot801[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot801;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [release_half_local_cta_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic half %val, ptr addrspace(5) %slot syncscope("block") release, align 2
+ ret void
+}
+
+define void @release_half_local_cluster(half %val) {
+; CHECK-LABEL: release_half_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot802[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot802;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [release_half_local_cluster_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic half %val, ptr addrspace(5) %slot syncscope("cluster") release, align 2
+ ret void
+}
+
+define void @release_half_local_gpu(half %val) {
+; CHECK-LABEL: release_half_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot803[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot803;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [release_half_local_gpu_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic half %val, ptr addrspace(5) %slot syncscope("device") release, align 2
+ ret void
+}
+
+define void @release_half_local_sys_volatile(half %val) {
+; CHECK-LABEL: release_half_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot804[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot804;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [release_half_local_sys_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile half %val, ptr addrspace(5) %slot syncscope("") release, align 2
+ ret void
+}
+
+define void @release_half_local_cta_volatile(half %val) {
+; CHECK-LABEL: release_half_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot805[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot805;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [release_half_local_cta_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile half %val, ptr addrspace(5) %slot syncscope("block") release, align 2
+ ret void
+}
+
+define void @release_half_local_cluster_volatile(half %val) {
+; CHECK-LABEL: release_half_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot806[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot806;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [release_half_local_cluster_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile half %val, ptr addrspace(5) %slot syncscope("cluster") release, align 2
+ ret void
+}
+
+define void @release_half_local_gpu_volatile(half %val) {
+; CHECK-LABEL: release_half_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot807[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot807;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [release_half_local_gpu_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile half %val, ptr addrspace(5) %slot syncscope("device") release, align 2
+ ret void
+}
+
+define void @release_bfloat_local_sys(bfloat %val) {
+; CHECK-LABEL: release_bfloat_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot808[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot808;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [release_bfloat_local_sys_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic bfloat %val, ptr addrspace(5) %slot syncscope("") release, align 2
+ ret void
+}
+
+define void @release_bfloat_local_cta(bfloat %val) {
+; CHECK-LABEL: release_bfloat_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot809[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot809;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [release_bfloat_local_cta_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic bfloat %val, ptr addrspace(5) %slot syncscope("block") release, align 2
+ ret void
+}
+
+define void @release_bfloat_local_cluster(bfloat %val) {
+; CHECK-LABEL: release_bfloat_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot810[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot810;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [release_bfloat_local_cluster_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic bfloat %val, ptr addrspace(5) %slot syncscope("cluster") release, align 2
+ ret void
+}
+
+define void @release_bfloat_local_gpu(bfloat %val) {
+; CHECK-LABEL: release_bfloat_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot811[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot811;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [release_bfloat_local_gpu_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic bfloat %val, ptr addrspace(5) %slot syncscope("device") release, align 2
+ ret void
+}
+
+define void @release_bfloat_local_sys_volatile(bfloat %val) {
+; CHECK-LABEL: release_bfloat_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot812[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot812;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [release_bfloat_local_sys_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile bfloat %val, ptr addrspace(5) %slot syncscope("") release, align 2
+ ret void
+}
+
+define void @release_bfloat_local_cta_volatile(bfloat %val) {
+; CHECK-LABEL: release_bfloat_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot813[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot813;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [release_bfloat_local_cta_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile bfloat %val, ptr addrspace(5) %slot syncscope("block") release, align 2
+ ret void
+}
+
+define void @release_bfloat_local_cluster_volatile(bfloat %val) {
+; CHECK-LABEL: release_bfloat_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot814[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot814;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [release_bfloat_local_cluster_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile bfloat %val, ptr addrspace(5) %slot syncscope("cluster") release, align 2
+ ret void
+}
+
+define void @release_bfloat_local_gpu_volatile(bfloat %val) {
+; CHECK-LABEL: release_bfloat_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot815[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot815;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [release_bfloat_local_gpu_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile bfloat %val, ptr addrspace(5) %slot syncscope("device") release, align 2
+ ret void
+}
+
+define void @release_float_local_sys(float %val) {
+; CHECK-LABEL: release_float_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot816[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot816;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [release_float_local_sys_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic float %val, ptr addrspace(5) %slot syncscope("") release, align 4
+ ret void
+}
+
+define void @release_float_local_cta(float %val) {
+; CHECK-LABEL: release_float_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot817[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot817;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [release_float_local_cta_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic float %val, ptr addrspace(5) %slot syncscope("block") release, align 4
+ ret void
+}
+
+define void @release_float_local_cluster(float %val) {
+; CHECK-LABEL: release_float_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot818[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot818;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [release_float_local_cluster_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic float %val, ptr addrspace(5) %slot syncscope("cluster") release, align 4
+ ret void
+}
+
+define void @release_float_local_gpu(float %val) {
+; CHECK-LABEL: release_float_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot819[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot819;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [release_float_local_gpu_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic float %val, ptr addrspace(5) %slot syncscope("device") release, align 4
+ ret void
+}
+
+define void @release_float_local_sys_volatile(float %val) {
+; CHECK-LABEL: release_float_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot820[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot820;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [release_float_local_sys_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile float %val, ptr addrspace(5) %slot syncscope("") release, align 4
+ ret void
+}
+
+define void @release_float_local_cta_volatile(float %val) {
+; CHECK-LABEL: release_float_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot821[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot821;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [release_float_local_cta_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile float %val, ptr addrspace(5) %slot syncscope("block") release, align 4
+ ret void
+}
+
+define void @release_float_local_cluster_volatile(float %val) {
+; CHECK-LABEL: release_float_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot822[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot822;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [release_float_local_cluster_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile float %val, ptr addrspace(5) %slot syncscope("cluster") release, align 4
+ ret void
+}
+
+define void @release_float_local_gpu_volatile(float %val) {
+; CHECK-LABEL: release_float_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot823[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot823;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [release_float_local_gpu_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile float %val, ptr addrspace(5) %slot syncscope("device") release, align 4
+ ret void
+}
+
+define void @release_double_local_sys(double %val) {
+; CHECK-LABEL: release_double_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot824[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot824;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_double_local_sys_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic double %val, ptr addrspace(5) %slot syncscope("") release, align 8
+ ret void
+}
+
+define void @release_double_local_cta(double %val) {
+; CHECK-LABEL: release_double_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot825[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot825;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_double_local_cta_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic double %val, ptr addrspace(5) %slot syncscope("block") release, align 8
+ ret void
+}
+
+define void @release_double_local_cluster(double %val) {
+; CHECK-LABEL: release_double_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot826[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot826;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_double_local_cluster_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic double %val, ptr addrspace(5) %slot syncscope("cluster") release, align 8
+ ret void
+}
+
+define void @release_double_local_gpu(double %val) {
+; CHECK-LABEL: release_double_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot827[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot827;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_double_local_gpu_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic double %val, ptr addrspace(5) %slot syncscope("device") release, align 8
+ ret void
+}
+
+define void @release_double_local_sys_volatile(double %val) {
+; CHECK-LABEL: release_double_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot828[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot828;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_double_local_sys_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile double %val, ptr addrspace(5) %slot syncscope("") release, align 8
+ ret void
+}
+
+define void @release_double_local_cta_volatile(double %val) {
+; CHECK-LABEL: release_double_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot829[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot829;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_double_local_cta_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile double %val, ptr addrspace(5) %slot syncscope("block") release, align 8
+ ret void
+}
+
+define void @release_double_local_cluster_volatile(double %val) {
+; CHECK-LABEL: release_double_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot830[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot830;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_double_local_cluster_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile double %val, ptr addrspace(5) %slot syncscope("cluster") release, align 8
+ ret void
+}
+
+define void @release_double_local_gpu_volatile(double %val) {
+; CHECK-LABEL: release_double_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot831[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot831;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_double_local_gpu_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile double %val, ptr addrspace(5) %slot syncscope("device") release, align 8
+ ret void
+}
+
+define void @seq_cst_i8_generic_sys(i8 %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_i8_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [seq_cst_i8_generic_sys_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_generic_sys_param_1];
+; CHECK-NEXT: st.volatile.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr %addr syncscope("") seq_cst, align 1
+ ret void
+}
+
+define void @seq_cst_i8_generic_cta(i8 %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_i8_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [seq_cst_i8_generic_cta_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_generic_cta_param_1];
+; CHECK-NEXT: st.volatile.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr %addr syncscope("block") seq_cst, align 1
+ ret void
+}
+
+define void @seq_cst_i8_generic_cluster(i8 %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_i8_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [seq_cst_i8_generic_cluster_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_generic_cluster_param_1];
+; CHECK-NEXT: st.volatile.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr %addr syncscope("cluster") seq_cst, align 1
+ ret void
+}
+
+define void @seq_cst_i8_generic_gpu(i8 %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_i8_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [seq_cst_i8_generic_gpu_param_0];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_generic_gpu_param_1];
+; CHECK-NEXT: st.volatile.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr %addr syncscope("device") seq_cst, align 1
+ ret void
+}
+
+define void @seq_cst_i8_generic_sys_volatile(i8 %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_i8_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [seq_cst_i8_generic_sys_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_generic_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr %addr syncscope("") seq_cst, align 1
+ ret void
+}
+
+define void @seq_cst_i8_generic_cta_volatile(i8 %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_i8_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [seq_cst_i8_generic_cta_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_generic_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr %addr syncscope("block") seq_cst, align 1
+ ret void
+}
+
+define void @seq_cst_i8_generic_cluster_volatile(i8 %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_i8_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [seq_cst_i8_generic_cluster_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_generic_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr %addr syncscope("cluster") seq_cst, align 1
+ ret void
+}
+
+define void @seq_cst_i8_generic_gpu_volatile(i8 %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_i8_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [seq_cst_i8_generic_gpu_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_generic_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr %addr syncscope("device") seq_cst, align 1
+ ret void
+}
+
+define void @seq_cst_i16_generic_sys(i16 %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_i16_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_i16_generic_sys_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_generic_sys_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr %addr syncscope("") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_i16_generic_cta(i16 %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_i16_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_i16_generic_cta_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_generic_cta_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr %addr syncscope("block") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_i16_generic_cluster(i16 %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_i16_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_i16_generic_cluster_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_generic_cluster_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr %addr syncscope("cluster") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_i16_generic_gpu(i16 %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_i16_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_i16_generic_gpu_param_0];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_generic_gpu_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr %addr syncscope("device") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_i16_generic_sys_volatile(i16 %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_i16_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_i16_generic_sys_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_generic_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr %addr syncscope("") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_i16_generic_cta_volatile(i16 %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_i16_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_i16_generic_cta_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_generic_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr %addr syncscope("block") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_i16_generic_cluster_volatile(i16 %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_i16_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_i16_generic_cluster_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_generic_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr %addr syncscope("cluster") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_i16_generic_gpu_volatile(i16 %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_i16_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_i16_generic_gpu_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_generic_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr %addr syncscope("device") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_i32_generic_sys(i32 %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_i32_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_i32_generic_sys_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_generic_sys_param_1];
+; CHECK-NEXT: st.volatile.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr %addr syncscope("") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_i32_generic_cta(i32 %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_i32_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_i32_generic_cta_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_generic_cta_param_1];
+; CHECK-NEXT: st.volatile.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr %addr syncscope("block") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_i32_generic_cluster(i32 %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_i32_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_i32_generic_cluster_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_generic_cluster_param_1];
+; CHECK-NEXT: st.volatile.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr %addr syncscope("cluster") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_i32_generic_gpu(i32 %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_i32_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_i32_generic_gpu_param_0];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_generic_gpu_param_1];
+; CHECK-NEXT: st.volatile.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr %addr syncscope("device") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_i32_generic_sys_volatile(i32 %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_i32_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_i32_generic_sys_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_generic_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr %addr syncscope("") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_i32_generic_cta_volatile(i32 %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_i32_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_i32_generic_cta_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_generic_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr %addr syncscope("block") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_i32_generic_cluster_volatile(i32 %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_i32_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_i32_generic_cluster_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_generic_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr %addr syncscope("cluster") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_i32_generic_gpu_volatile(i32 %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_i32_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_i32_generic_gpu_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_generic_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr %addr syncscope("device") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_i64_generic_sys(i64 %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_i64_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_generic_sys_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_i64_generic_sys_param_1];
+; CHECK-NEXT: st.volatile.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr %addr syncscope("") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_i64_generic_cta(i64 %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_i64_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_generic_cta_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_i64_generic_cta_param_1];
+; CHECK-NEXT: st.volatile.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr %addr syncscope("block") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_i64_generic_cluster(i64 %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_i64_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_generic_cluster_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_i64_generic_cluster_param_1];
+; CHECK-NEXT: st.volatile.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr %addr syncscope("cluster") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_i64_generic_gpu(i64 %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_i64_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_generic_gpu_param_0];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_i64_generic_gpu_param_1];
+; CHECK-NEXT: st.volatile.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr %addr syncscope("device") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_i64_generic_sys_volatile(i64 %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_i64_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_generic_sys_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_i64_generic_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr %addr syncscope("") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_i64_generic_cta_volatile(i64 %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_i64_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_generic_cta_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_i64_generic_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr %addr syncscope("block") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_i64_generic_cluster_volatile(i64 %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_i64_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_generic_cluster_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_i64_generic_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr %addr syncscope("cluster") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_i64_generic_gpu_volatile(i64 %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_i64_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_generic_gpu_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_i64_generic_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr %addr syncscope("device") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_half_generic_sys(half %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_half_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_half_generic_sys_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_generic_sys_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr %addr syncscope("") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_half_generic_cta(half %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_half_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_half_generic_cta_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_generic_cta_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr %addr syncscope("block") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_half_generic_cluster(half %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_half_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_half_generic_cluster_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_generic_cluster_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr %addr syncscope("cluster") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_half_generic_gpu(half %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_half_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_half_generic_gpu_param_0];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_generic_gpu_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr %addr syncscope("device") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_half_generic_sys_volatile(half %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_half_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_half_generic_sys_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_generic_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr %addr syncscope("") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_half_generic_cta_volatile(half %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_half_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_half_generic_cta_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_generic_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr %addr syncscope("block") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_half_generic_cluster_volatile(half %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_half_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_half_generic_cluster_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_generic_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr %addr syncscope("cluster") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_half_generic_gpu_volatile(half %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_half_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_half_generic_gpu_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_generic_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr %addr syncscope("device") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_bfloat_generic_sys(bfloat %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_bfloat_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_bfloat_generic_sys_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_generic_sys_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr %addr syncscope("") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_bfloat_generic_cta(bfloat %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_bfloat_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_bfloat_generic_cta_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_generic_cta_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr %addr syncscope("block") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_bfloat_generic_cluster(bfloat %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_bfloat_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_bfloat_generic_cluster_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_generic_cluster_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr %addr syncscope("cluster") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_bfloat_generic_gpu(bfloat %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_bfloat_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_bfloat_generic_gpu_param_0];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_generic_gpu_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr %addr syncscope("device") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_bfloat_generic_sys_volatile(bfloat %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_bfloat_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_bfloat_generic_sys_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_generic_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr %addr syncscope("") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_bfloat_generic_cta_volatile(bfloat %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_bfloat_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_bfloat_generic_cta_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_generic_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr %addr syncscope("block") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_bfloat_generic_cluster_volatile(bfloat %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_bfloat_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_bfloat_generic_cluster_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_generic_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr %addr syncscope("cluster") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_bfloat_generic_gpu_volatile(bfloat %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_bfloat_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_bfloat_generic_gpu_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_generic_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr %addr syncscope("device") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_float_generic_sys(float %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_float_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_float_generic_sys_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_generic_sys_param_1];
+; CHECK-NEXT: st.volatile.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr %addr syncscope("") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_float_generic_cta(float %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_float_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_float_generic_cta_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_generic_cta_param_1];
+; CHECK-NEXT: st.volatile.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr %addr syncscope("block") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_float_generic_cluster(float %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_float_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_float_generic_cluster_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_generic_cluster_param_1];
+; CHECK-NEXT: st.volatile.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr %addr syncscope("cluster") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_float_generic_gpu(float %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_float_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_float_generic_gpu_param_0];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_generic_gpu_param_1];
+; CHECK-NEXT: st.volatile.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr %addr syncscope("device") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_float_generic_sys_volatile(float %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_float_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_float_generic_sys_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_generic_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr %addr syncscope("") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_float_generic_cta_volatile(float %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_float_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_float_generic_cta_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_generic_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr %addr syncscope("block") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_float_generic_cluster_volatile(float %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_float_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_float_generic_cluster_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_generic_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr %addr syncscope("cluster") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_float_generic_gpu_volatile(float %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_float_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_float_generic_gpu_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_generic_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr %addr syncscope("device") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_double_generic_sys(double %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_double_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_generic_sys_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_double_generic_sys_param_1];
+; CHECK-NEXT: st.volatile.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr %addr syncscope("") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_double_generic_cta(double %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_double_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_generic_cta_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_double_generic_cta_param_1];
+; CHECK-NEXT: st.volatile.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr %addr syncscope("block") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_double_generic_cluster(double %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_double_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_generic_cluster_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_double_generic_cluster_param_1];
+; CHECK-NEXT: st.volatile.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr %addr syncscope("cluster") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_double_generic_gpu(double %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_double_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_generic_gpu_param_0];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_double_generic_gpu_param_1];
+; CHECK-NEXT: st.volatile.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr %addr syncscope("device") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_double_generic_sys_volatile(double %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_double_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_generic_sys_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_double_generic_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr %addr syncscope("") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_double_generic_cta_volatile(double %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_double_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_generic_cta_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_double_generic_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr %addr syncscope("block") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_double_generic_cluster_volatile(double %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_double_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_generic_cluster_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_double_generic_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr %addr syncscope("cluster") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_double_generic_gpu_volatile(double %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_double_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_generic_gpu_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_double_generic_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr %addr syncscope("device") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_i8_global_sys(i8 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i8_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [seq_cst_i8_global_sys_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_global_sys_param_1];
+; CHECK-NEXT: st.volatile.global.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr addrspace(1) %addr syncscope("") seq_cst, align 1
+ ret void
+}
+
+define void @seq_cst_i8_global_cta(i8 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i8_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [seq_cst_i8_global_cta_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_global_cta_param_1];
+; CHECK-NEXT: st.volatile.global.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr addrspace(1) %addr syncscope("block") seq_cst, align 1
+ ret void
+}
+
+define void @seq_cst_i8_global_cluster(i8 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i8_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [seq_cst_i8_global_cluster_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_global_cluster_param_1];
+; CHECK-NEXT: st.volatile.global.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr addrspace(1) %addr syncscope("cluster") seq_cst, align 1
+ ret void
+}
+
+define void @seq_cst_i8_global_gpu(i8 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i8_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [seq_cst_i8_global_gpu_param_0];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_global_gpu_param_1];
+; CHECK-NEXT: st.volatile.global.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr addrspace(1) %addr syncscope("device") seq_cst, align 1
+ ret void
+}
+
+define void @seq_cst_i8_global_sys_volatile(i8 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i8_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [seq_cst_i8_global_sys_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_global_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr addrspace(1) %addr syncscope("") seq_cst, align 1
+ ret void
+}
+
+define void @seq_cst_i8_global_cta_volatile(i8 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i8_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [seq_cst_i8_global_cta_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_global_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr addrspace(1) %addr syncscope("block") seq_cst, align 1
+ ret void
+}
+
+define void @seq_cst_i8_global_cluster_volatile(i8 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i8_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [seq_cst_i8_global_cluster_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_global_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr addrspace(1) %addr syncscope("cluster") seq_cst, align 1
+ ret void
+}
+
+define void @seq_cst_i8_global_gpu_volatile(i8 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i8_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [seq_cst_i8_global_gpu_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_global_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr addrspace(1) %addr syncscope("device") seq_cst, align 1
+ ret void
+}
+
+define void @seq_cst_i16_global_sys(i16 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i16_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_i16_global_sys_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_global_sys_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr addrspace(1) %addr syncscope("") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_i16_global_cta(i16 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i16_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_i16_global_cta_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_global_cta_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr addrspace(1) %addr syncscope("block") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_i16_global_cluster(i16 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i16_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_i16_global_cluster_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_global_cluster_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr addrspace(1) %addr syncscope("cluster") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_i16_global_gpu(i16 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i16_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_i16_global_gpu_param_0];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_global_gpu_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr addrspace(1) %addr syncscope("device") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_i16_global_sys_volatile(i16 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i16_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_i16_global_sys_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_global_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr addrspace(1) %addr syncscope("") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_i16_global_cta_volatile(i16 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i16_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_i16_global_cta_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_global_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr addrspace(1) %addr syncscope("block") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_i16_global_cluster_volatile(i16 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i16_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_i16_global_cluster_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_global_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr addrspace(1) %addr syncscope("cluster") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_i16_global_gpu_volatile(i16 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i16_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_i16_global_gpu_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_global_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr addrspace(1) %addr syncscope("device") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_i32_global_sys(i32 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i32_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_i32_global_sys_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_global_sys_param_1];
+; CHECK-NEXT: st.volatile.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr addrspace(1) %addr syncscope("") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_i32_global_cta(i32 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i32_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_i32_global_cta_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_global_cta_param_1];
+; CHECK-NEXT: st.volatile.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr addrspace(1) %addr syncscope("block") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_i32_global_cluster(i32 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i32_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_i32_global_cluster_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_global_cluster_param_1];
+; CHECK-NEXT: st.volatile.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr addrspace(1) %addr syncscope("cluster") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_i32_global_gpu(i32 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i32_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_i32_global_gpu_param_0];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_global_gpu_param_1];
+; CHECK-NEXT: st.volatile.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr addrspace(1) %addr syncscope("device") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_i32_global_sys_volatile(i32 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i32_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_i32_global_sys_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_global_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr addrspace(1) %addr syncscope("") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_i32_global_cta_volatile(i32 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i32_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_i32_global_cta_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_global_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr addrspace(1) %addr syncscope("block") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_i32_global_cluster_volatile(i32 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i32_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_i32_global_cluster_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_global_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr addrspace(1) %addr syncscope("cluster") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_i32_global_gpu_volatile(i32 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i32_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_i32_global_gpu_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_global_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr addrspace(1) %addr syncscope("device") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_i64_global_sys(i64 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i64_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_global_sys_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_i64_global_sys_param_1];
+; CHECK-NEXT: st.volatile.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr addrspace(1) %addr syncscope("") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_i64_global_cta(i64 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i64_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_global_cta_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_i64_global_cta_param_1];
+; CHECK-NEXT: st.volatile.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr addrspace(1) %addr syncscope("block") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_i64_global_cluster(i64 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i64_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_global_cluster_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_i64_global_cluster_param_1];
+; CHECK-NEXT: st.volatile.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr addrspace(1) %addr syncscope("cluster") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_i64_global_gpu(i64 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i64_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_global_gpu_param_0];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_i64_global_gpu_param_1];
+; CHECK-NEXT: st.volatile.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr addrspace(1) %addr syncscope("device") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_i64_global_sys_volatile(i64 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i64_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_global_sys_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_i64_global_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr addrspace(1) %addr syncscope("") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_i64_global_cta_volatile(i64 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i64_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_global_cta_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_i64_global_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr addrspace(1) %addr syncscope("block") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_i64_global_cluster_volatile(i64 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i64_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_global_cluster_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_i64_global_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr addrspace(1) %addr syncscope("cluster") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_i64_global_gpu_volatile(i64 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i64_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_global_gpu_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_i64_global_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr addrspace(1) %addr syncscope("device") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_half_global_sys(half %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_half_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_half_global_sys_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_global_sys_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr addrspace(1) %addr syncscope("") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_half_global_cta(half %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_half_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_half_global_cta_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_global_cta_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr addrspace(1) %addr syncscope("block") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_half_global_cluster(half %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_half_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_half_global_cluster_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_global_cluster_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr addrspace(1) %addr syncscope("cluster") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_half_global_gpu(half %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_half_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_half_global_gpu_param_0];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_global_gpu_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr addrspace(1) %addr syncscope("device") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_half_global_sys_volatile(half %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_half_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_half_global_sys_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_global_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr addrspace(1) %addr syncscope("") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_half_global_cta_volatile(half %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_half_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_half_global_cta_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_global_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr addrspace(1) %addr syncscope("block") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_half_global_cluster_volatile(half %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_half_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_half_global_cluster_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_global_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr addrspace(1) %addr syncscope("cluster") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_half_global_gpu_volatile(half %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_half_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_half_global_gpu_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_global_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr addrspace(1) %addr syncscope("device") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_bfloat_global_sys(bfloat %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_bfloat_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_bfloat_global_sys_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_global_sys_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr addrspace(1) %addr syncscope("") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_bfloat_global_cta(bfloat %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_bfloat_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_bfloat_global_cta_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_global_cta_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr addrspace(1) %addr syncscope("block") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_bfloat_global_cluster(bfloat %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_bfloat_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_bfloat_global_cluster_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_global_cluster_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr addrspace(1) %addr syncscope("cluster") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_bfloat_global_gpu(bfloat %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_bfloat_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_bfloat_global_gpu_param_0];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_global_gpu_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr addrspace(1) %addr syncscope("device") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_bfloat_global_sys_volatile(bfloat %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_bfloat_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_bfloat_global_sys_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_global_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr addrspace(1) %addr syncscope("") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_bfloat_global_cta_volatile(bfloat %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_bfloat_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_bfloat_global_cta_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_global_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr addrspace(1) %addr syncscope("block") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_bfloat_global_cluster_volatile(bfloat %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_bfloat_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_bfloat_global_cluster_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_global_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr addrspace(1) %addr syncscope("cluster") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_bfloat_global_gpu_volatile(bfloat %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_bfloat_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_bfloat_global_gpu_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_global_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr addrspace(1) %addr syncscope("device") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_float_global_sys(float %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_float_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_float_global_sys_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_global_sys_param_1];
+; CHECK-NEXT: st.volatile.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr addrspace(1) %addr syncscope("") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_float_global_cta(float %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_float_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_float_global_cta_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_global_cta_param_1];
+; CHECK-NEXT: st.volatile.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr addrspace(1) %addr syncscope("block") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_float_global_cluster(float %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_float_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_float_global_cluster_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_global_cluster_param_1];
+; CHECK-NEXT: st.volatile.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr addrspace(1) %addr syncscope("cluster") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_float_global_gpu(float %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_float_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_float_global_gpu_param_0];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_global_gpu_param_1];
+; CHECK-NEXT: st.volatile.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr addrspace(1) %addr syncscope("device") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_float_global_sys_volatile(float %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_float_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_float_global_sys_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_global_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr addrspace(1) %addr syncscope("") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_float_global_cta_volatile(float %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_float_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_float_global_cta_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_global_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr addrspace(1) %addr syncscope("block") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_float_global_cluster_volatile(float %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_float_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_float_global_cluster_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_global_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr addrspace(1) %addr syncscope("cluster") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_float_global_gpu_volatile(float %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_float_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_float_global_gpu_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_global_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr addrspace(1) %addr syncscope("device") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_double_global_sys(double %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_double_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_global_sys_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_double_global_sys_param_1];
+; CHECK-NEXT: st.volatile.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr addrspace(1) %addr syncscope("") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_double_global_cta(double %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_double_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_global_cta_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_double_global_cta_param_1];
+; CHECK-NEXT: st.volatile.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr addrspace(1) %addr syncscope("block") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_double_global_cluster(double %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_double_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_global_cluster_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_double_global_cluster_param_1];
+; CHECK-NEXT: st.volatile.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr addrspace(1) %addr syncscope("cluster") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_double_global_gpu(double %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_double_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_global_gpu_param_0];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_double_global_gpu_param_1];
+; CHECK-NEXT: st.volatile.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr addrspace(1) %addr syncscope("device") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_double_global_sys_volatile(double %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_double_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_global_sys_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_double_global_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr addrspace(1) %addr syncscope("") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_double_global_cta_volatile(double %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_double_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_global_cta_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_double_global_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr addrspace(1) %addr syncscope("block") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_double_global_cluster_volatile(double %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_double_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_global_cluster_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_double_global_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr addrspace(1) %addr syncscope("cluster") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_double_global_gpu_volatile(double %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_double_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_global_gpu_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_double_global_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr addrspace(1) %addr syncscope("device") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_i8_shared_sys(i8 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i8_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [seq_cst_i8_shared_sys_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_shared_sys_param_1];
+; CHECK-NEXT: st.volatile.shared.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr addrspace(3) %addr syncscope("") seq_cst, align 1
+ ret void
+}
+
+define void @seq_cst_i8_shared_cta(i8 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i8_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [seq_cst_i8_shared_cta_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_shared_cta_param_1];
+; CHECK-NEXT: st.volatile.shared.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr addrspace(3) %addr syncscope("block") seq_cst, align 1
+ ret void
+}
+
+define void @seq_cst_i8_shared_cluster(i8 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i8_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [seq_cst_i8_shared_cluster_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_shared_cluster_param_1];
+; CHECK-NEXT: st.volatile.shared.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr addrspace(3) %addr syncscope("cluster") seq_cst, align 1
+ ret void
+}
+
+define void @seq_cst_i8_shared_gpu(i8 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i8_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [seq_cst_i8_shared_gpu_param_0];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_shared_gpu_param_1];
+; CHECK-NEXT: st.volatile.shared.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr addrspace(3) %addr syncscope("device") seq_cst, align 1
+ ret void
+}
+
+define void @seq_cst_i8_shared_sys_volatile(i8 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i8_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [seq_cst_i8_shared_sys_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_shared_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr addrspace(3) %addr syncscope("") seq_cst, align 1
+ ret void
+}
+
+define void @seq_cst_i8_shared_cta_volatile(i8 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i8_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [seq_cst_i8_shared_cta_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_shared_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr addrspace(3) %addr syncscope("block") seq_cst, align 1
+ ret void
+}
+
+define void @seq_cst_i8_shared_cluster_volatile(i8 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i8_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [seq_cst_i8_shared_cluster_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_shared_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr addrspace(3) %addr syncscope("cluster") seq_cst, align 1
+ ret void
+}
+
+define void @seq_cst_i8_shared_gpu_volatile(i8 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i8_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [seq_cst_i8_shared_gpu_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_shared_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr addrspace(3) %addr syncscope("device") seq_cst, align 1
+ ret void
+}
+
+define void @seq_cst_i16_shared_sys(i16 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i16_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_i16_shared_sys_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_shared_sys_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr addrspace(3) %addr syncscope("") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_i16_shared_cta(i16 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i16_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_i16_shared_cta_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_shared_cta_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr addrspace(3) %addr syncscope("block") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_i16_shared_cluster(i16 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i16_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_i16_shared_cluster_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_shared_cluster_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr addrspace(3) %addr syncscope("cluster") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_i16_shared_gpu(i16 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i16_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_i16_shared_gpu_param_0];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_shared_gpu_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr addrspace(3) %addr syncscope("device") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_i16_shared_sys_volatile(i16 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i16_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_i16_shared_sys_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_shared_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr addrspace(3) %addr syncscope("") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_i16_shared_cta_volatile(i16 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i16_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_i16_shared_cta_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_shared_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr addrspace(3) %addr syncscope("block") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_i16_shared_cluster_volatile(i16 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i16_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_i16_shared_cluster_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_shared_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr addrspace(3) %addr syncscope("cluster") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_i16_shared_gpu_volatile(i16 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i16_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_i16_shared_gpu_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_shared_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr addrspace(3) %addr syncscope("device") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_i32_shared_sys(i32 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i32_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_i32_shared_sys_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_shared_sys_param_1];
+; CHECK-NEXT: st.volatile.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr addrspace(3) %addr syncscope("") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_i32_shared_cta(i32 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i32_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_i32_shared_cta_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_shared_cta_param_1];
+; CHECK-NEXT: st.volatile.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr addrspace(3) %addr syncscope("block") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_i32_shared_cluster(i32 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i32_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_i32_shared_cluster_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_shared_cluster_param_1];
+; CHECK-NEXT: st.volatile.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr addrspace(3) %addr syncscope("cluster") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_i32_shared_gpu(i32 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i32_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_i32_shared_gpu_param_0];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_shared_gpu_param_1];
+; CHECK-NEXT: st.volatile.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr addrspace(3) %addr syncscope("device") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_i32_shared_sys_volatile(i32 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i32_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_i32_shared_sys_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_shared_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr addrspace(3) %addr syncscope("") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_i32_shared_cta_volatile(i32 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i32_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_i32_shared_cta_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_shared_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr addrspace(3) %addr syncscope("block") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_i32_shared_cluster_volatile(i32 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i32_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_i32_shared_cluster_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_shared_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr addrspace(3) %addr syncscope("cluster") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_i32_shared_gpu_volatile(i32 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i32_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_i32_shared_gpu_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_shared_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr addrspace(3) %addr syncscope("device") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_i64_shared_sys(i64 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i64_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_shared_sys_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_i64_shared_sys_param_1];
+; CHECK-NEXT: st.volatile.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr addrspace(3) %addr syncscope("") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_i64_shared_cta(i64 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i64_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_shared_cta_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_i64_shared_cta_param_1];
+; CHECK-NEXT: st.volatile.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr addrspace(3) %addr syncscope("block") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_i64_shared_cluster(i64 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i64_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_shared_cluster_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_i64_shared_cluster_param_1];
+; CHECK-NEXT: st.volatile.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr addrspace(3) %addr syncscope("cluster") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_i64_shared_gpu(i64 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i64_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_shared_gpu_param_0];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_i64_shared_gpu_param_1];
+; CHECK-NEXT: st.volatile.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr addrspace(3) %addr syncscope("device") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_i64_shared_sys_volatile(i64 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i64_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_shared_sys_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_i64_shared_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr addrspace(3) %addr syncscope("") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_i64_shared_cta_volatile(i64 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i64_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_shared_cta_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_i64_shared_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr addrspace(3) %addr syncscope("block") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_i64_shared_cluster_volatile(i64 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i64_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_shared_cluster_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_i64_shared_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr addrspace(3) %addr syncscope("cluster") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_i64_shared_gpu_volatile(i64 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i64_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_shared_gpu_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_i64_shared_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr addrspace(3) %addr syncscope("device") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_half_shared_sys(half %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_half_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_half_shared_sys_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_shared_sys_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr addrspace(3) %addr syncscope("") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_half_shared_cta(half %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_half_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_half_shared_cta_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_shared_cta_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr addrspace(3) %addr syncscope("block") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_half_shared_cluster(half %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_half_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_half_shared_cluster_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_shared_cluster_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr addrspace(3) %addr syncscope("cluster") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_half_shared_gpu(half %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_half_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_half_shared_gpu_param_0];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_shared_gpu_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr addrspace(3) %addr syncscope("device") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_half_shared_sys_volatile(half %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_half_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_half_shared_sys_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_shared_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr addrspace(3) %addr syncscope("") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_half_shared_cta_volatile(half %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_half_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_half_shared_cta_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_shared_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr addrspace(3) %addr syncscope("block") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_half_shared_cluster_volatile(half %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_half_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_half_shared_cluster_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_shared_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr addrspace(3) %addr syncscope("cluster") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_half_shared_gpu_volatile(half %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_half_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_half_shared_gpu_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_shared_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr addrspace(3) %addr syncscope("device") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_bfloat_shared_sys(bfloat %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_bfloat_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_bfloat_shared_sys_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_shared_sys_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr addrspace(3) %addr syncscope("") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_bfloat_shared_cta(bfloat %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_bfloat_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_bfloat_shared_cta_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_shared_cta_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr addrspace(3) %addr syncscope("block") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_bfloat_shared_cluster(bfloat %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_bfloat_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_bfloat_shared_cluster_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_shared_cluster_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr addrspace(3) %addr syncscope("cluster") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_bfloat_shared_gpu(bfloat %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_bfloat_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_bfloat_shared_gpu_param_0];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_shared_gpu_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr addrspace(3) %addr syncscope("device") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_bfloat_shared_sys_volatile(bfloat %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_bfloat_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_bfloat_shared_sys_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_shared_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr addrspace(3) %addr syncscope("") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_bfloat_shared_cta_volatile(bfloat %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_bfloat_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_bfloat_shared_cta_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_shared_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr addrspace(3) %addr syncscope("block") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_bfloat_shared_cluster_volatile(bfloat %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_bfloat_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_bfloat_shared_cluster_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_shared_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr addrspace(3) %addr syncscope("cluster") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_bfloat_shared_gpu_volatile(bfloat %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_bfloat_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_bfloat_shared_gpu_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_shared_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr addrspace(3) %addr syncscope("device") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_float_shared_sys(float %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_float_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_float_shared_sys_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_shared_sys_param_1];
+; CHECK-NEXT: st.volatile.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr addrspace(3) %addr syncscope("") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_float_shared_cta(float %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_float_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_float_shared_cta_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_shared_cta_param_1];
+; CHECK-NEXT: st.volatile.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr addrspace(3) %addr syncscope("block") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_float_shared_cluster(float %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_float_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_float_shared_cluster_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_shared_cluster_param_1];
+; CHECK-NEXT: st.volatile.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr addrspace(3) %addr syncscope("cluster") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_float_shared_gpu(float %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_float_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_float_shared_gpu_param_0];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_shared_gpu_param_1];
+; CHECK-NEXT: st.volatile.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr addrspace(3) %addr syncscope("device") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_float_shared_sys_volatile(float %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_float_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_float_shared_sys_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_shared_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr addrspace(3) %addr syncscope("") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_float_shared_cta_volatile(float %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_float_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_float_shared_cta_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_shared_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr addrspace(3) %addr syncscope("block") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_float_shared_cluster_volatile(float %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_float_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_float_shared_cluster_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_shared_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr addrspace(3) %addr syncscope("cluster") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_float_shared_gpu_volatile(float %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_float_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_float_shared_gpu_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_shared_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr addrspace(3) %addr syncscope("device") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_double_shared_sys(double %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_double_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_shared_sys_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_double_shared_sys_param_1];
+; CHECK-NEXT: st.volatile.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr addrspace(3) %addr syncscope("") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_double_shared_cta(double %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_double_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_shared_cta_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_double_shared_cta_param_1];
+; CHECK-NEXT: st.volatile.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr addrspace(3) %addr syncscope("block") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_double_shared_cluster(double %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_double_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_shared_cluster_param_0];
+; CHECK-NEXT: membar.cta;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_double_shared_cluster_param_1];
+; CHECK-NEXT: st.volatile.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr addrspace(3) %addr syncscope("cluster") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_double_shared_gpu(double %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_double_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_shared_gpu_param_0];
+; CHECK-NEXT: membar.gl;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_double_shared_gpu_param_1];
+; CHECK-NEXT: st.volatile.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr addrspace(3) %addr syncscope("device") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_double_shared_sys_volatile(double %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_double_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_shared_sys_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_double_shared_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr addrspace(3) %addr syncscope("") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_double_shared_cta_volatile(double %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_double_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_shared_cta_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_double_shared_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr addrspace(3) %addr syncscope("block") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_double_shared_cluster_volatile(double %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_double_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_shared_cluster_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_double_shared_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr addrspace(3) %addr syncscope("cluster") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_double_shared_gpu_volatile(double %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_double_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_shared_gpu_volatile_param_0];
+; CHECK-NEXT: membar.sys;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_double_shared_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr addrspace(3) %addr syncscope("device") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_i8_local_sys(i8 %val) {
+; CHECK-LABEL: seq_cst_i8_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot1024[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1024;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b8 %rs1, [seq_cst_i8_local_sys_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b8 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i8 %val, ptr addrspace(5) %slot syncscope("") seq_cst, align 1
+ ret void
+}
+
+define void @seq_cst_i8_local_cta(i8 %val) {
+; CHECK-LABEL: seq_cst_i8_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot1025[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1025;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b8 %rs1, [seq_cst_i8_local_cta_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b8 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i8 %val, ptr addrspace(5) %slot syncscope("block") seq_cst, align 1
+ ret void
+}
+
+define void @seq_cst_i8_local_cluster(i8 %val) {
+; CHECK-LABEL: seq_cst_i8_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot1026[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1026;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b8 %rs1, [seq_cst_i8_local_cluster_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b8 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i8 %val, ptr addrspace(5) %slot syncscope("cluster") seq_cst, align 1
+ ret void
+}
+
+define void @seq_cst_i8_local_gpu(i8 %val) {
+; CHECK-LABEL: seq_cst_i8_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot1027[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1027;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b8 %rs1, [seq_cst_i8_local_gpu_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b8 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i8 %val, ptr addrspace(5) %slot syncscope("device") seq_cst, align 1
+ ret void
+}
+
+define void @seq_cst_i8_local_sys_volatile(i8 %val) {
+; CHECK-LABEL: seq_cst_i8_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot1028[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1028;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b8 %rs1, [seq_cst_i8_local_sys_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b8 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i8 %val, ptr addrspace(5) %slot syncscope("") seq_cst, align 1
+ ret void
+}
+
+define void @seq_cst_i8_local_cta_volatile(i8 %val) {
+; CHECK-LABEL: seq_cst_i8_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot1029[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1029;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b8 %rs1, [seq_cst_i8_local_cta_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b8 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i8 %val, ptr addrspace(5) %slot syncscope("block") seq_cst, align 1
+ ret void
+}
+
+define void @seq_cst_i8_local_cluster_volatile(i8 %val) {
+; CHECK-LABEL: seq_cst_i8_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot1030[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1030;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b8 %rs1, [seq_cst_i8_local_cluster_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b8 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i8 %val, ptr addrspace(5) %slot syncscope("cluster") seq_cst, align 1
+ ret void
+}
+
+define void @seq_cst_i8_local_gpu_volatile(i8 %val) {
+; CHECK-LABEL: seq_cst_i8_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot1031[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1031;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b8 %rs1, [seq_cst_i8_local_gpu_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b8 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i8 %val, ptr addrspace(5) %slot syncscope("device") seq_cst, align 1
+ ret void
+}
+
+define void @seq_cst_i16_local_sys(i16 %val) {
+; CHECK-LABEL: seq_cst_i16_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1032[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1032;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_i16_local_sys_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i16 %val, ptr addrspace(5) %slot syncscope("") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_i16_local_cta(i16 %val) {
+; CHECK-LABEL: seq_cst_i16_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1033[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1033;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_i16_local_cta_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i16 %val, ptr addrspace(5) %slot syncscope("block") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_i16_local_cluster(i16 %val) {
+; CHECK-LABEL: seq_cst_i16_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1034[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1034;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_i16_local_cluster_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i16 %val, ptr addrspace(5) %slot syncscope("cluster") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_i16_local_gpu(i16 %val) {
+; CHECK-LABEL: seq_cst_i16_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1035[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1035;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_i16_local_gpu_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i16 %val, ptr addrspace(5) %slot syncscope("device") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_i16_local_sys_volatile(i16 %val) {
+; CHECK-LABEL: seq_cst_i16_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1036[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1036;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_i16_local_sys_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i16 %val, ptr addrspace(5) %slot syncscope("") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_i16_local_cta_volatile(i16 %val) {
+; CHECK-LABEL: seq_cst_i16_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1037[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1037;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_i16_local_cta_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i16 %val, ptr addrspace(5) %slot syncscope("block") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_i16_local_cluster_volatile(i16 %val) {
+; CHECK-LABEL: seq_cst_i16_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1038[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1038;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_i16_local_cluster_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i16 %val, ptr addrspace(5) %slot syncscope("cluster") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_i16_local_gpu_volatile(i16 %val) {
+; CHECK-LABEL: seq_cst_i16_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1039[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1039;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_i16_local_gpu_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i16 %val, ptr addrspace(5) %slot syncscope("device") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_i32_local_sys(i32 %val) {
+; CHECK-LABEL: seq_cst_i32_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot1040[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1040;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [seq_cst_i32_local_sys_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i32 %val, ptr addrspace(5) %slot syncscope("") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_i32_local_cta(i32 %val) {
+; CHECK-LABEL: seq_cst_i32_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot1041[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1041;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [seq_cst_i32_local_cta_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i32 %val, ptr addrspace(5) %slot syncscope("block") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_i32_local_cluster(i32 %val) {
+; CHECK-LABEL: seq_cst_i32_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot1042[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1042;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [seq_cst_i32_local_cluster_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i32 %val, ptr addrspace(5) %slot syncscope("cluster") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_i32_local_gpu(i32 %val) {
+; CHECK-LABEL: seq_cst_i32_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot1043[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1043;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [seq_cst_i32_local_gpu_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i32 %val, ptr addrspace(5) %slot syncscope("device") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_i32_local_sys_volatile(i32 %val) {
+; CHECK-LABEL: seq_cst_i32_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot1044[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1044;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [seq_cst_i32_local_sys_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i32 %val, ptr addrspace(5) %slot syncscope("") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_i32_local_cta_volatile(i32 %val) {
+; CHECK-LABEL: seq_cst_i32_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot1045[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1045;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [seq_cst_i32_local_cta_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i32 %val, ptr addrspace(5) %slot syncscope("block") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_i32_local_cluster_volatile(i32 %val) {
+; CHECK-LABEL: seq_cst_i32_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot1046[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1046;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [seq_cst_i32_local_cluster_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i32 %val, ptr addrspace(5) %slot syncscope("cluster") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_i32_local_gpu_volatile(i32 %val) {
+; CHECK-LABEL: seq_cst_i32_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot1047[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1047;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [seq_cst_i32_local_gpu_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i32 %val, ptr addrspace(5) %slot syncscope("device") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_i64_local_sys(i64 %val) {
+; CHECK-LABEL: seq_cst_i64_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot1048[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1048;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_local_sys_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i64 %val, ptr addrspace(5) %slot syncscope("") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_i64_local_cta(i64 %val) {
+; CHECK-LABEL: seq_cst_i64_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot1049[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1049;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_local_cta_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i64 %val, ptr addrspace(5) %slot syncscope("block") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_i64_local_cluster(i64 %val) {
+; CHECK-LABEL: seq_cst_i64_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot1050[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1050;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_local_cluster_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i64 %val, ptr addrspace(5) %slot syncscope("cluster") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_i64_local_gpu(i64 %val) {
+; CHECK-LABEL: seq_cst_i64_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot1051[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1051;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_local_gpu_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i64 %val, ptr addrspace(5) %slot syncscope("device") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_i64_local_sys_volatile(i64 %val) {
+; CHECK-LABEL: seq_cst_i64_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot1052[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1052;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_local_sys_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i64 %val, ptr addrspace(5) %slot syncscope("") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_i64_local_cta_volatile(i64 %val) {
+; CHECK-LABEL: seq_cst_i64_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot1053[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1053;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_local_cta_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i64 %val, ptr addrspace(5) %slot syncscope("block") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_i64_local_cluster_volatile(i64 %val) {
+; CHECK-LABEL: seq_cst_i64_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot1054[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1054;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_local_cluster_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i64 %val, ptr addrspace(5) %slot syncscope("cluster") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_i64_local_gpu_volatile(i64 %val) {
+; CHECK-LABEL: seq_cst_i64_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot1055[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1055;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_local_gpu_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i64 %val, ptr addrspace(5) %slot syncscope("device") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_half_local_sys(half %val) {
+; CHECK-LABEL: seq_cst_half_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1056[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1056;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_half_local_sys_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic half %val, ptr addrspace(5) %slot syncscope("") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_half_local_cta(half %val) {
+; CHECK-LABEL: seq_cst_half_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1057[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1057;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_half_local_cta_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic half %val, ptr addrspace(5) %slot syncscope("block") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_half_local_cluster(half %val) {
+; CHECK-LABEL: seq_cst_half_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1058[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1058;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_half_local_cluster_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic half %val, ptr addrspace(5) %slot syncscope("cluster") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_half_local_gpu(half %val) {
+; CHECK-LABEL: seq_cst_half_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1059[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1059;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_half_local_gpu_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic half %val, ptr addrspace(5) %slot syncscope("device") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_half_local_sys_volatile(half %val) {
+; CHECK-LABEL: seq_cst_half_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1060[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1060;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_half_local_sys_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile half %val, ptr addrspace(5) %slot syncscope("") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_half_local_cta_volatile(half %val) {
+; CHECK-LABEL: seq_cst_half_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1061[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1061;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_half_local_cta_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile half %val, ptr addrspace(5) %slot syncscope("block") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_half_local_cluster_volatile(half %val) {
+; CHECK-LABEL: seq_cst_half_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1062[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1062;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_half_local_cluster_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile half %val, ptr addrspace(5) %slot syncscope("cluster") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_half_local_gpu_volatile(half %val) {
+; CHECK-LABEL: seq_cst_half_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1063[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1063;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_half_local_gpu_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile half %val, ptr addrspace(5) %slot syncscope("device") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_bfloat_local_sys(bfloat %val) {
+; CHECK-LABEL: seq_cst_bfloat_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1064[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1064;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_bfloat_local_sys_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic bfloat %val, ptr addrspace(5) %slot syncscope("") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_bfloat_local_cta(bfloat %val) {
+; CHECK-LABEL: seq_cst_bfloat_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1065[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1065;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_bfloat_local_cta_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic bfloat %val, ptr addrspace(5) %slot syncscope("block") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_bfloat_local_cluster(bfloat %val) {
+; CHECK-LABEL: seq_cst_bfloat_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1066[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1066;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_bfloat_local_cluster_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic bfloat %val, ptr addrspace(5) %slot syncscope("cluster") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_bfloat_local_gpu(bfloat %val) {
+; CHECK-LABEL: seq_cst_bfloat_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1067[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1067;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_bfloat_local_gpu_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic bfloat %val, ptr addrspace(5) %slot syncscope("device") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_bfloat_local_sys_volatile(bfloat %val) {
+; CHECK-LABEL: seq_cst_bfloat_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1068[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1068;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_bfloat_local_sys_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile bfloat %val, ptr addrspace(5) %slot syncscope("") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_bfloat_local_cta_volatile(bfloat %val) {
+; CHECK-LABEL: seq_cst_bfloat_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1069[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1069;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_bfloat_local_cta_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile bfloat %val, ptr addrspace(5) %slot syncscope("block") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_bfloat_local_cluster_volatile(bfloat %val) {
+; CHECK-LABEL: seq_cst_bfloat_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1070[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1070;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_bfloat_local_cluster_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile bfloat %val, ptr addrspace(5) %slot syncscope("cluster") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_bfloat_local_gpu_volatile(bfloat %val) {
+; CHECK-LABEL: seq_cst_bfloat_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1071[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1071;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_bfloat_local_gpu_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile bfloat %val, ptr addrspace(5) %slot syncscope("device") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_float_local_sys(float %val) {
+; CHECK-LABEL: seq_cst_float_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot1072[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1072;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [seq_cst_float_local_sys_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic float %val, ptr addrspace(5) %slot syncscope("") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_float_local_cta(float %val) {
+; CHECK-LABEL: seq_cst_float_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot1073[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1073;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [seq_cst_float_local_cta_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic float %val, ptr addrspace(5) %slot syncscope("block") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_float_local_cluster(float %val) {
+; CHECK-LABEL: seq_cst_float_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot1074[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1074;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [seq_cst_float_local_cluster_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic float %val, ptr addrspace(5) %slot syncscope("cluster") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_float_local_gpu(float %val) {
+; CHECK-LABEL: seq_cst_float_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot1075[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1075;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [seq_cst_float_local_gpu_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic float %val, ptr addrspace(5) %slot syncscope("device") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_float_local_sys_volatile(float %val) {
+; CHECK-LABEL: seq_cst_float_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot1076[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1076;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [seq_cst_float_local_sys_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile float %val, ptr addrspace(5) %slot syncscope("") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_float_local_cta_volatile(float %val) {
+; CHECK-LABEL: seq_cst_float_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot1077[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1077;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [seq_cst_float_local_cta_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile float %val, ptr addrspace(5) %slot syncscope("block") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_float_local_cluster_volatile(float %val) {
+; CHECK-LABEL: seq_cst_float_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot1078[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1078;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [seq_cst_float_local_cluster_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile float %val, ptr addrspace(5) %slot syncscope("cluster") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_float_local_gpu_volatile(float %val) {
+; CHECK-LABEL: seq_cst_float_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot1079[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1079;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [seq_cst_float_local_gpu_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile float %val, ptr addrspace(5) %slot syncscope("device") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_double_local_sys(double %val) {
+; CHECK-LABEL: seq_cst_double_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot1080[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1080;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_local_sys_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic double %val, ptr addrspace(5) %slot syncscope("") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_double_local_cta(double %val) {
+; CHECK-LABEL: seq_cst_double_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot1081[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1081;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_local_cta_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic double %val, ptr addrspace(5) %slot syncscope("block") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_double_local_cluster(double %val) {
+; CHECK-LABEL: seq_cst_double_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot1082[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1082;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_local_cluster_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic double %val, ptr addrspace(5) %slot syncscope("cluster") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_double_local_gpu(double %val) {
+; CHECK-LABEL: seq_cst_double_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot1083[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1083;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_local_gpu_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic double %val, ptr addrspace(5) %slot syncscope("device") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_double_local_sys_volatile(double %val) {
+; CHECK-LABEL: seq_cst_double_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot1084[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1084;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_local_sys_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile double %val, ptr addrspace(5) %slot syncscope("") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_double_local_cta_volatile(double %val) {
+; CHECK-LABEL: seq_cst_double_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot1085[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1085;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_local_cta_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile double %val, ptr addrspace(5) %slot syncscope("block") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_double_local_cluster_volatile(double %val) {
+; CHECK-LABEL: seq_cst_double_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot1086[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1086;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_local_cluster_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile double %val, ptr addrspace(5) %slot syncscope("cluster") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_double_local_gpu_volatile(double %val) {
+; CHECK-LABEL: seq_cst_double_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot1087[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1087;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_local_gpu_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile double %val, ptr addrspace(5) %slot syncscope("device") seq_cst, align 8
+ ret void
+}
+
diff --git a/llvm/test/CodeGen/NVPTX/store-sm70.ll b/llvm/test/CodeGen/NVPTX/store-sm70.ll
new file mode 100644
index 0000000000000..819124821e5e0
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/store-sm70.ll
@@ -0,0 +1,19168 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_70 -mattr=+ptx63 | FileCheck %s
+; RUN: %if ptxas-sm_70 && ptxas-isa-6.3 %{ llc < %s -march=nvptx64 -mcpu=sm_70 -mattr=+ptx63 | %ptxas-verify -arch=sm_70 %}
+
+define void @notatomic_i8_generic(i8 %val, ptr %addr) {
+; CHECK-LABEL: notatomic_i8_generic(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [notatomic_i8_generic_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i8_generic_param_1];
+; CHECK-NEXT: st.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store i8 %val, ptr %addr
+ ret void
+}
+
+define void @notatomic_i8_generic_volatile(i8 %val, ptr %addr) {
+; CHECK-LABEL: notatomic_i8_generic_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [notatomic_i8_generic_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i8_generic_volatile_param_1];
+; CHECK-NEXT: st.volatile.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store volatile i8 %val, ptr %addr
+ ret void
+}
+
+define void @notatomic_i16_generic(i16 %val, ptr %addr) {
+; CHECK-LABEL: notatomic_i16_generic(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [notatomic_i16_generic_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i16_generic_param_1];
+; CHECK-NEXT: st.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store i16 %val, ptr %addr
+ ret void
+}
+
+define void @notatomic_i16_generic_volatile(i16 %val, ptr %addr) {
+; CHECK-LABEL: notatomic_i16_generic_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [notatomic_i16_generic_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i16_generic_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store volatile i16 %val, ptr %addr
+ ret void
+}
+
+define void @notatomic_i32_generic(i32 %val, ptr %addr) {
+; CHECK-LABEL: notatomic_i32_generic(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [notatomic_i32_generic_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i32_generic_param_1];
+; CHECK-NEXT: st.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store i32 %val, ptr %addr
+ ret void
+}
+
+define void @notatomic_i32_generic_volatile(i32 %val, ptr %addr) {
+; CHECK-LABEL: notatomic_i32_generic_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [notatomic_i32_generic_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i32_generic_volatile_param_1];
+; CHECK-NEXT: st.volatile.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store volatile i32 %val, ptr %addr
+ ret void
+}
+
+define void @notatomic_i64_generic(i64 %val, ptr %addr) {
+; CHECK-LABEL: notatomic_i64_generic(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i64_generic_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [notatomic_i64_generic_param_1];
+; CHECK-NEXT: st.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store i64 %val, ptr %addr
+ ret void
+}
+
+define void @notatomic_i64_generic_volatile(i64 %val, ptr %addr) {
+; CHECK-LABEL: notatomic_i64_generic_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i64_generic_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [notatomic_i64_generic_volatile_param_1];
+; CHECK-NEXT: st.volatile.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store volatile i64 %val, ptr %addr
+ ret void
+}
+
+define void @notatomic_half_generic(half %val, ptr %addr) {
+; CHECK-LABEL: notatomic_half_generic(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [notatomic_half_generic_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_half_generic_param_1];
+; CHECK-NEXT: st.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store half %val, ptr %addr
+ ret void
+}
+
+define void @notatomic_half_generic_volatile(half %val, ptr %addr) {
+; CHECK-LABEL: notatomic_half_generic_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [notatomic_half_generic_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_half_generic_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store volatile half %val, ptr %addr
+ ret void
+}
+
+define void @notatomic_bfloat_generic(bfloat %val, ptr %addr) {
+; CHECK-LABEL: notatomic_bfloat_generic(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [notatomic_bfloat_generic_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_bfloat_generic_param_1];
+; CHECK-NEXT: st.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store bfloat %val, ptr %addr
+ ret void
+}
+
+define void @notatomic_bfloat_generic_volatile(bfloat %val, ptr %addr) {
+; CHECK-LABEL: notatomic_bfloat_generic_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [notatomic_bfloat_generic_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_bfloat_generic_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store volatile bfloat %val, ptr %addr
+ ret void
+}
+
+define void @notatomic_float_generic(float %val, ptr %addr) {
+; CHECK-LABEL: notatomic_float_generic(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [notatomic_float_generic_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_float_generic_param_1];
+; CHECK-NEXT: st.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store float %val, ptr %addr
+ ret void
+}
+
+define void @notatomic_float_generic_volatile(float %val, ptr %addr) {
+; CHECK-LABEL: notatomic_float_generic_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [notatomic_float_generic_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_float_generic_volatile_param_1];
+; CHECK-NEXT: st.volatile.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store volatile float %val, ptr %addr
+ ret void
+}
+
+define void @notatomic_double_generic(double %val, ptr %addr) {
+; CHECK-LABEL: notatomic_double_generic(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_double_generic_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [notatomic_double_generic_param_1];
+; CHECK-NEXT: st.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store double %val, ptr %addr
+ ret void
+}
+
+define void @notatomic_double_generic_volatile(double %val, ptr %addr) {
+; CHECK-LABEL: notatomic_double_generic_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_double_generic_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [notatomic_double_generic_volatile_param_1];
+; CHECK-NEXT: st.volatile.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store volatile double %val, ptr %addr
+ ret void
+}
+
+define void @notatomic_i8_global(i8 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: notatomic_i8_global(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [notatomic_i8_global_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i8_global_param_1];
+; CHECK-NEXT: st.global.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store i8 %val, ptr addrspace(1) %addr
+ ret void
+}
+
+define void @notatomic_i8_global_volatile(i8 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: notatomic_i8_global_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [notatomic_i8_global_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i8_global_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store volatile i8 %val, ptr addrspace(1) %addr
+ ret void
+}
+
+define void @notatomic_i16_global(i16 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: notatomic_i16_global(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [notatomic_i16_global_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i16_global_param_1];
+; CHECK-NEXT: st.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store i16 %val, ptr addrspace(1) %addr
+ ret void
+}
+
+define void @notatomic_i16_global_volatile(i16 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: notatomic_i16_global_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [notatomic_i16_global_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i16_global_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store volatile i16 %val, ptr addrspace(1) %addr
+ ret void
+}
+
+define void @notatomic_i32_global(i32 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: notatomic_i32_global(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [notatomic_i32_global_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i32_global_param_1];
+; CHECK-NEXT: st.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store i32 %val, ptr addrspace(1) %addr
+ ret void
+}
+
+define void @notatomic_i32_global_volatile(i32 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: notatomic_i32_global_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [notatomic_i32_global_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i32_global_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store volatile i32 %val, ptr addrspace(1) %addr
+ ret void
+}
+
+define void @notatomic_i64_global(i64 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: notatomic_i64_global(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i64_global_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [notatomic_i64_global_param_1];
+; CHECK-NEXT: st.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store i64 %val, ptr addrspace(1) %addr
+ ret void
+}
+
+define void @notatomic_i64_global_volatile(i64 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: notatomic_i64_global_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i64_global_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [notatomic_i64_global_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store volatile i64 %val, ptr addrspace(1) %addr
+ ret void
+}
+
+define void @notatomic_half_global(half %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: notatomic_half_global(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [notatomic_half_global_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_half_global_param_1];
+; CHECK-NEXT: st.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store half %val, ptr addrspace(1) %addr
+ ret void
+}
+
+define void @notatomic_half_global_volatile(half %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: notatomic_half_global_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [notatomic_half_global_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_half_global_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store volatile half %val, ptr addrspace(1) %addr
+ ret void
+}
+
+define void @notatomic_bfloat_global(bfloat %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: notatomic_bfloat_global(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [notatomic_bfloat_global_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_bfloat_global_param_1];
+; CHECK-NEXT: st.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store bfloat %val, ptr addrspace(1) %addr
+ ret void
+}
+
+define void @notatomic_bfloat_global_volatile(bfloat %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: notatomic_bfloat_global_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [notatomic_bfloat_global_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_bfloat_global_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store volatile bfloat %val, ptr addrspace(1) %addr
+ ret void
+}
+
+define void @notatomic_float_global(float %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: notatomic_float_global(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [notatomic_float_global_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_float_global_param_1];
+; CHECK-NEXT: st.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store float %val, ptr addrspace(1) %addr
+ ret void
+}
+
+define void @notatomic_float_global_volatile(float %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: notatomic_float_global_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [notatomic_float_global_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_float_global_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store volatile float %val, ptr addrspace(1) %addr
+ ret void
+}
+
+define void @notatomic_double_global(double %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: notatomic_double_global(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_double_global_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [notatomic_double_global_param_1];
+; CHECK-NEXT: st.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store double %val, ptr addrspace(1) %addr
+ ret void
+}
+
+define void @notatomic_double_global_volatile(double %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: notatomic_double_global_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_double_global_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [notatomic_double_global_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store volatile double %val, ptr addrspace(1) %addr
+ ret void
+}
+
+define void @notatomic_i8_shared(i8 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: notatomic_i8_shared(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [notatomic_i8_shared_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i8_shared_param_1];
+; CHECK-NEXT: st.shared.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store i8 %val, ptr addrspace(3) %addr
+ ret void
+}
+
+define void @notatomic_i8_shared_volatile(i8 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: notatomic_i8_shared_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [notatomic_i8_shared_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i8_shared_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store volatile i8 %val, ptr addrspace(3) %addr
+ ret void
+}
+
+define void @notatomic_i16_shared(i16 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: notatomic_i16_shared(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [notatomic_i16_shared_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i16_shared_param_1];
+; CHECK-NEXT: st.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store i16 %val, ptr addrspace(3) %addr
+ ret void
+}
+
+define void @notatomic_i16_shared_volatile(i16 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: notatomic_i16_shared_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [notatomic_i16_shared_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i16_shared_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store volatile i16 %val, ptr addrspace(3) %addr
+ ret void
+}
+
+define void @notatomic_i32_shared(i32 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: notatomic_i32_shared(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [notatomic_i32_shared_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i32_shared_param_1];
+; CHECK-NEXT: st.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store i32 %val, ptr addrspace(3) %addr
+ ret void
+}
+
+define void @notatomic_i32_shared_volatile(i32 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: notatomic_i32_shared_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [notatomic_i32_shared_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i32_shared_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store volatile i32 %val, ptr addrspace(3) %addr
+ ret void
+}
+
+define void @notatomic_i64_shared(i64 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: notatomic_i64_shared(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i64_shared_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [notatomic_i64_shared_param_1];
+; CHECK-NEXT: st.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store i64 %val, ptr addrspace(3) %addr
+ ret void
+}
+
+define void @notatomic_i64_shared_volatile(i64 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: notatomic_i64_shared_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i64_shared_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [notatomic_i64_shared_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store volatile i64 %val, ptr addrspace(3) %addr
+ ret void
+}
+
+define void @notatomic_half_shared(half %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: notatomic_half_shared(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [notatomic_half_shared_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_half_shared_param_1];
+; CHECK-NEXT: st.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store half %val, ptr addrspace(3) %addr
+ ret void
+}
+
+define void @notatomic_half_shared_volatile(half %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: notatomic_half_shared_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [notatomic_half_shared_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_half_shared_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store volatile half %val, ptr addrspace(3) %addr
+ ret void
+}
+
+define void @notatomic_bfloat_shared(bfloat %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: notatomic_bfloat_shared(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [notatomic_bfloat_shared_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_bfloat_shared_param_1];
+; CHECK-NEXT: st.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store bfloat %val, ptr addrspace(3) %addr
+ ret void
+}
+
+define void @notatomic_bfloat_shared_volatile(bfloat %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: notatomic_bfloat_shared_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [notatomic_bfloat_shared_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_bfloat_shared_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store volatile bfloat %val, ptr addrspace(3) %addr
+ ret void
+}
+
+define void @notatomic_float_shared(float %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: notatomic_float_shared(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [notatomic_float_shared_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_float_shared_param_1];
+; CHECK-NEXT: st.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store float %val, ptr addrspace(3) %addr
+ ret void
+}
+
+define void @notatomic_float_shared_volatile(float %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: notatomic_float_shared_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [notatomic_float_shared_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_float_shared_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store volatile float %val, ptr addrspace(3) %addr
+ ret void
+}
+
+define void @notatomic_double_shared(double %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: notatomic_double_shared(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_double_shared_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [notatomic_double_shared_param_1];
+; CHECK-NEXT: st.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store double %val, ptr addrspace(3) %addr
+ ret void
+}
+
+define void @notatomic_double_shared_volatile(double %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: notatomic_double_shared_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_double_shared_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [notatomic_double_shared_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store volatile double %val, ptr addrspace(3) %addr
+ ret void
+}
+
+define void @notatomic_i8_local(i8 %val) {
+; CHECK-LABEL: notatomic_i8_local(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot48[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot48;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b8 %rs1, [notatomic_i8_local_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b8 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store i8 %val, ptr addrspace(5) %slot
+ ret void
+}
+
+define void @notatomic_i8_local_volatile(i8 %val) {
+; CHECK-LABEL: notatomic_i8_local_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot49[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot49;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b8 %rs1, [notatomic_i8_local_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b8 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store volatile i8 %val, ptr addrspace(5) %slot
+ ret void
+}
+
+define void @notatomic_i16_local(i16 %val) {
+; CHECK-LABEL: notatomic_i16_local(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot50[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot50;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [notatomic_i16_local_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store i16 %val, ptr addrspace(5) %slot
+ ret void
+}
+
+define void @notatomic_i16_local_volatile(i16 %val) {
+; CHECK-LABEL: notatomic_i16_local_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot51[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot51;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [notatomic_i16_local_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store volatile i16 %val, ptr addrspace(5) %slot
+ ret void
+}
+
+define void @notatomic_i32_local(i32 %val) {
+; CHECK-LABEL: notatomic_i32_local(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot52[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot52;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [notatomic_i32_local_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store i32 %val, ptr addrspace(5) %slot
+ ret void
+}
+
+define void @notatomic_i32_local_volatile(i32 %val) {
+; CHECK-LABEL: notatomic_i32_local_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot53[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot53;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [notatomic_i32_local_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store volatile i32 %val, ptr addrspace(5) %slot
+ ret void
+}
+
+define void @notatomic_i64_local(i64 %val) {
+; CHECK-LABEL: notatomic_i64_local(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot54[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot54;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i64_local_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store i64 %val, ptr addrspace(5) %slot
+ ret void
+}
+
+define void @notatomic_i64_local_volatile(i64 %val) {
+; CHECK-LABEL: notatomic_i64_local_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot55[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot55;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i64_local_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store volatile i64 %val, ptr addrspace(5) %slot
+ ret void
+}
+
+define void @notatomic_half_local(half %val) {
+; CHECK-LABEL: notatomic_half_local(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot56[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot56;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [notatomic_half_local_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store half %val, ptr addrspace(5) %slot
+ ret void
+}
+
+define void @notatomic_half_local_volatile(half %val) {
+; CHECK-LABEL: notatomic_half_local_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot57[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot57;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [notatomic_half_local_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store volatile half %val, ptr addrspace(5) %slot
+ ret void
+}
+
+define void @notatomic_bfloat_local(bfloat %val) {
+; CHECK-LABEL: notatomic_bfloat_local(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot58[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot58;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [notatomic_bfloat_local_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store bfloat %val, ptr addrspace(5) %slot
+ ret void
+}
+
+define void @notatomic_bfloat_local_volatile(bfloat %val) {
+; CHECK-LABEL: notatomic_bfloat_local_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot59[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot59;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [notatomic_bfloat_local_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store volatile bfloat %val, ptr addrspace(5) %slot
+ ret void
+}
+
+define void @notatomic_float_local(float %val) {
+; CHECK-LABEL: notatomic_float_local(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot60[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot60;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [notatomic_float_local_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store float %val, ptr addrspace(5) %slot
+ ret void
+}
+
+define void @notatomic_float_local_volatile(float %val) {
+; CHECK-LABEL: notatomic_float_local_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot61[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot61;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [notatomic_float_local_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store volatile float %val, ptr addrspace(5) %slot
+ ret void
+}
+
+define void @notatomic_double_local(double %val) {
+; CHECK-LABEL: notatomic_double_local(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot62[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot62;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_double_local_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca double, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store double %val, ptr addrspace(5) %slot
+ ret void
+}
+
+define void @notatomic_double_local_volatile(double %val) {
+; CHECK-LABEL: notatomic_double_local_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot63[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot63;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_double_local_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca double, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store volatile double %val, ptr addrspace(5) %slot
+ ret void
+}
+
+define void @unordered_i8_generic_sys(i8 %val, ptr %addr) {
+; CHECK-LABEL: unordered_i8_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [unordered_i8_generic_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_generic_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr %addr syncscope("") unordered, align 1
+ ret void
+}
+
+define void @unordered_i8_generic_cta(i8 %val, ptr %addr) {
+; CHECK-LABEL: unordered_i8_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [unordered_i8_generic_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_generic_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr %addr syncscope("block") unordered, align 1
+ ret void
+}
+
+define void @unordered_i8_generic_cluster(i8 %val, ptr %addr) {
+; CHECK-LABEL: unordered_i8_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [unordered_i8_generic_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_generic_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cta.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr %addr syncscope("cluster") unordered, align 1
+ ret void
+}
+
+define void @unordered_i8_generic_gpu(i8 %val, ptr %addr) {
+; CHECK-LABEL: unordered_i8_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [unordered_i8_generic_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_generic_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr %addr syncscope("device") unordered, align 1
+ ret void
+}
+
+define void @unordered_i8_generic_sys_volatile(i8 %val, ptr %addr) {
+; CHECK-LABEL: unordered_i8_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [unordered_i8_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_generic_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr %addr syncscope("") unordered, align 1
+ ret void
+}
+
+define void @unordered_i8_generic_cta_volatile(i8 %val, ptr %addr) {
+; CHECK-LABEL: unordered_i8_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [unordered_i8_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_generic_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr %addr syncscope("block") unordered, align 1
+ ret void
+}
+
+define void @unordered_i8_generic_cluster_volatile(i8 %val, ptr %addr) {
+; CHECK-LABEL: unordered_i8_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [unordered_i8_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_generic_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr %addr syncscope("cluster") unordered, align 1
+ ret void
+}
+
+define void @unordered_i8_generic_gpu_volatile(i8 %val, ptr %addr) {
+; CHECK-LABEL: unordered_i8_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [unordered_i8_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_generic_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr %addr syncscope("device") unordered, align 1
+ ret void
+}
+
+define void @unordered_i16_generic_sys(i16 %val, ptr %addr) {
+; CHECK-LABEL: unordered_i16_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_i16_generic_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_generic_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr %addr syncscope("") unordered, align 2
+ ret void
+}
+
+define void @unordered_i16_generic_cta(i16 %val, ptr %addr) {
+; CHECK-LABEL: unordered_i16_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_i16_generic_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_generic_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr %addr syncscope("block") unordered, align 2
+ ret void
+}
+
+define void @unordered_i16_generic_cluster(i16 %val, ptr %addr) {
+; CHECK-LABEL: unordered_i16_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_i16_generic_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_generic_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cta.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr %addr syncscope("cluster") unordered, align 2
+ ret void
+}
+
+define void @unordered_i16_generic_gpu(i16 %val, ptr %addr) {
+; CHECK-LABEL: unordered_i16_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_i16_generic_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_generic_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr %addr syncscope("device") unordered, align 2
+ ret void
+}
+
+define void @unordered_i16_generic_sys_volatile(i16 %val, ptr %addr) {
+; CHECK-LABEL: unordered_i16_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_i16_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_generic_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr %addr syncscope("") unordered, align 2
+ ret void
+}
+
+define void @unordered_i16_generic_cta_volatile(i16 %val, ptr %addr) {
+; CHECK-LABEL: unordered_i16_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_i16_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_generic_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr %addr syncscope("block") unordered, align 2
+ ret void
+}
+
+define void @unordered_i16_generic_cluster_volatile(i16 %val, ptr %addr) {
+; CHECK-LABEL: unordered_i16_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_i16_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_generic_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr %addr syncscope("cluster") unordered, align 2
+ ret void
+}
+
+define void @unordered_i16_generic_gpu_volatile(i16 %val, ptr %addr) {
+; CHECK-LABEL: unordered_i16_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_i16_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_generic_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr %addr syncscope("device") unordered, align 2
+ ret void
+}
+
+define void @unordered_i32_generic_sys(i32 %val, ptr %addr) {
+; CHECK-LABEL: unordered_i32_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_i32_generic_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_generic_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr %addr syncscope("") unordered, align 4
+ ret void
+}
+
+define void @unordered_i32_generic_cta(i32 %val, ptr %addr) {
+; CHECK-LABEL: unordered_i32_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_i32_generic_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_generic_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr %addr syncscope("block") unordered, align 4
+ ret void
+}
+
+define void @unordered_i32_generic_cluster(i32 %val, ptr %addr) {
+; CHECK-LABEL: unordered_i32_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_i32_generic_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_generic_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cta.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr %addr syncscope("cluster") unordered, align 4
+ ret void
+}
+
+define void @unordered_i32_generic_gpu(i32 %val, ptr %addr) {
+; CHECK-LABEL: unordered_i32_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_i32_generic_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_generic_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr %addr syncscope("device") unordered, align 4
+ ret void
+}
+
+define void @unordered_i32_generic_sys_volatile(i32 %val, ptr %addr) {
+; CHECK-LABEL: unordered_i32_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_i32_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_generic_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr %addr syncscope("") unordered, align 4
+ ret void
+}
+
+define void @unordered_i32_generic_cta_volatile(i32 %val, ptr %addr) {
+; CHECK-LABEL: unordered_i32_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_i32_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_generic_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr %addr syncscope("block") unordered, align 4
+ ret void
+}
+
+define void @unordered_i32_generic_cluster_volatile(i32 %val, ptr %addr) {
+; CHECK-LABEL: unordered_i32_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_i32_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_generic_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr %addr syncscope("cluster") unordered, align 4
+ ret void
+}
+
+define void @unordered_i32_generic_gpu_volatile(i32 %val, ptr %addr) {
+; CHECK-LABEL: unordered_i32_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_i32_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_generic_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr %addr syncscope("device") unordered, align 4
+ ret void
+}
+
+define void @unordered_i64_generic_sys(i64 %val, ptr %addr) {
+; CHECK-LABEL: unordered_i64_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_generic_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_i64_generic_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr %addr syncscope("") unordered, align 8
+ ret void
+}
+
+define void @unordered_i64_generic_cta(i64 %val, ptr %addr) {
+; CHECK-LABEL: unordered_i64_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_generic_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_i64_generic_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr %addr syncscope("block") unordered, align 8
+ ret void
+}
+
+define void @unordered_i64_generic_cluster(i64 %val, ptr %addr) {
+; CHECK-LABEL: unordered_i64_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_generic_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_i64_generic_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cta.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr %addr syncscope("cluster") unordered, align 8
+ ret void
+}
+
+define void @unordered_i64_generic_gpu(i64 %val, ptr %addr) {
+; CHECK-LABEL: unordered_i64_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_generic_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_i64_generic_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr %addr syncscope("device") unordered, align 8
+ ret void
+}
+
+define void @unordered_i64_generic_sys_volatile(i64 %val, ptr %addr) {
+; CHECK-LABEL: unordered_i64_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_i64_generic_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr %addr syncscope("") unordered, align 8
+ ret void
+}
+
+define void @unordered_i64_generic_cta_volatile(i64 %val, ptr %addr) {
+; CHECK-LABEL: unordered_i64_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_i64_generic_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr %addr syncscope("block") unordered, align 8
+ ret void
+}
+
+define void @unordered_i64_generic_cluster_volatile(i64 %val, ptr %addr) {
+; CHECK-LABEL: unordered_i64_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_i64_generic_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr %addr syncscope("cluster") unordered, align 8
+ ret void
+}
+
+define void @unordered_i64_generic_gpu_volatile(i64 %val, ptr %addr) {
+; CHECK-LABEL: unordered_i64_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_i64_generic_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr %addr syncscope("device") unordered, align 8
+ ret void
+}
+
+define void @unordered_half_generic_sys(half %val, ptr %addr) {
+; CHECK-LABEL: unordered_half_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_half_generic_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_generic_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr %addr syncscope("") unordered, align 2
+ ret void
+}
+
+define void @unordered_half_generic_cta(half %val, ptr %addr) {
+; CHECK-LABEL: unordered_half_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_half_generic_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_generic_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr %addr syncscope("block") unordered, align 2
+ ret void
+}
+
+define void @unordered_half_generic_cluster(half %val, ptr %addr) {
+; CHECK-LABEL: unordered_half_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_half_generic_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_generic_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cta.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr %addr syncscope("cluster") unordered, align 2
+ ret void
+}
+
+define void @unordered_half_generic_gpu(half %val, ptr %addr) {
+; CHECK-LABEL: unordered_half_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_half_generic_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_generic_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr %addr syncscope("device") unordered, align 2
+ ret void
+}
+
+define void @unordered_half_generic_sys_volatile(half %val, ptr %addr) {
+; CHECK-LABEL: unordered_half_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_half_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_generic_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr %addr syncscope("") unordered, align 2
+ ret void
+}
+
+define void @unordered_half_generic_cta_volatile(half %val, ptr %addr) {
+; CHECK-LABEL: unordered_half_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_half_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_generic_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr %addr syncscope("block") unordered, align 2
+ ret void
+}
+
+define void @unordered_half_generic_cluster_volatile(half %val, ptr %addr) {
+; CHECK-LABEL: unordered_half_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_half_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_generic_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr %addr syncscope("cluster") unordered, align 2
+ ret void
+}
+
+define void @unordered_half_generic_gpu_volatile(half %val, ptr %addr) {
+; CHECK-LABEL: unordered_half_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_half_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_generic_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr %addr syncscope("device") unordered, align 2
+ ret void
+}
+
+define void @unordered_bfloat_generic_sys(bfloat %val, ptr %addr) {
+; CHECK-LABEL: unordered_bfloat_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_bfloat_generic_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_generic_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr %addr syncscope("") unordered, align 2
+ ret void
+}
+
+define void @unordered_bfloat_generic_cta(bfloat %val, ptr %addr) {
+; CHECK-LABEL: unordered_bfloat_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_bfloat_generic_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_generic_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr %addr syncscope("block") unordered, align 2
+ ret void
+}
+
+define void @unordered_bfloat_generic_cluster(bfloat %val, ptr %addr) {
+; CHECK-LABEL: unordered_bfloat_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_bfloat_generic_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_generic_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cta.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr %addr syncscope("cluster") unordered, align 2
+ ret void
+}
+
+define void @unordered_bfloat_generic_gpu(bfloat %val, ptr %addr) {
+; CHECK-LABEL: unordered_bfloat_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_bfloat_generic_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_generic_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr %addr syncscope("device") unordered, align 2
+ ret void
+}
+
+define void @unordered_bfloat_generic_sys_volatile(bfloat %val, ptr %addr) {
+; CHECK-LABEL: unordered_bfloat_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_bfloat_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_generic_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr %addr syncscope("") unordered, align 2
+ ret void
+}
+
+define void @unordered_bfloat_generic_cta_volatile(bfloat %val, ptr %addr) {
+; CHECK-LABEL: unordered_bfloat_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_bfloat_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_generic_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr %addr syncscope("block") unordered, align 2
+ ret void
+}
+
+define void @unordered_bfloat_generic_cluster_volatile(bfloat %val, ptr %addr) {
+; CHECK-LABEL: unordered_bfloat_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_bfloat_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_generic_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr %addr syncscope("cluster") unordered, align 2
+ ret void
+}
+
+define void @unordered_bfloat_generic_gpu_volatile(bfloat %val, ptr %addr) {
+; CHECK-LABEL: unordered_bfloat_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_bfloat_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_generic_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr %addr syncscope("device") unordered, align 2
+ ret void
+}
+
+define void @unordered_float_generic_sys(float %val, ptr %addr) {
+; CHECK-LABEL: unordered_float_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_float_generic_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_generic_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr %addr syncscope("") unordered, align 4
+ ret void
+}
+
+define void @unordered_float_generic_cta(float %val, ptr %addr) {
+; CHECK-LABEL: unordered_float_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_float_generic_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_generic_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr %addr syncscope("block") unordered, align 4
+ ret void
+}
+
+define void @unordered_float_generic_cluster(float %val, ptr %addr) {
+; CHECK-LABEL: unordered_float_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_float_generic_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_generic_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cta.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr %addr syncscope("cluster") unordered, align 4
+ ret void
+}
+
+define void @unordered_float_generic_gpu(float %val, ptr %addr) {
+; CHECK-LABEL: unordered_float_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_float_generic_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_generic_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr %addr syncscope("device") unordered, align 4
+ ret void
+}
+
+define void @unordered_float_generic_sys_volatile(float %val, ptr %addr) {
+; CHECK-LABEL: unordered_float_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_float_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_generic_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr %addr syncscope("") unordered, align 4
+ ret void
+}
+
+define void @unordered_float_generic_cta_volatile(float %val, ptr %addr) {
+; CHECK-LABEL: unordered_float_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_float_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_generic_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr %addr syncscope("block") unordered, align 4
+ ret void
+}
+
+define void @unordered_float_generic_cluster_volatile(float %val, ptr %addr) {
+; CHECK-LABEL: unordered_float_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_float_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_generic_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr %addr syncscope("cluster") unordered, align 4
+ ret void
+}
+
+define void @unordered_float_generic_gpu_volatile(float %val, ptr %addr) {
+; CHECK-LABEL: unordered_float_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_float_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_generic_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr %addr syncscope("device") unordered, align 4
+ ret void
+}
+
+define void @unordered_double_generic_sys(double %val, ptr %addr) {
+; CHECK-LABEL: unordered_double_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_generic_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_double_generic_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr %addr syncscope("") unordered, align 8
+ ret void
+}
+
+define void @unordered_double_generic_cta(double %val, ptr %addr) {
+; CHECK-LABEL: unordered_double_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_generic_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_double_generic_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr %addr syncscope("block") unordered, align 8
+ ret void
+}
+
+define void @unordered_double_generic_cluster(double %val, ptr %addr) {
+; CHECK-LABEL: unordered_double_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_generic_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_double_generic_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cta.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr %addr syncscope("cluster") unordered, align 8
+ ret void
+}
+
+define void @unordered_double_generic_gpu(double %val, ptr %addr) {
+; CHECK-LABEL: unordered_double_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_generic_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_double_generic_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr %addr syncscope("device") unordered, align 8
+ ret void
+}
+
+define void @unordered_double_generic_sys_volatile(double %val, ptr %addr) {
+; CHECK-LABEL: unordered_double_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_double_generic_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr %addr syncscope("") unordered, align 8
+ ret void
+}
+
+define void @unordered_double_generic_cta_volatile(double %val, ptr %addr) {
+; CHECK-LABEL: unordered_double_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_double_generic_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr %addr syncscope("block") unordered, align 8
+ ret void
+}
+
+define void @unordered_double_generic_cluster_volatile(double %val, ptr %addr) {
+; CHECK-LABEL: unordered_double_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_double_generic_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr %addr syncscope("cluster") unordered, align 8
+ ret void
+}
+
+define void @unordered_double_generic_gpu_volatile(double %val, ptr %addr) {
+; CHECK-LABEL: unordered_double_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_double_generic_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr %addr syncscope("device") unordered, align 8
+ ret void
+}
+
+define void @unordered_i8_global_sys(i8 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i8_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [unordered_i8_global_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_global_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.global.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr addrspace(1) %addr syncscope("") unordered, align 1
+ ret void
+}
+
+define void @unordered_i8_global_cta(i8 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i8_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [unordered_i8_global_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_global_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.global.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr addrspace(1) %addr syncscope("block") unordered, align 1
+ ret void
+}
+
+define void @unordered_i8_global_cluster(i8 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i8_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [unordered_i8_global_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_global_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cta.global.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr addrspace(1) %addr syncscope("cluster") unordered, align 1
+ ret void
+}
+
+define void @unordered_i8_global_gpu(i8 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i8_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [unordered_i8_global_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_global_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.global.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr addrspace(1) %addr syncscope("device") unordered, align 1
+ ret void
+}
+
+define void @unordered_i8_global_sys_volatile(i8 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i8_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [unordered_i8_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_global_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr addrspace(1) %addr syncscope("") unordered, align 1
+ ret void
+}
+
+define void @unordered_i8_global_cta_volatile(i8 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i8_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [unordered_i8_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_global_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr addrspace(1) %addr syncscope("block") unordered, align 1
+ ret void
+}
+
+define void @unordered_i8_global_cluster_volatile(i8 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i8_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [unordered_i8_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_global_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr addrspace(1) %addr syncscope("cluster") unordered, align 1
+ ret void
+}
+
+define void @unordered_i8_global_gpu_volatile(i8 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i8_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [unordered_i8_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_global_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr addrspace(1) %addr syncscope("device") unordered, align 1
+ ret void
+}
+
+define void @unordered_i16_global_sys(i16 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i16_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_i16_global_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_global_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr addrspace(1) %addr syncscope("") unordered, align 2
+ ret void
+}
+
+define void @unordered_i16_global_cta(i16 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i16_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_i16_global_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_global_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr addrspace(1) %addr syncscope("block") unordered, align 2
+ ret void
+}
+
+define void @unordered_i16_global_cluster(i16 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i16_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_i16_global_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_global_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cta.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr addrspace(1) %addr syncscope("cluster") unordered, align 2
+ ret void
+}
+
+define void @unordered_i16_global_gpu(i16 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i16_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_i16_global_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_global_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr addrspace(1) %addr syncscope("device") unordered, align 2
+ ret void
+}
+
+define void @unordered_i16_global_sys_volatile(i16 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i16_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_i16_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_global_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr addrspace(1) %addr syncscope("") unordered, align 2
+ ret void
+}
+
+define void @unordered_i16_global_cta_volatile(i16 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i16_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_i16_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_global_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr addrspace(1) %addr syncscope("block") unordered, align 2
+ ret void
+}
+
+define void @unordered_i16_global_cluster_volatile(i16 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i16_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_i16_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_global_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr addrspace(1) %addr syncscope("cluster") unordered, align 2
+ ret void
+}
+
+define void @unordered_i16_global_gpu_volatile(i16 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i16_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_i16_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_global_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr addrspace(1) %addr syncscope("device") unordered, align 2
+ ret void
+}
+
+define void @unordered_i32_global_sys(i32 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i32_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_i32_global_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_global_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr addrspace(1) %addr syncscope("") unordered, align 4
+ ret void
+}
+
+define void @unordered_i32_global_cta(i32 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i32_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_i32_global_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_global_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr addrspace(1) %addr syncscope("block") unordered, align 4
+ ret void
+}
+
+define void @unordered_i32_global_cluster(i32 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i32_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_i32_global_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_global_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cta.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr addrspace(1) %addr syncscope("cluster") unordered, align 4
+ ret void
+}
+
+define void @unordered_i32_global_gpu(i32 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i32_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_i32_global_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_global_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr addrspace(1) %addr syncscope("device") unordered, align 4
+ ret void
+}
+
+define void @unordered_i32_global_sys_volatile(i32 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i32_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_i32_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_global_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr addrspace(1) %addr syncscope("") unordered, align 4
+ ret void
+}
+
+define void @unordered_i32_global_cta_volatile(i32 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i32_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_i32_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_global_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr addrspace(1) %addr syncscope("block") unordered, align 4
+ ret void
+}
+
+define void @unordered_i32_global_cluster_volatile(i32 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i32_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_i32_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_global_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr addrspace(1) %addr syncscope("cluster") unordered, align 4
+ ret void
+}
+
+define void @unordered_i32_global_gpu_volatile(i32 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i32_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_i32_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_global_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr addrspace(1) %addr syncscope("device") unordered, align 4
+ ret void
+}
+
+define void @unordered_i64_global_sys(i64 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i64_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_global_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_i64_global_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr addrspace(1) %addr syncscope("") unordered, align 8
+ ret void
+}
+
+define void @unordered_i64_global_cta(i64 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i64_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_global_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_i64_global_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr addrspace(1) %addr syncscope("block") unordered, align 8
+ ret void
+}
+
+define void @unordered_i64_global_cluster(i64 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i64_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_global_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_i64_global_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cta.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr addrspace(1) %addr syncscope("cluster") unordered, align 8
+ ret void
+}
+
+define void @unordered_i64_global_gpu(i64 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i64_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_global_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_i64_global_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr addrspace(1) %addr syncscope("device") unordered, align 8
+ ret void
+}
+
+define void @unordered_i64_global_sys_volatile(i64 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i64_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_i64_global_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr addrspace(1) %addr syncscope("") unordered, align 8
+ ret void
+}
+
+define void @unordered_i64_global_cta_volatile(i64 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i64_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_i64_global_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr addrspace(1) %addr syncscope("block") unordered, align 8
+ ret void
+}
+
+define void @unordered_i64_global_cluster_volatile(i64 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i64_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_i64_global_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr addrspace(1) %addr syncscope("cluster") unordered, align 8
+ ret void
+}
+
+define void @unordered_i64_global_gpu_volatile(i64 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i64_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_i64_global_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr addrspace(1) %addr syncscope("device") unordered, align 8
+ ret void
+}
+
+define void @unordered_half_global_sys(half %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_half_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_half_global_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_global_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr addrspace(1) %addr syncscope("") unordered, align 2
+ ret void
+}
+
+define void @unordered_half_global_cta(half %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_half_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_half_global_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_global_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr addrspace(1) %addr syncscope("block") unordered, align 2
+ ret void
+}
+
+define void @unordered_half_global_cluster(half %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_half_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_half_global_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_global_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cta.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr addrspace(1) %addr syncscope("cluster") unordered, align 2
+ ret void
+}
+
+define void @unordered_half_global_gpu(half %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_half_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_half_global_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_global_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr addrspace(1) %addr syncscope("device") unordered, align 2
+ ret void
+}
+
+define void @unordered_half_global_sys_volatile(half %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_half_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_half_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_global_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr addrspace(1) %addr syncscope("") unordered, align 2
+ ret void
+}
+
+define void @unordered_half_global_cta_volatile(half %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_half_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_half_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_global_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr addrspace(1) %addr syncscope("block") unordered, align 2
+ ret void
+}
+
+define void @unordered_half_global_cluster_volatile(half %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_half_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_half_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_global_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr addrspace(1) %addr syncscope("cluster") unordered, align 2
+ ret void
+}
+
+define void @unordered_half_global_gpu_volatile(half %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_half_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_half_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_global_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr addrspace(1) %addr syncscope("device") unordered, align 2
+ ret void
+}
+
+define void @unordered_bfloat_global_sys(bfloat %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_bfloat_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_bfloat_global_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_global_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr addrspace(1) %addr syncscope("") unordered, align 2
+ ret void
+}
+
+define void @unordered_bfloat_global_cta(bfloat %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_bfloat_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_bfloat_global_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_global_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr addrspace(1) %addr syncscope("block") unordered, align 2
+ ret void
+}
+
+define void @unordered_bfloat_global_cluster(bfloat %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_bfloat_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_bfloat_global_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_global_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cta.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr addrspace(1) %addr syncscope("cluster") unordered, align 2
+ ret void
+}
+
+define void @unordered_bfloat_global_gpu(bfloat %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_bfloat_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_bfloat_global_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_global_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr addrspace(1) %addr syncscope("device") unordered, align 2
+ ret void
+}
+
+define void @unordered_bfloat_global_sys_volatile(bfloat %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_bfloat_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_bfloat_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_global_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr addrspace(1) %addr syncscope("") unordered, align 2
+ ret void
+}
+
+define void @unordered_bfloat_global_cta_volatile(bfloat %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_bfloat_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_bfloat_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_global_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr addrspace(1) %addr syncscope("block") unordered, align 2
+ ret void
+}
+
+define void @unordered_bfloat_global_cluster_volatile(bfloat %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_bfloat_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_bfloat_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_global_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr addrspace(1) %addr syncscope("cluster") unordered, align 2
+ ret void
+}
+
+define void @unordered_bfloat_global_gpu_volatile(bfloat %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_bfloat_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_bfloat_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_global_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr addrspace(1) %addr syncscope("device") unordered, align 2
+ ret void
+}
+
+define void @unordered_float_global_sys(float %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_float_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_float_global_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_global_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr addrspace(1) %addr syncscope("") unordered, align 4
+ ret void
+}
+
+define void @unordered_float_global_cta(float %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_float_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_float_global_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_global_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr addrspace(1) %addr syncscope("block") unordered, align 4
+ ret void
+}
+
+define void @unordered_float_global_cluster(float %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_float_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_float_global_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_global_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cta.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr addrspace(1) %addr syncscope("cluster") unordered, align 4
+ ret void
+}
+
+define void @unordered_float_global_gpu(float %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_float_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_float_global_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_global_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr addrspace(1) %addr syncscope("device") unordered, align 4
+ ret void
+}
+
+define void @unordered_float_global_sys_volatile(float %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_float_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_float_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_global_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr addrspace(1) %addr syncscope("") unordered, align 4
+ ret void
+}
+
+define void @unordered_float_global_cta_volatile(float %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_float_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_float_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_global_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr addrspace(1) %addr syncscope("block") unordered, align 4
+ ret void
+}
+
+define void @unordered_float_global_cluster_volatile(float %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_float_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_float_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_global_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr addrspace(1) %addr syncscope("cluster") unordered, align 4
+ ret void
+}
+
+define void @unordered_float_global_gpu_volatile(float %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_float_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_float_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_global_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr addrspace(1) %addr syncscope("device") unordered, align 4
+ ret void
+}
+
+define void @unordered_double_global_sys(double %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_double_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_global_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_double_global_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr addrspace(1) %addr syncscope("") unordered, align 8
+ ret void
+}
+
+define void @unordered_double_global_cta(double %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_double_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_global_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_double_global_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr addrspace(1) %addr syncscope("block") unordered, align 8
+ ret void
+}
+
+define void @unordered_double_global_cluster(double %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_double_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_global_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_double_global_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cta.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr addrspace(1) %addr syncscope("cluster") unordered, align 8
+ ret void
+}
+
+define void @unordered_double_global_gpu(double %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_double_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_global_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_double_global_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr addrspace(1) %addr syncscope("device") unordered, align 8
+ ret void
+}
+
+define void @unordered_double_global_sys_volatile(double %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_double_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_double_global_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr addrspace(1) %addr syncscope("") unordered, align 8
+ ret void
+}
+
+define void @unordered_double_global_cta_volatile(double %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_double_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_double_global_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr addrspace(1) %addr syncscope("block") unordered, align 8
+ ret void
+}
+
+define void @unordered_double_global_cluster_volatile(double %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_double_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_double_global_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr addrspace(1) %addr syncscope("cluster") unordered, align 8
+ ret void
+}
+
+define void @unordered_double_global_gpu_volatile(double %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_double_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_double_global_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr addrspace(1) %addr syncscope("device") unordered, align 8
+ ret void
+}
+
+define void @unordered_i8_shared_sys(i8 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i8_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [unordered_i8_shared_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_shared_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.shared.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr addrspace(3) %addr syncscope("") unordered, align 1
+ ret void
+}
+
+define void @unordered_i8_shared_cta(i8 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i8_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [unordered_i8_shared_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_shared_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.shared.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr addrspace(3) %addr syncscope("block") unordered, align 1
+ ret void
+}
+
+define void @unordered_i8_shared_cluster(i8 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i8_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [unordered_i8_shared_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_shared_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cta.shared.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr addrspace(3) %addr syncscope("cluster") unordered, align 1
+ ret void
+}
+
+define void @unordered_i8_shared_gpu(i8 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i8_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [unordered_i8_shared_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_shared_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.shared.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr addrspace(3) %addr syncscope("device") unordered, align 1
+ ret void
+}
+
+define void @unordered_i8_shared_sys_volatile(i8 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i8_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [unordered_i8_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_shared_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr addrspace(3) %addr syncscope("") unordered, align 1
+ ret void
+}
+
+define void @unordered_i8_shared_cta_volatile(i8 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i8_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [unordered_i8_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_shared_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr addrspace(3) %addr syncscope("block") unordered, align 1
+ ret void
+}
+
+define void @unordered_i8_shared_cluster_volatile(i8 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i8_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [unordered_i8_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_shared_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr addrspace(3) %addr syncscope("cluster") unordered, align 1
+ ret void
+}
+
+define void @unordered_i8_shared_gpu_volatile(i8 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i8_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [unordered_i8_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_shared_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr addrspace(3) %addr syncscope("device") unordered, align 1
+ ret void
+}
+
+define void @unordered_i16_shared_sys(i16 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i16_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_i16_shared_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_shared_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr addrspace(3) %addr syncscope("") unordered, align 2
+ ret void
+}
+
+define void @unordered_i16_shared_cta(i16 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i16_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_i16_shared_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_shared_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr addrspace(3) %addr syncscope("block") unordered, align 2
+ ret void
+}
+
+define void @unordered_i16_shared_cluster(i16 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i16_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_i16_shared_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_shared_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cta.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr addrspace(3) %addr syncscope("cluster") unordered, align 2
+ ret void
+}
+
+define void @unordered_i16_shared_gpu(i16 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i16_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_i16_shared_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_shared_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr addrspace(3) %addr syncscope("device") unordered, align 2
+ ret void
+}
+
+define void @unordered_i16_shared_sys_volatile(i16 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i16_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_i16_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_shared_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr addrspace(3) %addr syncscope("") unordered, align 2
+ ret void
+}
+
+define void @unordered_i16_shared_cta_volatile(i16 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i16_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_i16_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_shared_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr addrspace(3) %addr syncscope("block") unordered, align 2
+ ret void
+}
+
+define void @unordered_i16_shared_cluster_volatile(i16 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i16_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_i16_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_shared_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr addrspace(3) %addr syncscope("cluster") unordered, align 2
+ ret void
+}
+
+define void @unordered_i16_shared_gpu_volatile(i16 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i16_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_i16_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_shared_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr addrspace(3) %addr syncscope("device") unordered, align 2
+ ret void
+}
+
+define void @unordered_i32_shared_sys(i32 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i32_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_i32_shared_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_shared_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr addrspace(3) %addr syncscope("") unordered, align 4
+ ret void
+}
+
+define void @unordered_i32_shared_cta(i32 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i32_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_i32_shared_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_shared_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr addrspace(3) %addr syncscope("block") unordered, align 4
+ ret void
+}
+
+define void @unordered_i32_shared_cluster(i32 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i32_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_i32_shared_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_shared_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cta.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr addrspace(3) %addr syncscope("cluster") unordered, align 4
+ ret void
+}
+
+define void @unordered_i32_shared_gpu(i32 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i32_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_i32_shared_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_shared_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr addrspace(3) %addr syncscope("device") unordered, align 4
+ ret void
+}
+
+define void @unordered_i32_shared_sys_volatile(i32 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i32_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_i32_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_shared_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr addrspace(3) %addr syncscope("") unordered, align 4
+ ret void
+}
+
+define void @unordered_i32_shared_cta_volatile(i32 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i32_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_i32_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_shared_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr addrspace(3) %addr syncscope("block") unordered, align 4
+ ret void
+}
+
+define void @unordered_i32_shared_cluster_volatile(i32 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i32_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_i32_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_shared_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr addrspace(3) %addr syncscope("cluster") unordered, align 4
+ ret void
+}
+
+define void @unordered_i32_shared_gpu_volatile(i32 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i32_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_i32_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_shared_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr addrspace(3) %addr syncscope("device") unordered, align 4
+ ret void
+}
+
+define void @unordered_i64_shared_sys(i64 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i64_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_shared_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_i64_shared_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr addrspace(3) %addr syncscope("") unordered, align 8
+ ret void
+}
+
+define void @unordered_i64_shared_cta(i64 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i64_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_shared_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_i64_shared_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr addrspace(3) %addr syncscope("block") unordered, align 8
+ ret void
+}
+
+define void @unordered_i64_shared_cluster(i64 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i64_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_shared_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_i64_shared_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cta.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr addrspace(3) %addr syncscope("cluster") unordered, align 8
+ ret void
+}
+
+define void @unordered_i64_shared_gpu(i64 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i64_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_shared_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_i64_shared_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr addrspace(3) %addr syncscope("device") unordered, align 8
+ ret void
+}
+
+define void @unordered_i64_shared_sys_volatile(i64 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i64_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_i64_shared_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr addrspace(3) %addr syncscope("") unordered, align 8
+ ret void
+}
+
+define void @unordered_i64_shared_cta_volatile(i64 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i64_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_i64_shared_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr addrspace(3) %addr syncscope("block") unordered, align 8
+ ret void
+}
+
+define void @unordered_i64_shared_cluster_volatile(i64 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i64_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_i64_shared_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr addrspace(3) %addr syncscope("cluster") unordered, align 8
+ ret void
+}
+
+define void @unordered_i64_shared_gpu_volatile(i64 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i64_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_i64_shared_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr addrspace(3) %addr syncscope("device") unordered, align 8
+ ret void
+}
+
+define void @unordered_half_shared_sys(half %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_half_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_half_shared_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_shared_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr addrspace(3) %addr syncscope("") unordered, align 2
+ ret void
+}
+
+define void @unordered_half_shared_cta(half %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_half_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_half_shared_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_shared_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr addrspace(3) %addr syncscope("block") unordered, align 2
+ ret void
+}
+
+define void @unordered_half_shared_cluster(half %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_half_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_half_shared_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_shared_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cta.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr addrspace(3) %addr syncscope("cluster") unordered, align 2
+ ret void
+}
+
+define void @unordered_half_shared_gpu(half %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_half_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_half_shared_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_shared_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr addrspace(3) %addr syncscope("device") unordered, align 2
+ ret void
+}
+
+define void @unordered_half_shared_sys_volatile(half %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_half_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_half_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_shared_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr addrspace(3) %addr syncscope("") unordered, align 2
+ ret void
+}
+
+define void @unordered_half_shared_cta_volatile(half %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_half_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_half_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_shared_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr addrspace(3) %addr syncscope("block") unordered, align 2
+ ret void
+}
+
+define void @unordered_half_shared_cluster_volatile(half %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_half_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_half_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_shared_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr addrspace(3) %addr syncscope("cluster") unordered, align 2
+ ret void
+}
+
+define void @unordered_half_shared_gpu_volatile(half %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_half_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_half_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_shared_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr addrspace(3) %addr syncscope("device") unordered, align 2
+ ret void
+}
+
+define void @unordered_bfloat_shared_sys(bfloat %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_bfloat_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_bfloat_shared_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_shared_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr addrspace(3) %addr syncscope("") unordered, align 2
+ ret void
+}
+
+define void @unordered_bfloat_shared_cta(bfloat %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_bfloat_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_bfloat_shared_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_shared_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr addrspace(3) %addr syncscope("block") unordered, align 2
+ ret void
+}
+
+define void @unordered_bfloat_shared_cluster(bfloat %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_bfloat_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_bfloat_shared_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_shared_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cta.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr addrspace(3) %addr syncscope("cluster") unordered, align 2
+ ret void
+}
+
+define void @unordered_bfloat_shared_gpu(bfloat %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_bfloat_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_bfloat_shared_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_shared_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr addrspace(3) %addr syncscope("device") unordered, align 2
+ ret void
+}
+
+define void @unordered_bfloat_shared_sys_volatile(bfloat %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_bfloat_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_bfloat_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_shared_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr addrspace(3) %addr syncscope("") unordered, align 2
+ ret void
+}
+
+define void @unordered_bfloat_shared_cta_volatile(bfloat %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_bfloat_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_bfloat_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_shared_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr addrspace(3) %addr syncscope("block") unordered, align 2
+ ret void
+}
+
+define void @unordered_bfloat_shared_cluster_volatile(bfloat %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_bfloat_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_bfloat_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_shared_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr addrspace(3) %addr syncscope("cluster") unordered, align 2
+ ret void
+}
+
+define void @unordered_bfloat_shared_gpu_volatile(bfloat %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_bfloat_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_bfloat_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_shared_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr addrspace(3) %addr syncscope("device") unordered, align 2
+ ret void
+}
+
+define void @unordered_float_shared_sys(float %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_float_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_float_shared_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_shared_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr addrspace(3) %addr syncscope("") unordered, align 4
+ ret void
+}
+
+define void @unordered_float_shared_cta(float %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_float_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_float_shared_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_shared_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr addrspace(3) %addr syncscope("block") unordered, align 4
+ ret void
+}
+
+define void @unordered_float_shared_cluster(float %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_float_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_float_shared_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_shared_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cta.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr addrspace(3) %addr syncscope("cluster") unordered, align 4
+ ret void
+}
+
+define void @unordered_float_shared_gpu(float %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_float_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_float_shared_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_shared_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr addrspace(3) %addr syncscope("device") unordered, align 4
+ ret void
+}
+
+define void @unordered_float_shared_sys_volatile(float %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_float_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_float_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_shared_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr addrspace(3) %addr syncscope("") unordered, align 4
+ ret void
+}
+
+define void @unordered_float_shared_cta_volatile(float %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_float_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_float_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_shared_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr addrspace(3) %addr syncscope("block") unordered, align 4
+ ret void
+}
+
+define void @unordered_float_shared_cluster_volatile(float %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_float_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_float_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_shared_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr addrspace(3) %addr syncscope("cluster") unordered, align 4
+ ret void
+}
+
+define void @unordered_float_shared_gpu_volatile(float %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_float_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_float_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_shared_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr addrspace(3) %addr syncscope("device") unordered, align 4
+ ret void
+}
+
+define void @unordered_double_shared_sys(double %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_double_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_shared_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_double_shared_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr addrspace(3) %addr syncscope("") unordered, align 8
+ ret void
+}
+
+define void @unordered_double_shared_cta(double %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_double_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_shared_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_double_shared_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr addrspace(3) %addr syncscope("block") unordered, align 8
+ ret void
+}
+
+define void @unordered_double_shared_cluster(double %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_double_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_shared_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_double_shared_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cta.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr addrspace(3) %addr syncscope("cluster") unordered, align 8
+ ret void
+}
+
+define void @unordered_double_shared_gpu(double %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_double_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_shared_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_double_shared_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr addrspace(3) %addr syncscope("device") unordered, align 8
+ ret void
+}
+
+define void @unordered_double_shared_sys_volatile(double %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_double_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_double_shared_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr addrspace(3) %addr syncscope("") unordered, align 8
+ ret void
+}
+
+define void @unordered_double_shared_cta_volatile(double %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_double_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_double_shared_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr addrspace(3) %addr syncscope("block") unordered, align 8
+ ret void
+}
+
+define void @unordered_double_shared_cluster_volatile(double %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_double_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_double_shared_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr addrspace(3) %addr syncscope("cluster") unordered, align 8
+ ret void
+}
+
+define void @unordered_double_shared_gpu_volatile(double %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_double_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_double_shared_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr addrspace(3) %addr syncscope("device") unordered, align 8
+ ret void
+}
+
+define void @unordered_i8_local_sys(i8 %val) {
+; CHECK-LABEL: unordered_i8_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot256[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot256;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b8 %rs1, [unordered_i8_local_sys_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b8 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i8 %val, ptr addrspace(5) %slot syncscope("") unordered, align 1
+ ret void
+}
+
+define void @unordered_i8_local_cta(i8 %val) {
+; CHECK-LABEL: unordered_i8_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot257[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot257;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b8 %rs1, [unordered_i8_local_cta_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b8 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i8 %val, ptr addrspace(5) %slot syncscope("block") unordered, align 1
+ ret void
+}
+
+define void @unordered_i8_local_cluster(i8 %val) {
+; CHECK-LABEL: unordered_i8_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot258[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot258;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b8 %rs1, [unordered_i8_local_cluster_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b8 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i8 %val, ptr addrspace(5) %slot syncscope("cluster") unordered, align 1
+ ret void
+}
+
+define void @unordered_i8_local_gpu(i8 %val) {
+; CHECK-LABEL: unordered_i8_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot259[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot259;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b8 %rs1, [unordered_i8_local_gpu_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b8 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i8 %val, ptr addrspace(5) %slot syncscope("device") unordered, align 1
+ ret void
+}
+
+define void @unordered_i8_local_sys_volatile(i8 %val) {
+; CHECK-LABEL: unordered_i8_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot260[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot260;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b8 %rs1, [unordered_i8_local_sys_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b8 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i8 %val, ptr addrspace(5) %slot syncscope("") unordered, align 1
+ ret void
+}
+
+define void @unordered_i8_local_cta_volatile(i8 %val) {
+; CHECK-LABEL: unordered_i8_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot261[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot261;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b8 %rs1, [unordered_i8_local_cta_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b8 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i8 %val, ptr addrspace(5) %slot syncscope("block") unordered, align 1
+ ret void
+}
+
+define void @unordered_i8_local_cluster_volatile(i8 %val) {
+; CHECK-LABEL: unordered_i8_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot262[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot262;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b8 %rs1, [unordered_i8_local_cluster_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b8 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i8 %val, ptr addrspace(5) %slot syncscope("cluster") unordered, align 1
+ ret void
+}
+
+define void @unordered_i8_local_gpu_volatile(i8 %val) {
+; CHECK-LABEL: unordered_i8_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot263[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot263;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b8 %rs1, [unordered_i8_local_gpu_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b8 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i8 %val, ptr addrspace(5) %slot syncscope("device") unordered, align 1
+ ret void
+}
+
+define void @unordered_i16_local_sys(i16 %val) {
+; CHECK-LABEL: unordered_i16_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot264[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot264;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_i16_local_sys_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i16 %val, ptr addrspace(5) %slot syncscope("") unordered, align 2
+ ret void
+}
+
+define void @unordered_i16_local_cta(i16 %val) {
+; CHECK-LABEL: unordered_i16_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot265[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot265;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_i16_local_cta_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i16 %val, ptr addrspace(5) %slot syncscope("block") unordered, align 2
+ ret void
+}
+
+define void @unordered_i16_local_cluster(i16 %val) {
+; CHECK-LABEL: unordered_i16_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot266[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot266;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_i16_local_cluster_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i16 %val, ptr addrspace(5) %slot syncscope("cluster") unordered, align 2
+ ret void
+}
+
+define void @unordered_i16_local_gpu(i16 %val) {
+; CHECK-LABEL: unordered_i16_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot267[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot267;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_i16_local_gpu_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i16 %val, ptr addrspace(5) %slot syncscope("device") unordered, align 2
+ ret void
+}
+
+define void @unordered_i16_local_sys_volatile(i16 %val) {
+; CHECK-LABEL: unordered_i16_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot268[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot268;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_i16_local_sys_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i16 %val, ptr addrspace(5) %slot syncscope("") unordered, align 2
+ ret void
+}
+
+define void @unordered_i16_local_cta_volatile(i16 %val) {
+; CHECK-LABEL: unordered_i16_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot269[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot269;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_i16_local_cta_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i16 %val, ptr addrspace(5) %slot syncscope("block") unordered, align 2
+ ret void
+}
+
+define void @unordered_i16_local_cluster_volatile(i16 %val) {
+; CHECK-LABEL: unordered_i16_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot270[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot270;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_i16_local_cluster_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i16 %val, ptr addrspace(5) %slot syncscope("cluster") unordered, align 2
+ ret void
+}
+
+define void @unordered_i16_local_gpu_volatile(i16 %val) {
+; CHECK-LABEL: unordered_i16_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot271[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot271;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_i16_local_gpu_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i16 %val, ptr addrspace(5) %slot syncscope("device") unordered, align 2
+ ret void
+}
+
+define void @unordered_i32_local_sys(i32 %val) {
+; CHECK-LABEL: unordered_i32_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot272[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot272;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [unordered_i32_local_sys_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i32 %val, ptr addrspace(5) %slot syncscope("") unordered, align 4
+ ret void
+}
+
+define void @unordered_i32_local_cta(i32 %val) {
+; CHECK-LABEL: unordered_i32_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot273[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot273;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [unordered_i32_local_cta_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i32 %val, ptr addrspace(5) %slot syncscope("block") unordered, align 4
+ ret void
+}
+
+define void @unordered_i32_local_cluster(i32 %val) {
+; CHECK-LABEL: unordered_i32_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot274[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot274;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [unordered_i32_local_cluster_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i32 %val, ptr addrspace(5) %slot syncscope("cluster") unordered, align 4
+ ret void
+}
+
+define void @unordered_i32_local_gpu(i32 %val) {
+; CHECK-LABEL: unordered_i32_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot275[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot275;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [unordered_i32_local_gpu_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i32 %val, ptr addrspace(5) %slot syncscope("device") unordered, align 4
+ ret void
+}
+
+define void @unordered_i32_local_sys_volatile(i32 %val) {
+; CHECK-LABEL: unordered_i32_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot276[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot276;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [unordered_i32_local_sys_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i32 %val, ptr addrspace(5) %slot syncscope("") unordered, align 4
+ ret void
+}
+
+define void @unordered_i32_local_cta_volatile(i32 %val) {
+; CHECK-LABEL: unordered_i32_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot277[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot277;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [unordered_i32_local_cta_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i32 %val, ptr addrspace(5) %slot syncscope("block") unordered, align 4
+ ret void
+}
+
+define void @unordered_i32_local_cluster_volatile(i32 %val) {
+; CHECK-LABEL: unordered_i32_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot278[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot278;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [unordered_i32_local_cluster_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i32 %val, ptr addrspace(5) %slot syncscope("cluster") unordered, align 4
+ ret void
+}
+
+define void @unordered_i32_local_gpu_volatile(i32 %val) {
+; CHECK-LABEL: unordered_i32_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot279[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot279;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [unordered_i32_local_gpu_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i32 %val, ptr addrspace(5) %slot syncscope("device") unordered, align 4
+ ret void
+}
+
+define void @unordered_i64_local_sys(i64 %val) {
+; CHECK-LABEL: unordered_i64_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot280[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot280;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_local_sys_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i64 %val, ptr addrspace(5) %slot syncscope("") unordered, align 8
+ ret void
+}
+
+define void @unordered_i64_local_cta(i64 %val) {
+; CHECK-LABEL: unordered_i64_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot281[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot281;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_local_cta_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i64 %val, ptr addrspace(5) %slot syncscope("block") unordered, align 8
+ ret void
+}
+
+define void @unordered_i64_local_cluster(i64 %val) {
+; CHECK-LABEL: unordered_i64_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot282[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot282;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_local_cluster_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i64 %val, ptr addrspace(5) %slot syncscope("cluster") unordered, align 8
+ ret void
+}
+
+define void @unordered_i64_local_gpu(i64 %val) {
+; CHECK-LABEL: unordered_i64_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot283[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot283;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_local_gpu_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i64 %val, ptr addrspace(5) %slot syncscope("device") unordered, align 8
+ ret void
+}
+
+define void @unordered_i64_local_sys_volatile(i64 %val) {
+; CHECK-LABEL: unordered_i64_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot284[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot284;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_local_sys_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i64 %val, ptr addrspace(5) %slot syncscope("") unordered, align 8
+ ret void
+}
+
+define void @unordered_i64_local_cta_volatile(i64 %val) {
+; CHECK-LABEL: unordered_i64_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot285[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot285;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_local_cta_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i64 %val, ptr addrspace(5) %slot syncscope("block") unordered, align 8
+ ret void
+}
+
+define void @unordered_i64_local_cluster_volatile(i64 %val) {
+; CHECK-LABEL: unordered_i64_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot286[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot286;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_local_cluster_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i64 %val, ptr addrspace(5) %slot syncscope("cluster") unordered, align 8
+ ret void
+}
+
+define void @unordered_i64_local_gpu_volatile(i64 %val) {
+; CHECK-LABEL: unordered_i64_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot287[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot287;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_local_gpu_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i64 %val, ptr addrspace(5) %slot syncscope("device") unordered, align 8
+ ret void
+}
+
+define void @unordered_half_local_sys(half %val) {
+; CHECK-LABEL: unordered_half_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot288[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot288;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_half_local_sys_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic half %val, ptr addrspace(5) %slot syncscope("") unordered, align 2
+ ret void
+}
+
+define void @unordered_half_local_cta(half %val) {
+; CHECK-LABEL: unordered_half_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot289[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot289;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_half_local_cta_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic half %val, ptr addrspace(5) %slot syncscope("block") unordered, align 2
+ ret void
+}
+
+define void @unordered_half_local_cluster(half %val) {
+; CHECK-LABEL: unordered_half_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot290[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot290;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_half_local_cluster_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic half %val, ptr addrspace(5) %slot syncscope("cluster") unordered, align 2
+ ret void
+}
+
+define void @unordered_half_local_gpu(half %val) {
+; CHECK-LABEL: unordered_half_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot291[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot291;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_half_local_gpu_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic half %val, ptr addrspace(5) %slot syncscope("device") unordered, align 2
+ ret void
+}
+
+define void @unordered_half_local_sys_volatile(half %val) {
+; CHECK-LABEL: unordered_half_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot292[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot292;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_half_local_sys_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile half %val, ptr addrspace(5) %slot syncscope("") unordered, align 2
+ ret void
+}
+
+define void @unordered_half_local_cta_volatile(half %val) {
+; CHECK-LABEL: unordered_half_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot293[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot293;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_half_local_cta_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile half %val, ptr addrspace(5) %slot syncscope("block") unordered, align 2
+ ret void
+}
+
+define void @unordered_half_local_cluster_volatile(half %val) {
+; CHECK-LABEL: unordered_half_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot294[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot294;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_half_local_cluster_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile half %val, ptr addrspace(5) %slot syncscope("cluster") unordered, align 2
+ ret void
+}
+
+define void @unordered_half_local_gpu_volatile(half %val) {
+; CHECK-LABEL: unordered_half_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot295[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot295;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_half_local_gpu_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile half %val, ptr addrspace(5) %slot syncscope("device") unordered, align 2
+ ret void
+}
+
+define void @unordered_bfloat_local_sys(bfloat %val) {
+; CHECK-LABEL: unordered_bfloat_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot296[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot296;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_bfloat_local_sys_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic bfloat %val, ptr addrspace(5) %slot syncscope("") unordered, align 2
+ ret void
+}
+
+define void @unordered_bfloat_local_cta(bfloat %val) {
+; CHECK-LABEL: unordered_bfloat_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot297[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot297;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_bfloat_local_cta_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic bfloat %val, ptr addrspace(5) %slot syncscope("block") unordered, align 2
+ ret void
+}
+
+define void @unordered_bfloat_local_cluster(bfloat %val) {
+; CHECK-LABEL: unordered_bfloat_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot298[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot298;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_bfloat_local_cluster_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic bfloat %val, ptr addrspace(5) %slot syncscope("cluster") unordered, align 2
+ ret void
+}
+
+define void @unordered_bfloat_local_gpu(bfloat %val) {
+; CHECK-LABEL: unordered_bfloat_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot299[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot299;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_bfloat_local_gpu_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic bfloat %val, ptr addrspace(5) %slot syncscope("device") unordered, align 2
+ ret void
+}
+
+define void @unordered_bfloat_local_sys_volatile(bfloat %val) {
+; CHECK-LABEL: unordered_bfloat_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot300[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot300;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_bfloat_local_sys_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile bfloat %val, ptr addrspace(5) %slot syncscope("") unordered, align 2
+ ret void
+}
+
+define void @unordered_bfloat_local_cta_volatile(bfloat %val) {
+; CHECK-LABEL: unordered_bfloat_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot301[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot301;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_bfloat_local_cta_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile bfloat %val, ptr addrspace(5) %slot syncscope("block") unordered, align 2
+ ret void
+}
+
+define void @unordered_bfloat_local_cluster_volatile(bfloat %val) {
+; CHECK-LABEL: unordered_bfloat_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot302[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot302;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_bfloat_local_cluster_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile bfloat %val, ptr addrspace(5) %slot syncscope("cluster") unordered, align 2
+ ret void
+}
+
+define void @unordered_bfloat_local_gpu_volatile(bfloat %val) {
+; CHECK-LABEL: unordered_bfloat_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot303[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot303;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_bfloat_local_gpu_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile bfloat %val, ptr addrspace(5) %slot syncscope("device") unordered, align 2
+ ret void
+}
+
+define void @unordered_float_local_sys(float %val) {
+; CHECK-LABEL: unordered_float_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot304[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot304;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [unordered_float_local_sys_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic float %val, ptr addrspace(5) %slot syncscope("") unordered, align 4
+ ret void
+}
+
+define void @unordered_float_local_cta(float %val) {
+; CHECK-LABEL: unordered_float_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot305[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot305;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [unordered_float_local_cta_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic float %val, ptr addrspace(5) %slot syncscope("block") unordered, align 4
+ ret void
+}
+
+define void @unordered_float_local_cluster(float %val) {
+; CHECK-LABEL: unordered_float_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot306[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot306;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [unordered_float_local_cluster_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic float %val, ptr addrspace(5) %slot syncscope("cluster") unordered, align 4
+ ret void
+}
+
+define void @unordered_float_local_gpu(float %val) {
+; CHECK-LABEL: unordered_float_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot307[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot307;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [unordered_float_local_gpu_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic float %val, ptr addrspace(5) %slot syncscope("device") unordered, align 4
+ ret void
+}
+
+define void @unordered_float_local_sys_volatile(float %val) {
+; CHECK-LABEL: unordered_float_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot308[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot308;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [unordered_float_local_sys_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile float %val, ptr addrspace(5) %slot syncscope("") unordered, align 4
+ ret void
+}
+
+define void @unordered_float_local_cta_volatile(float %val) {
+; CHECK-LABEL: unordered_float_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot309[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot309;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [unordered_float_local_cta_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile float %val, ptr addrspace(5) %slot syncscope("block") unordered, align 4
+ ret void
+}
+
+define void @unordered_float_local_cluster_volatile(float %val) {
+; CHECK-LABEL: unordered_float_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot310[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot310;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [unordered_float_local_cluster_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile float %val, ptr addrspace(5) %slot syncscope("cluster") unordered, align 4
+ ret void
+}
+
+define void @unordered_float_local_gpu_volatile(float %val) {
+; CHECK-LABEL: unordered_float_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot311[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot311;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [unordered_float_local_gpu_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile float %val, ptr addrspace(5) %slot syncscope("device") unordered, align 4
+ ret void
+}
+
+define void @unordered_double_local_sys(double %val) {
+; CHECK-LABEL: unordered_double_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot312[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot312;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_local_sys_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic double %val, ptr addrspace(5) %slot syncscope("") unordered, align 8
+ ret void
+}
+
+define void @unordered_double_local_cta(double %val) {
+; CHECK-LABEL: unordered_double_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot313[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot313;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_local_cta_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic double %val, ptr addrspace(5) %slot syncscope("block") unordered, align 8
+ ret void
+}
+
+define void @unordered_double_local_cluster(double %val) {
+; CHECK-LABEL: unordered_double_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot314[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot314;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_local_cluster_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic double %val, ptr addrspace(5) %slot syncscope("cluster") unordered, align 8
+ ret void
+}
+
+define void @unordered_double_local_gpu(double %val) {
+; CHECK-LABEL: unordered_double_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot315[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot315;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_local_gpu_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic double %val, ptr addrspace(5) %slot syncscope("device") unordered, align 8
+ ret void
+}
+
+define void @unordered_double_local_sys_volatile(double %val) {
+; CHECK-LABEL: unordered_double_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot316[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot316;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_local_sys_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile double %val, ptr addrspace(5) %slot syncscope("") unordered, align 8
+ ret void
+}
+
+define void @unordered_double_local_cta_volatile(double %val) {
+; CHECK-LABEL: unordered_double_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot317[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot317;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_local_cta_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile double %val, ptr addrspace(5) %slot syncscope("block") unordered, align 8
+ ret void
+}
+
+define void @unordered_double_local_cluster_volatile(double %val) {
+; CHECK-LABEL: unordered_double_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot318[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot318;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_local_cluster_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile double %val, ptr addrspace(5) %slot syncscope("cluster") unordered, align 8
+ ret void
+}
+
+define void @unordered_double_local_gpu_volatile(double %val) {
+; CHECK-LABEL: unordered_double_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot319[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot319;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_local_gpu_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile double %val, ptr addrspace(5) %slot syncscope("device") unordered, align 8
+ ret void
+}
+
+define void @monotonic_i8_generic_sys(i8 %val, ptr %addr) {
+; CHECK-LABEL: monotonic_i8_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [monotonic_i8_generic_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_generic_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr %addr syncscope("") monotonic, align 1
+ ret void
+}
+
+define void @monotonic_i8_generic_cta(i8 %val, ptr %addr) {
+; CHECK-LABEL: monotonic_i8_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [monotonic_i8_generic_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_generic_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr %addr syncscope("block") monotonic, align 1
+ ret void
+}
+
+define void @monotonic_i8_generic_cluster(i8 %val, ptr %addr) {
+; CHECK-LABEL: monotonic_i8_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [monotonic_i8_generic_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_generic_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cta.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr %addr syncscope("cluster") monotonic, align 1
+ ret void
+}
+
+define void @monotonic_i8_generic_gpu(i8 %val, ptr %addr) {
+; CHECK-LABEL: monotonic_i8_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [monotonic_i8_generic_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_generic_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr %addr syncscope("device") monotonic, align 1
+ ret void
+}
+
+define void @monotonic_i8_generic_sys_volatile(i8 %val, ptr %addr) {
+; CHECK-LABEL: monotonic_i8_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [monotonic_i8_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_generic_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr %addr syncscope("") monotonic, align 1
+ ret void
+}
+
+define void @monotonic_i8_generic_cta_volatile(i8 %val, ptr %addr) {
+; CHECK-LABEL: monotonic_i8_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [monotonic_i8_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_generic_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr %addr syncscope("block") monotonic, align 1
+ ret void
+}
+
+define void @monotonic_i8_generic_cluster_volatile(i8 %val, ptr %addr) {
+; CHECK-LABEL: monotonic_i8_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [monotonic_i8_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_generic_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr %addr syncscope("cluster") monotonic, align 1
+ ret void
+}
+
+define void @monotonic_i8_generic_gpu_volatile(i8 %val, ptr %addr) {
+; CHECK-LABEL: monotonic_i8_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [monotonic_i8_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_generic_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr %addr syncscope("device") monotonic, align 1
+ ret void
+}
+
+define void @monotonic_i16_generic_sys(i16 %val, ptr %addr) {
+; CHECK-LABEL: monotonic_i16_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_i16_generic_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_generic_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr %addr syncscope("") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_i16_generic_cta(i16 %val, ptr %addr) {
+; CHECK-LABEL: monotonic_i16_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_i16_generic_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_generic_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr %addr syncscope("block") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_i16_generic_cluster(i16 %val, ptr %addr) {
+; CHECK-LABEL: monotonic_i16_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_i16_generic_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_generic_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cta.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr %addr syncscope("cluster") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_i16_generic_gpu(i16 %val, ptr %addr) {
+; CHECK-LABEL: monotonic_i16_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_i16_generic_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_generic_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr %addr syncscope("device") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_i16_generic_sys_volatile(i16 %val, ptr %addr) {
+; CHECK-LABEL: monotonic_i16_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_i16_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_generic_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr %addr syncscope("") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_i16_generic_cta_volatile(i16 %val, ptr %addr) {
+; CHECK-LABEL: monotonic_i16_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_i16_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_generic_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr %addr syncscope("block") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_i16_generic_cluster_volatile(i16 %val, ptr %addr) {
+; CHECK-LABEL: monotonic_i16_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_i16_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_generic_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr %addr syncscope("cluster") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_i16_generic_gpu_volatile(i16 %val, ptr %addr) {
+; CHECK-LABEL: monotonic_i16_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_i16_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_generic_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr %addr syncscope("device") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_i32_generic_sys(i32 %val, ptr %addr) {
+; CHECK-LABEL: monotonic_i32_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_i32_generic_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_generic_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr %addr syncscope("") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_i32_generic_cta(i32 %val, ptr %addr) {
+; CHECK-LABEL: monotonic_i32_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_i32_generic_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_generic_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr %addr syncscope("block") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_i32_generic_cluster(i32 %val, ptr %addr) {
+; CHECK-LABEL: monotonic_i32_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_i32_generic_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_generic_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cta.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr %addr syncscope("cluster") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_i32_generic_gpu(i32 %val, ptr %addr) {
+; CHECK-LABEL: monotonic_i32_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_i32_generic_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_generic_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr %addr syncscope("device") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_i32_generic_sys_volatile(i32 %val, ptr %addr) {
+; CHECK-LABEL: monotonic_i32_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_i32_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_generic_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr %addr syncscope("") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_i32_generic_cta_volatile(i32 %val, ptr %addr) {
+; CHECK-LABEL: monotonic_i32_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_i32_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_generic_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr %addr syncscope("block") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_i32_generic_cluster_volatile(i32 %val, ptr %addr) {
+; CHECK-LABEL: monotonic_i32_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_i32_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_generic_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr %addr syncscope("cluster") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_i32_generic_gpu_volatile(i32 %val, ptr %addr) {
+; CHECK-LABEL: monotonic_i32_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_i32_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_generic_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr %addr syncscope("device") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_i64_generic_sys(i64 %val, ptr %addr) {
+; CHECK-LABEL: monotonic_i64_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_generic_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_i64_generic_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr %addr syncscope("") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_i64_generic_cta(i64 %val, ptr %addr) {
+; CHECK-LABEL: monotonic_i64_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_generic_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_i64_generic_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr %addr syncscope("block") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_i64_generic_cluster(i64 %val, ptr %addr) {
+; CHECK-LABEL: monotonic_i64_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_generic_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_i64_generic_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cta.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr %addr syncscope("cluster") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_i64_generic_gpu(i64 %val, ptr %addr) {
+; CHECK-LABEL: monotonic_i64_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_generic_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_i64_generic_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr %addr syncscope("device") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_i64_generic_sys_volatile(i64 %val, ptr %addr) {
+; CHECK-LABEL: monotonic_i64_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_i64_generic_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr %addr syncscope("") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_i64_generic_cta_volatile(i64 %val, ptr %addr) {
+; CHECK-LABEL: monotonic_i64_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_i64_generic_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr %addr syncscope("block") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_i64_generic_cluster_volatile(i64 %val, ptr %addr) {
+; CHECK-LABEL: monotonic_i64_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_i64_generic_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr %addr syncscope("cluster") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_i64_generic_gpu_volatile(i64 %val, ptr %addr) {
+; CHECK-LABEL: monotonic_i64_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_i64_generic_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr %addr syncscope("device") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_half_generic_sys(half %val, ptr %addr) {
+; CHECK-LABEL: monotonic_half_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_half_generic_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_generic_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr %addr syncscope("") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_half_generic_cta(half %val, ptr %addr) {
+; CHECK-LABEL: monotonic_half_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_half_generic_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_generic_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr %addr syncscope("block") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_half_generic_cluster(half %val, ptr %addr) {
+; CHECK-LABEL: monotonic_half_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_half_generic_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_generic_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cta.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr %addr syncscope("cluster") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_half_generic_gpu(half %val, ptr %addr) {
+; CHECK-LABEL: monotonic_half_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_half_generic_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_generic_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr %addr syncscope("device") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_half_generic_sys_volatile(half %val, ptr %addr) {
+; CHECK-LABEL: monotonic_half_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_half_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_generic_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr %addr syncscope("") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_half_generic_cta_volatile(half %val, ptr %addr) {
+; CHECK-LABEL: monotonic_half_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_half_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_generic_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr %addr syncscope("block") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_half_generic_cluster_volatile(half %val, ptr %addr) {
+; CHECK-LABEL: monotonic_half_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_half_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_generic_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr %addr syncscope("cluster") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_half_generic_gpu_volatile(half %val, ptr %addr) {
+; CHECK-LABEL: monotonic_half_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_half_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_generic_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr %addr syncscope("device") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_bfloat_generic_sys(bfloat %val, ptr %addr) {
+; CHECK-LABEL: monotonic_bfloat_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_bfloat_generic_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_generic_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr %addr syncscope("") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_bfloat_generic_cta(bfloat %val, ptr %addr) {
+; CHECK-LABEL: monotonic_bfloat_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_bfloat_generic_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_generic_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr %addr syncscope("block") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_bfloat_generic_cluster(bfloat %val, ptr %addr) {
+; CHECK-LABEL: monotonic_bfloat_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_bfloat_generic_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_generic_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cta.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr %addr syncscope("cluster") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_bfloat_generic_gpu(bfloat %val, ptr %addr) {
+; CHECK-LABEL: monotonic_bfloat_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_bfloat_generic_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_generic_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr %addr syncscope("device") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_bfloat_generic_sys_volatile(bfloat %val, ptr %addr) {
+; CHECK-LABEL: monotonic_bfloat_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_bfloat_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_generic_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr %addr syncscope("") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_bfloat_generic_cta_volatile(bfloat %val, ptr %addr) {
+; CHECK-LABEL: monotonic_bfloat_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_bfloat_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_generic_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr %addr syncscope("block") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_bfloat_generic_cluster_volatile(bfloat %val, ptr %addr) {
+; CHECK-LABEL: monotonic_bfloat_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_bfloat_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_generic_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr %addr syncscope("cluster") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_bfloat_generic_gpu_volatile(bfloat %val, ptr %addr) {
+; CHECK-LABEL: monotonic_bfloat_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_bfloat_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_generic_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr %addr syncscope("device") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_float_generic_sys(float %val, ptr %addr) {
+; CHECK-LABEL: monotonic_float_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_float_generic_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_generic_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr %addr syncscope("") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_float_generic_cta(float %val, ptr %addr) {
+; CHECK-LABEL: monotonic_float_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_float_generic_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_generic_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr %addr syncscope("block") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_float_generic_cluster(float %val, ptr %addr) {
+; CHECK-LABEL: monotonic_float_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_float_generic_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_generic_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cta.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr %addr syncscope("cluster") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_float_generic_gpu(float %val, ptr %addr) {
+; CHECK-LABEL: monotonic_float_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_float_generic_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_generic_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr %addr syncscope("device") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_float_generic_sys_volatile(float %val, ptr %addr) {
+; CHECK-LABEL: monotonic_float_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_float_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_generic_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr %addr syncscope("") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_float_generic_cta_volatile(float %val, ptr %addr) {
+; CHECK-LABEL: monotonic_float_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_float_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_generic_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr %addr syncscope("block") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_float_generic_cluster_volatile(float %val, ptr %addr) {
+; CHECK-LABEL: monotonic_float_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_float_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_generic_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr %addr syncscope("cluster") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_float_generic_gpu_volatile(float %val, ptr %addr) {
+; CHECK-LABEL: monotonic_float_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_float_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_generic_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr %addr syncscope("device") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_double_generic_sys(double %val, ptr %addr) {
+; CHECK-LABEL: monotonic_double_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_generic_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_double_generic_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr %addr syncscope("") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_double_generic_cta(double %val, ptr %addr) {
+; CHECK-LABEL: monotonic_double_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_generic_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_double_generic_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr %addr syncscope("block") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_double_generic_cluster(double %val, ptr %addr) {
+; CHECK-LABEL: monotonic_double_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_generic_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_double_generic_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cta.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr %addr syncscope("cluster") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_double_generic_gpu(double %val, ptr %addr) {
+; CHECK-LABEL: monotonic_double_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_generic_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_double_generic_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr %addr syncscope("device") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_double_generic_sys_volatile(double %val, ptr %addr) {
+; CHECK-LABEL: monotonic_double_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_double_generic_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr %addr syncscope("") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_double_generic_cta_volatile(double %val, ptr %addr) {
+; CHECK-LABEL: monotonic_double_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_double_generic_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr %addr syncscope("block") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_double_generic_cluster_volatile(double %val, ptr %addr) {
+; CHECK-LABEL: monotonic_double_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_double_generic_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr %addr syncscope("cluster") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_double_generic_gpu_volatile(double %val, ptr %addr) {
+; CHECK-LABEL: monotonic_double_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_double_generic_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr %addr syncscope("device") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_i8_global_sys(i8 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i8_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [monotonic_i8_global_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_global_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.global.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr addrspace(1) %addr syncscope("") monotonic, align 1
+ ret void
+}
+
+define void @monotonic_i8_global_cta(i8 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i8_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [monotonic_i8_global_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_global_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.global.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr addrspace(1) %addr syncscope("block") monotonic, align 1
+ ret void
+}
+
+define void @monotonic_i8_global_cluster(i8 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i8_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [monotonic_i8_global_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_global_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cta.global.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr addrspace(1) %addr syncscope("cluster") monotonic, align 1
+ ret void
+}
+
+define void @monotonic_i8_global_gpu(i8 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i8_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [monotonic_i8_global_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_global_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.global.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr addrspace(1) %addr syncscope("device") monotonic, align 1
+ ret void
+}
+
+define void @monotonic_i8_global_sys_volatile(i8 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i8_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [monotonic_i8_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_global_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr addrspace(1) %addr syncscope("") monotonic, align 1
+ ret void
+}
+
+define void @monotonic_i8_global_cta_volatile(i8 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i8_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [monotonic_i8_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_global_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr addrspace(1) %addr syncscope("block") monotonic, align 1
+ ret void
+}
+
+define void @monotonic_i8_global_cluster_volatile(i8 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i8_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [monotonic_i8_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_global_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr addrspace(1) %addr syncscope("cluster") monotonic, align 1
+ ret void
+}
+
+define void @monotonic_i8_global_gpu_volatile(i8 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i8_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [monotonic_i8_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_global_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr addrspace(1) %addr syncscope("device") monotonic, align 1
+ ret void
+}
+
+define void @monotonic_i16_global_sys(i16 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i16_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_i16_global_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_global_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr addrspace(1) %addr syncscope("") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_i16_global_cta(i16 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i16_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_i16_global_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_global_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr addrspace(1) %addr syncscope("block") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_i16_global_cluster(i16 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i16_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_i16_global_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_global_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cta.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr addrspace(1) %addr syncscope("cluster") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_i16_global_gpu(i16 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i16_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_i16_global_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_global_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr addrspace(1) %addr syncscope("device") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_i16_global_sys_volatile(i16 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i16_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_i16_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_global_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr addrspace(1) %addr syncscope("") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_i16_global_cta_volatile(i16 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i16_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_i16_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_global_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr addrspace(1) %addr syncscope("block") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_i16_global_cluster_volatile(i16 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i16_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_i16_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_global_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr addrspace(1) %addr syncscope("cluster") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_i16_global_gpu_volatile(i16 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i16_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_i16_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_global_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr addrspace(1) %addr syncscope("device") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_i32_global_sys(i32 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i32_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_i32_global_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_global_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr addrspace(1) %addr syncscope("") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_i32_global_cta(i32 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i32_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_i32_global_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_global_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr addrspace(1) %addr syncscope("block") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_i32_global_cluster(i32 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i32_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_i32_global_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_global_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cta.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr addrspace(1) %addr syncscope("cluster") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_i32_global_gpu(i32 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i32_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_i32_global_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_global_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr addrspace(1) %addr syncscope("device") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_i32_global_sys_volatile(i32 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i32_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_i32_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_global_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr addrspace(1) %addr syncscope("") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_i32_global_cta_volatile(i32 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i32_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_i32_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_global_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr addrspace(1) %addr syncscope("block") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_i32_global_cluster_volatile(i32 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i32_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_i32_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_global_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr addrspace(1) %addr syncscope("cluster") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_i32_global_gpu_volatile(i32 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i32_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_i32_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_global_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr addrspace(1) %addr syncscope("device") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_i64_global_sys(i64 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i64_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_global_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_i64_global_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr addrspace(1) %addr syncscope("") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_i64_global_cta(i64 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i64_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_global_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_i64_global_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr addrspace(1) %addr syncscope("block") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_i64_global_cluster(i64 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i64_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_global_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_i64_global_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cta.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr addrspace(1) %addr syncscope("cluster") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_i64_global_gpu(i64 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i64_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_global_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_i64_global_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr addrspace(1) %addr syncscope("device") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_i64_global_sys_volatile(i64 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i64_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_i64_global_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr addrspace(1) %addr syncscope("") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_i64_global_cta_volatile(i64 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i64_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_i64_global_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr addrspace(1) %addr syncscope("block") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_i64_global_cluster_volatile(i64 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i64_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_i64_global_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr addrspace(1) %addr syncscope("cluster") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_i64_global_gpu_volatile(i64 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i64_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_i64_global_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr addrspace(1) %addr syncscope("device") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_half_global_sys(half %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_half_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_half_global_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_global_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr addrspace(1) %addr syncscope("") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_half_global_cta(half %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_half_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_half_global_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_global_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr addrspace(1) %addr syncscope("block") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_half_global_cluster(half %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_half_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_half_global_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_global_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cta.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr addrspace(1) %addr syncscope("cluster") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_half_global_gpu(half %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_half_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_half_global_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_global_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr addrspace(1) %addr syncscope("device") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_half_global_sys_volatile(half %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_half_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_half_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_global_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr addrspace(1) %addr syncscope("") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_half_global_cta_volatile(half %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_half_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_half_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_global_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr addrspace(1) %addr syncscope("block") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_half_global_cluster_volatile(half %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_half_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_half_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_global_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr addrspace(1) %addr syncscope("cluster") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_half_global_gpu_volatile(half %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_half_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_half_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_global_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr addrspace(1) %addr syncscope("device") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_bfloat_global_sys(bfloat %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_bfloat_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_bfloat_global_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_global_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr addrspace(1) %addr syncscope("") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_bfloat_global_cta(bfloat %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_bfloat_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_bfloat_global_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_global_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr addrspace(1) %addr syncscope("block") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_bfloat_global_cluster(bfloat %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_bfloat_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_bfloat_global_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_global_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cta.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr addrspace(1) %addr syncscope("cluster") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_bfloat_global_gpu(bfloat %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_bfloat_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_bfloat_global_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_global_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr addrspace(1) %addr syncscope("device") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_bfloat_global_sys_volatile(bfloat %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_bfloat_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_bfloat_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_global_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr addrspace(1) %addr syncscope("") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_bfloat_global_cta_volatile(bfloat %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_bfloat_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_bfloat_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_global_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr addrspace(1) %addr syncscope("block") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_bfloat_global_cluster_volatile(bfloat %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_bfloat_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_bfloat_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_global_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr addrspace(1) %addr syncscope("cluster") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_bfloat_global_gpu_volatile(bfloat %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_bfloat_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_bfloat_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_global_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr addrspace(1) %addr syncscope("device") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_float_global_sys(float %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_float_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_float_global_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_global_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr addrspace(1) %addr syncscope("") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_float_global_cta(float %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_float_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_float_global_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_global_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr addrspace(1) %addr syncscope("block") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_float_global_cluster(float %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_float_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_float_global_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_global_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cta.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr addrspace(1) %addr syncscope("cluster") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_float_global_gpu(float %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_float_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_float_global_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_global_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr addrspace(1) %addr syncscope("device") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_float_global_sys_volatile(float %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_float_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_float_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_global_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr addrspace(1) %addr syncscope("") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_float_global_cta_volatile(float %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_float_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_float_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_global_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr addrspace(1) %addr syncscope("block") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_float_global_cluster_volatile(float %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_float_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_float_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_global_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr addrspace(1) %addr syncscope("cluster") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_float_global_gpu_volatile(float %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_float_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_float_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_global_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr addrspace(1) %addr syncscope("device") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_double_global_sys(double %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_double_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_global_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_double_global_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr addrspace(1) %addr syncscope("") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_double_global_cta(double %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_double_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_global_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_double_global_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr addrspace(1) %addr syncscope("block") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_double_global_cluster(double %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_double_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_global_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_double_global_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cta.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr addrspace(1) %addr syncscope("cluster") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_double_global_gpu(double %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_double_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_global_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_double_global_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr addrspace(1) %addr syncscope("device") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_double_global_sys_volatile(double %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_double_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_double_global_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr addrspace(1) %addr syncscope("") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_double_global_cta_volatile(double %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_double_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_double_global_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr addrspace(1) %addr syncscope("block") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_double_global_cluster_volatile(double %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_double_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_double_global_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr addrspace(1) %addr syncscope("cluster") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_double_global_gpu_volatile(double %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_double_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_double_global_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr addrspace(1) %addr syncscope("device") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_i8_shared_sys(i8 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i8_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [monotonic_i8_shared_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_shared_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.shared.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr addrspace(3) %addr syncscope("") monotonic, align 1
+ ret void
+}
+
+define void @monotonic_i8_shared_cta(i8 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i8_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [monotonic_i8_shared_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_shared_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.shared.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr addrspace(3) %addr syncscope("block") monotonic, align 1
+ ret void
+}
+
+define void @monotonic_i8_shared_cluster(i8 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i8_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [monotonic_i8_shared_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_shared_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cta.shared.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr addrspace(3) %addr syncscope("cluster") monotonic, align 1
+ ret void
+}
+
+define void @monotonic_i8_shared_gpu(i8 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i8_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [monotonic_i8_shared_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_shared_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.shared.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr addrspace(3) %addr syncscope("device") monotonic, align 1
+ ret void
+}
+
+define void @monotonic_i8_shared_sys_volatile(i8 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i8_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [monotonic_i8_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_shared_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr addrspace(3) %addr syncscope("") monotonic, align 1
+ ret void
+}
+
+define void @monotonic_i8_shared_cta_volatile(i8 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i8_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [monotonic_i8_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_shared_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr addrspace(3) %addr syncscope("block") monotonic, align 1
+ ret void
+}
+
+define void @monotonic_i8_shared_cluster_volatile(i8 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i8_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [monotonic_i8_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_shared_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr addrspace(3) %addr syncscope("cluster") monotonic, align 1
+ ret void
+}
+
+define void @monotonic_i8_shared_gpu_volatile(i8 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i8_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [monotonic_i8_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_shared_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr addrspace(3) %addr syncscope("device") monotonic, align 1
+ ret void
+}
+
+define void @monotonic_i16_shared_sys(i16 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i16_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_i16_shared_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_shared_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr addrspace(3) %addr syncscope("") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_i16_shared_cta(i16 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i16_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_i16_shared_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_shared_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr addrspace(3) %addr syncscope("block") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_i16_shared_cluster(i16 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i16_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_i16_shared_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_shared_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cta.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr addrspace(3) %addr syncscope("cluster") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_i16_shared_gpu(i16 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i16_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_i16_shared_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_shared_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr addrspace(3) %addr syncscope("device") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_i16_shared_sys_volatile(i16 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i16_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_i16_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_shared_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr addrspace(3) %addr syncscope("") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_i16_shared_cta_volatile(i16 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i16_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_i16_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_shared_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr addrspace(3) %addr syncscope("block") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_i16_shared_cluster_volatile(i16 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i16_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_i16_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_shared_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr addrspace(3) %addr syncscope("cluster") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_i16_shared_gpu_volatile(i16 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i16_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_i16_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_shared_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr addrspace(3) %addr syncscope("device") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_i32_shared_sys(i32 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i32_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_i32_shared_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_shared_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr addrspace(3) %addr syncscope("") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_i32_shared_cta(i32 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i32_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_i32_shared_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_shared_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr addrspace(3) %addr syncscope("block") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_i32_shared_cluster(i32 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i32_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_i32_shared_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_shared_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cta.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr addrspace(3) %addr syncscope("cluster") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_i32_shared_gpu(i32 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i32_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_i32_shared_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_shared_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr addrspace(3) %addr syncscope("device") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_i32_shared_sys_volatile(i32 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i32_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_i32_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_shared_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr addrspace(3) %addr syncscope("") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_i32_shared_cta_volatile(i32 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i32_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_i32_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_shared_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr addrspace(3) %addr syncscope("block") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_i32_shared_cluster_volatile(i32 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i32_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_i32_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_shared_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr addrspace(3) %addr syncscope("cluster") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_i32_shared_gpu_volatile(i32 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i32_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_i32_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_shared_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr addrspace(3) %addr syncscope("device") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_i64_shared_sys(i64 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i64_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_shared_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_i64_shared_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr addrspace(3) %addr syncscope("") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_i64_shared_cta(i64 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i64_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_shared_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_i64_shared_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr addrspace(3) %addr syncscope("block") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_i64_shared_cluster(i64 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i64_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_shared_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_i64_shared_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cta.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr addrspace(3) %addr syncscope("cluster") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_i64_shared_gpu(i64 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i64_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_shared_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_i64_shared_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr addrspace(3) %addr syncscope("device") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_i64_shared_sys_volatile(i64 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i64_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_i64_shared_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr addrspace(3) %addr syncscope("") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_i64_shared_cta_volatile(i64 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i64_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_i64_shared_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr addrspace(3) %addr syncscope("block") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_i64_shared_cluster_volatile(i64 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i64_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_i64_shared_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr addrspace(3) %addr syncscope("cluster") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_i64_shared_gpu_volatile(i64 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i64_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_i64_shared_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr addrspace(3) %addr syncscope("device") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_half_shared_sys(half %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_half_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_half_shared_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_shared_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr addrspace(3) %addr syncscope("") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_half_shared_cta(half %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_half_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_half_shared_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_shared_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr addrspace(3) %addr syncscope("block") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_half_shared_cluster(half %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_half_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_half_shared_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_shared_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cta.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr addrspace(3) %addr syncscope("cluster") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_half_shared_gpu(half %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_half_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_half_shared_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_shared_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr addrspace(3) %addr syncscope("device") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_half_shared_sys_volatile(half %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_half_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_half_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_shared_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr addrspace(3) %addr syncscope("") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_half_shared_cta_volatile(half %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_half_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_half_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_shared_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr addrspace(3) %addr syncscope("block") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_half_shared_cluster_volatile(half %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_half_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_half_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_shared_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr addrspace(3) %addr syncscope("cluster") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_half_shared_gpu_volatile(half %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_half_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_half_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_shared_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr addrspace(3) %addr syncscope("device") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_bfloat_shared_sys(bfloat %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_bfloat_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_bfloat_shared_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_shared_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr addrspace(3) %addr syncscope("") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_bfloat_shared_cta(bfloat %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_bfloat_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_bfloat_shared_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_shared_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr addrspace(3) %addr syncscope("block") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_bfloat_shared_cluster(bfloat %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_bfloat_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_bfloat_shared_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_shared_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cta.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr addrspace(3) %addr syncscope("cluster") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_bfloat_shared_gpu(bfloat %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_bfloat_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_bfloat_shared_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_shared_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr addrspace(3) %addr syncscope("device") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_bfloat_shared_sys_volatile(bfloat %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_bfloat_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_bfloat_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_shared_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr addrspace(3) %addr syncscope("") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_bfloat_shared_cta_volatile(bfloat %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_bfloat_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_bfloat_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_shared_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr addrspace(3) %addr syncscope("block") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_bfloat_shared_cluster_volatile(bfloat %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_bfloat_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_bfloat_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_shared_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr addrspace(3) %addr syncscope("cluster") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_bfloat_shared_gpu_volatile(bfloat %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_bfloat_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_bfloat_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_shared_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr addrspace(3) %addr syncscope("device") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_float_shared_sys(float %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_float_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_float_shared_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_shared_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr addrspace(3) %addr syncscope("") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_float_shared_cta(float %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_float_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_float_shared_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_shared_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr addrspace(3) %addr syncscope("block") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_float_shared_cluster(float %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_float_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_float_shared_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_shared_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cta.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr addrspace(3) %addr syncscope("cluster") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_float_shared_gpu(float %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_float_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_float_shared_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_shared_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr addrspace(3) %addr syncscope("device") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_float_shared_sys_volatile(float %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_float_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_float_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_shared_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr addrspace(3) %addr syncscope("") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_float_shared_cta_volatile(float %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_float_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_float_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_shared_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr addrspace(3) %addr syncscope("block") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_float_shared_cluster_volatile(float %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_float_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_float_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_shared_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr addrspace(3) %addr syncscope("cluster") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_float_shared_gpu_volatile(float %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_float_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_float_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_shared_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr addrspace(3) %addr syncscope("device") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_double_shared_sys(double %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_double_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_shared_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_double_shared_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr addrspace(3) %addr syncscope("") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_double_shared_cta(double %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_double_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_shared_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_double_shared_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr addrspace(3) %addr syncscope("block") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_double_shared_cluster(double %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_double_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_shared_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_double_shared_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cta.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr addrspace(3) %addr syncscope("cluster") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_double_shared_gpu(double %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_double_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_shared_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_double_shared_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr addrspace(3) %addr syncscope("device") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_double_shared_sys_volatile(double %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_double_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_double_shared_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr addrspace(3) %addr syncscope("") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_double_shared_cta_volatile(double %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_double_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_double_shared_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr addrspace(3) %addr syncscope("block") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_double_shared_cluster_volatile(double %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_double_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_double_shared_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr addrspace(3) %addr syncscope("cluster") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_double_shared_gpu_volatile(double %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_double_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_double_shared_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr addrspace(3) %addr syncscope("device") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_i8_local_sys(i8 %val) {
+; CHECK-LABEL: monotonic_i8_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot512[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot512;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b8 %rs1, [monotonic_i8_local_sys_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b8 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i8 %val, ptr addrspace(5) %slot syncscope("") monotonic, align 1
+ ret void
+}
+
+define void @monotonic_i8_local_cta(i8 %val) {
+; CHECK-LABEL: monotonic_i8_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot513[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot513;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b8 %rs1, [monotonic_i8_local_cta_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b8 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i8 %val, ptr addrspace(5) %slot syncscope("block") monotonic, align 1
+ ret void
+}
+
+define void @monotonic_i8_local_cluster(i8 %val) {
+; CHECK-LABEL: monotonic_i8_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot514[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot514;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b8 %rs1, [monotonic_i8_local_cluster_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b8 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i8 %val, ptr addrspace(5) %slot syncscope("cluster") monotonic, align 1
+ ret void
+}
+
+define void @monotonic_i8_local_gpu(i8 %val) {
+; CHECK-LABEL: monotonic_i8_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot515[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot515;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b8 %rs1, [monotonic_i8_local_gpu_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b8 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i8 %val, ptr addrspace(5) %slot syncscope("device") monotonic, align 1
+ ret void
+}
+
+define void @monotonic_i8_local_sys_volatile(i8 %val) {
+; CHECK-LABEL: monotonic_i8_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot516[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot516;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b8 %rs1, [monotonic_i8_local_sys_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b8 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i8 %val, ptr addrspace(5) %slot syncscope("") monotonic, align 1
+ ret void
+}
+
+define void @monotonic_i8_local_cta_volatile(i8 %val) {
+; CHECK-LABEL: monotonic_i8_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot517[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot517;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b8 %rs1, [monotonic_i8_local_cta_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b8 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i8 %val, ptr addrspace(5) %slot syncscope("block") monotonic, align 1
+ ret void
+}
+
+define void @monotonic_i8_local_cluster_volatile(i8 %val) {
+; CHECK-LABEL: monotonic_i8_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot518[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot518;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b8 %rs1, [monotonic_i8_local_cluster_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b8 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i8 %val, ptr addrspace(5) %slot syncscope("cluster") monotonic, align 1
+ ret void
+}
+
+define void @monotonic_i8_local_gpu_volatile(i8 %val) {
+; CHECK-LABEL: monotonic_i8_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot519[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot519;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b8 %rs1, [monotonic_i8_local_gpu_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b8 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i8 %val, ptr addrspace(5) %slot syncscope("device") monotonic, align 1
+ ret void
+}
+
+define void @monotonic_i16_local_sys(i16 %val) {
+; CHECK-LABEL: monotonic_i16_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot520[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot520;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_i16_local_sys_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i16 %val, ptr addrspace(5) %slot syncscope("") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_i16_local_cta(i16 %val) {
+; CHECK-LABEL: monotonic_i16_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot521[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot521;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_i16_local_cta_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i16 %val, ptr addrspace(5) %slot syncscope("block") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_i16_local_cluster(i16 %val) {
+; CHECK-LABEL: monotonic_i16_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot522[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot522;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_i16_local_cluster_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i16 %val, ptr addrspace(5) %slot syncscope("cluster") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_i16_local_gpu(i16 %val) {
+; CHECK-LABEL: monotonic_i16_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot523[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot523;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_i16_local_gpu_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i16 %val, ptr addrspace(5) %slot syncscope("device") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_i16_local_sys_volatile(i16 %val) {
+; CHECK-LABEL: monotonic_i16_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot524[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot524;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_i16_local_sys_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i16 %val, ptr addrspace(5) %slot syncscope("") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_i16_local_cta_volatile(i16 %val) {
+; CHECK-LABEL: monotonic_i16_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot525[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot525;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_i16_local_cta_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i16 %val, ptr addrspace(5) %slot syncscope("block") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_i16_local_cluster_volatile(i16 %val) {
+; CHECK-LABEL: monotonic_i16_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot526[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot526;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_i16_local_cluster_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i16 %val, ptr addrspace(5) %slot syncscope("cluster") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_i16_local_gpu_volatile(i16 %val) {
+; CHECK-LABEL: monotonic_i16_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot527[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot527;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_i16_local_gpu_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i16 %val, ptr addrspace(5) %slot syncscope("device") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_i32_local_sys(i32 %val) {
+; CHECK-LABEL: monotonic_i32_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot528[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot528;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [monotonic_i32_local_sys_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i32 %val, ptr addrspace(5) %slot syncscope("") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_i32_local_cta(i32 %val) {
+; CHECK-LABEL: monotonic_i32_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot529[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot529;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [monotonic_i32_local_cta_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i32 %val, ptr addrspace(5) %slot syncscope("block") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_i32_local_cluster(i32 %val) {
+; CHECK-LABEL: monotonic_i32_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot530[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot530;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [monotonic_i32_local_cluster_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i32 %val, ptr addrspace(5) %slot syncscope("cluster") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_i32_local_gpu(i32 %val) {
+; CHECK-LABEL: monotonic_i32_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot531[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot531;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [monotonic_i32_local_gpu_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i32 %val, ptr addrspace(5) %slot syncscope("device") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_i32_local_sys_volatile(i32 %val) {
+; CHECK-LABEL: monotonic_i32_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot532[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot532;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [monotonic_i32_local_sys_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i32 %val, ptr addrspace(5) %slot syncscope("") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_i32_local_cta_volatile(i32 %val) {
+; CHECK-LABEL: monotonic_i32_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot533[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot533;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [monotonic_i32_local_cta_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i32 %val, ptr addrspace(5) %slot syncscope("block") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_i32_local_cluster_volatile(i32 %val) {
+; CHECK-LABEL: monotonic_i32_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot534[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot534;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [monotonic_i32_local_cluster_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i32 %val, ptr addrspace(5) %slot syncscope("cluster") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_i32_local_gpu_volatile(i32 %val) {
+; CHECK-LABEL: monotonic_i32_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot535[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot535;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [monotonic_i32_local_gpu_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i32 %val, ptr addrspace(5) %slot syncscope("device") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_i64_local_sys(i64 %val) {
+; CHECK-LABEL: monotonic_i64_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot536[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot536;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_local_sys_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i64 %val, ptr addrspace(5) %slot syncscope("") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_i64_local_cta(i64 %val) {
+; CHECK-LABEL: monotonic_i64_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot537[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot537;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_local_cta_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i64 %val, ptr addrspace(5) %slot syncscope("block") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_i64_local_cluster(i64 %val) {
+; CHECK-LABEL: monotonic_i64_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot538[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot538;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_local_cluster_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i64 %val, ptr addrspace(5) %slot syncscope("cluster") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_i64_local_gpu(i64 %val) {
+; CHECK-LABEL: monotonic_i64_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot539[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot539;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_local_gpu_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i64 %val, ptr addrspace(5) %slot syncscope("device") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_i64_local_sys_volatile(i64 %val) {
+; CHECK-LABEL: monotonic_i64_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot540[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot540;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_local_sys_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i64 %val, ptr addrspace(5) %slot syncscope("") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_i64_local_cta_volatile(i64 %val) {
+; CHECK-LABEL: monotonic_i64_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot541[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot541;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_local_cta_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i64 %val, ptr addrspace(5) %slot syncscope("block") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_i64_local_cluster_volatile(i64 %val) {
+; CHECK-LABEL: monotonic_i64_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot542[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot542;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_local_cluster_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i64 %val, ptr addrspace(5) %slot syncscope("cluster") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_i64_local_gpu_volatile(i64 %val) {
+; CHECK-LABEL: monotonic_i64_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot543[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot543;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_local_gpu_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i64 %val, ptr addrspace(5) %slot syncscope("device") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_half_local_sys(half %val) {
+; CHECK-LABEL: monotonic_half_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot544[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot544;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_half_local_sys_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic half %val, ptr addrspace(5) %slot syncscope("") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_half_local_cta(half %val) {
+; CHECK-LABEL: monotonic_half_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot545[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot545;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_half_local_cta_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic half %val, ptr addrspace(5) %slot syncscope("block") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_half_local_cluster(half %val) {
+; CHECK-LABEL: monotonic_half_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot546[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot546;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_half_local_cluster_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic half %val, ptr addrspace(5) %slot syncscope("cluster") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_half_local_gpu(half %val) {
+; CHECK-LABEL: monotonic_half_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot547[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot547;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_half_local_gpu_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic half %val, ptr addrspace(5) %slot syncscope("device") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_half_local_sys_volatile(half %val) {
+; CHECK-LABEL: monotonic_half_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot548[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot548;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_half_local_sys_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile half %val, ptr addrspace(5) %slot syncscope("") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_half_local_cta_volatile(half %val) {
+; CHECK-LABEL: monotonic_half_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot549[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot549;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_half_local_cta_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile half %val, ptr addrspace(5) %slot syncscope("block") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_half_local_cluster_volatile(half %val) {
+; CHECK-LABEL: monotonic_half_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot550[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot550;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_half_local_cluster_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile half %val, ptr addrspace(5) %slot syncscope("cluster") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_half_local_gpu_volatile(half %val) {
+; CHECK-LABEL: monotonic_half_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot551[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot551;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_half_local_gpu_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile half %val, ptr addrspace(5) %slot syncscope("device") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_bfloat_local_sys(bfloat %val) {
+; CHECK-LABEL: monotonic_bfloat_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot552[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot552;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_bfloat_local_sys_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic bfloat %val, ptr addrspace(5) %slot syncscope("") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_bfloat_local_cta(bfloat %val) {
+; CHECK-LABEL: monotonic_bfloat_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot553[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot553;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_bfloat_local_cta_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic bfloat %val, ptr addrspace(5) %slot syncscope("block") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_bfloat_local_cluster(bfloat %val) {
+; CHECK-LABEL: monotonic_bfloat_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot554[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot554;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_bfloat_local_cluster_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic bfloat %val, ptr addrspace(5) %slot syncscope("cluster") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_bfloat_local_gpu(bfloat %val) {
+; CHECK-LABEL: monotonic_bfloat_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot555[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot555;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_bfloat_local_gpu_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic bfloat %val, ptr addrspace(5) %slot syncscope("device") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_bfloat_local_sys_volatile(bfloat %val) {
+; CHECK-LABEL: monotonic_bfloat_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot556[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot556;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_bfloat_local_sys_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile bfloat %val, ptr addrspace(5) %slot syncscope("") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_bfloat_local_cta_volatile(bfloat %val) {
+; CHECK-LABEL: monotonic_bfloat_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot557[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot557;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_bfloat_local_cta_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile bfloat %val, ptr addrspace(5) %slot syncscope("block") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_bfloat_local_cluster_volatile(bfloat %val) {
+; CHECK-LABEL: monotonic_bfloat_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot558[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot558;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_bfloat_local_cluster_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile bfloat %val, ptr addrspace(5) %slot syncscope("cluster") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_bfloat_local_gpu_volatile(bfloat %val) {
+; CHECK-LABEL: monotonic_bfloat_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot559[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot559;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_bfloat_local_gpu_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile bfloat %val, ptr addrspace(5) %slot syncscope("device") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_float_local_sys(float %val) {
+; CHECK-LABEL: monotonic_float_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot560[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot560;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [monotonic_float_local_sys_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic float %val, ptr addrspace(5) %slot syncscope("") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_float_local_cta(float %val) {
+; CHECK-LABEL: monotonic_float_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot561[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot561;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [monotonic_float_local_cta_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic float %val, ptr addrspace(5) %slot syncscope("block") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_float_local_cluster(float %val) {
+; CHECK-LABEL: monotonic_float_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot562[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot562;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [monotonic_float_local_cluster_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic float %val, ptr addrspace(5) %slot syncscope("cluster") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_float_local_gpu(float %val) {
+; CHECK-LABEL: monotonic_float_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot563[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot563;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [monotonic_float_local_gpu_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic float %val, ptr addrspace(5) %slot syncscope("device") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_float_local_sys_volatile(float %val) {
+; CHECK-LABEL: monotonic_float_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot564[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot564;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [monotonic_float_local_sys_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile float %val, ptr addrspace(5) %slot syncscope("") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_float_local_cta_volatile(float %val) {
+; CHECK-LABEL: monotonic_float_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot565[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot565;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [monotonic_float_local_cta_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile float %val, ptr addrspace(5) %slot syncscope("block") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_float_local_cluster_volatile(float %val) {
+; CHECK-LABEL: monotonic_float_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot566[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot566;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [monotonic_float_local_cluster_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile float %val, ptr addrspace(5) %slot syncscope("cluster") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_float_local_gpu_volatile(float %val) {
+; CHECK-LABEL: monotonic_float_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot567[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot567;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [monotonic_float_local_gpu_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile float %val, ptr addrspace(5) %slot syncscope("device") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_double_local_sys(double %val) {
+; CHECK-LABEL: monotonic_double_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot568[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot568;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_local_sys_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic double %val, ptr addrspace(5) %slot syncscope("") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_double_local_cta(double %val) {
+; CHECK-LABEL: monotonic_double_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot569[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot569;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_local_cta_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic double %val, ptr addrspace(5) %slot syncscope("block") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_double_local_cluster(double %val) {
+; CHECK-LABEL: monotonic_double_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot570[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot570;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_local_cluster_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic double %val, ptr addrspace(5) %slot syncscope("cluster") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_double_local_gpu(double %val) {
+; CHECK-LABEL: monotonic_double_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot571[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot571;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_local_gpu_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic double %val, ptr addrspace(5) %slot syncscope("device") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_double_local_sys_volatile(double %val) {
+; CHECK-LABEL: monotonic_double_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot572[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot572;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_local_sys_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile double %val, ptr addrspace(5) %slot syncscope("") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_double_local_cta_volatile(double %val) {
+; CHECK-LABEL: monotonic_double_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot573[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot573;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_local_cta_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile double %val, ptr addrspace(5) %slot syncscope("block") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_double_local_cluster_volatile(double %val) {
+; CHECK-LABEL: monotonic_double_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot574[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot574;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_local_cluster_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile double %val, ptr addrspace(5) %slot syncscope("cluster") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_double_local_gpu_volatile(double %val) {
+; CHECK-LABEL: monotonic_double_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot575[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot575;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_local_gpu_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile double %val, ptr addrspace(5) %slot syncscope("device") monotonic, align 8
+ ret void
+}
+
+define void @release_i8_generic_sys(i8 %val, ptr %addr) {
+; CHECK-LABEL: release_i8_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [release_i8_generic_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i8_generic_sys_param_1];
+; CHECK-NEXT: st.release.sys.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr %addr syncscope("") release, align 1
+ ret void
+}
+
+define void @release_i8_generic_cta(i8 %val, ptr %addr) {
+; CHECK-LABEL: release_i8_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [release_i8_generic_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i8_generic_cta_param_1];
+; CHECK-NEXT: st.release.cta.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr %addr syncscope("block") release, align 1
+ ret void
+}
+
+define void @release_i8_generic_cluster(i8 %val, ptr %addr) {
+; CHECK-LABEL: release_i8_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [release_i8_generic_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i8_generic_cluster_param_1];
+; CHECK-NEXT: st.release.cta.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr %addr syncscope("cluster") release, align 1
+ ret void
+}
+
+define void @release_i8_generic_gpu(i8 %val, ptr %addr) {
+; CHECK-LABEL: release_i8_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [release_i8_generic_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i8_generic_gpu_param_1];
+; CHECK-NEXT: st.release.gpu.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr %addr syncscope("device") release, align 1
+ ret void
+}
+
+define void @release_i8_generic_sys_volatile(i8 %val, ptr %addr) {
+; CHECK-LABEL: release_i8_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [release_i8_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i8_generic_sys_volatile_param_1];
+; CHECK-NEXT: st.release.sys.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr %addr syncscope("") release, align 1
+ ret void
+}
+
+define void @release_i8_generic_cta_volatile(i8 %val, ptr %addr) {
+; CHECK-LABEL: release_i8_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [release_i8_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i8_generic_cta_volatile_param_1];
+; CHECK-NEXT: st.release.sys.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr %addr syncscope("block") release, align 1
+ ret void
+}
+
+define void @release_i8_generic_cluster_volatile(i8 %val, ptr %addr) {
+; CHECK-LABEL: release_i8_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [release_i8_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i8_generic_cluster_volatile_param_1];
+; CHECK-NEXT: st.release.sys.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr %addr syncscope("cluster") release, align 1
+ ret void
+}
+
+define void @release_i8_generic_gpu_volatile(i8 %val, ptr %addr) {
+; CHECK-LABEL: release_i8_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [release_i8_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i8_generic_gpu_volatile_param_1];
+; CHECK-NEXT: st.release.sys.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr %addr syncscope("device") release, align 1
+ ret void
+}
+
+define void @release_i16_generic_sys(i16 %val, ptr %addr) {
+; CHECK-LABEL: release_i16_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_i16_generic_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i16_generic_sys_param_1];
+; CHECK-NEXT: st.release.sys.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr %addr syncscope("") release, align 2
+ ret void
+}
+
+define void @release_i16_generic_cta(i16 %val, ptr %addr) {
+; CHECK-LABEL: release_i16_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_i16_generic_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i16_generic_cta_param_1];
+; CHECK-NEXT: st.release.cta.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr %addr syncscope("block") release, align 2
+ ret void
+}
+
+define void @release_i16_generic_cluster(i16 %val, ptr %addr) {
+; CHECK-LABEL: release_i16_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_i16_generic_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i16_generic_cluster_param_1];
+; CHECK-NEXT: st.release.cta.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr %addr syncscope("cluster") release, align 2
+ ret void
+}
+
+define void @release_i16_generic_gpu(i16 %val, ptr %addr) {
+; CHECK-LABEL: release_i16_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_i16_generic_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i16_generic_gpu_param_1];
+; CHECK-NEXT: st.release.gpu.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr %addr syncscope("device") release, align 2
+ ret void
+}
+
+define void @release_i16_generic_sys_volatile(i16 %val, ptr %addr) {
+; CHECK-LABEL: release_i16_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_i16_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i16_generic_sys_volatile_param_1];
+; CHECK-NEXT: st.release.sys.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr %addr syncscope("") release, align 2
+ ret void
+}
+
+define void @release_i16_generic_cta_volatile(i16 %val, ptr %addr) {
+; CHECK-LABEL: release_i16_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_i16_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i16_generic_cta_volatile_param_1];
+; CHECK-NEXT: st.release.sys.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr %addr syncscope("block") release, align 2
+ ret void
+}
+
+define void @release_i16_generic_cluster_volatile(i16 %val, ptr %addr) {
+; CHECK-LABEL: release_i16_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_i16_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i16_generic_cluster_volatile_param_1];
+; CHECK-NEXT: st.release.sys.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr %addr syncscope("cluster") release, align 2
+ ret void
+}
+
+define void @release_i16_generic_gpu_volatile(i16 %val, ptr %addr) {
+; CHECK-LABEL: release_i16_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_i16_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i16_generic_gpu_volatile_param_1];
+; CHECK-NEXT: st.release.sys.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr %addr syncscope("device") release, align 2
+ ret void
+}
+
+define void @release_i32_generic_sys(i32 %val, ptr %addr) {
+; CHECK-LABEL: release_i32_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_i32_generic_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i32_generic_sys_param_1];
+; CHECK-NEXT: st.release.sys.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr %addr syncscope("") release, align 4
+ ret void
+}
+
+define void @release_i32_generic_cta(i32 %val, ptr %addr) {
+; CHECK-LABEL: release_i32_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_i32_generic_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i32_generic_cta_param_1];
+; CHECK-NEXT: st.release.cta.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr %addr syncscope("block") release, align 4
+ ret void
+}
+
+define void @release_i32_generic_cluster(i32 %val, ptr %addr) {
+; CHECK-LABEL: release_i32_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_i32_generic_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i32_generic_cluster_param_1];
+; CHECK-NEXT: st.release.cta.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr %addr syncscope("cluster") release, align 4
+ ret void
+}
+
+define void @release_i32_generic_gpu(i32 %val, ptr %addr) {
+; CHECK-LABEL: release_i32_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_i32_generic_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i32_generic_gpu_param_1];
+; CHECK-NEXT: st.release.gpu.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr %addr syncscope("device") release, align 4
+ ret void
+}
+
+define void @release_i32_generic_sys_volatile(i32 %val, ptr %addr) {
+; CHECK-LABEL: release_i32_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_i32_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i32_generic_sys_volatile_param_1];
+; CHECK-NEXT: st.release.sys.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr %addr syncscope("") release, align 4
+ ret void
+}
+
+define void @release_i32_generic_cta_volatile(i32 %val, ptr %addr) {
+; CHECK-LABEL: release_i32_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_i32_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i32_generic_cta_volatile_param_1];
+; CHECK-NEXT: st.release.sys.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr %addr syncscope("block") release, align 4
+ ret void
+}
+
+define void @release_i32_generic_cluster_volatile(i32 %val, ptr %addr) {
+; CHECK-LABEL: release_i32_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_i32_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i32_generic_cluster_volatile_param_1];
+; CHECK-NEXT: st.release.sys.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr %addr syncscope("cluster") release, align 4
+ ret void
+}
+
+define void @release_i32_generic_gpu_volatile(i32 %val, ptr %addr) {
+; CHECK-LABEL: release_i32_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_i32_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i32_generic_gpu_volatile_param_1];
+; CHECK-NEXT: st.release.sys.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr %addr syncscope("device") release, align 4
+ ret void
+}
+
+define void @release_i64_generic_sys(i64 %val, ptr %addr) {
+; CHECK-LABEL: release_i64_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i64_generic_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [release_i64_generic_sys_param_1];
+; CHECK-NEXT: st.release.sys.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr %addr syncscope("") release, align 8
+ ret void
+}
+
+define void @release_i64_generic_cta(i64 %val, ptr %addr) {
+; CHECK-LABEL: release_i64_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i64_generic_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [release_i64_generic_cta_param_1];
+; CHECK-NEXT: st.release.cta.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr %addr syncscope("block") release, align 8
+ ret void
+}
+
+define void @release_i64_generic_cluster(i64 %val, ptr %addr) {
+; CHECK-LABEL: release_i64_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i64_generic_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [release_i64_generic_cluster_param_1];
+; CHECK-NEXT: st.release.cta.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr %addr syncscope("cluster") release, align 8
+ ret void
+}
+
+define void @release_i64_generic_gpu(i64 %val, ptr %addr) {
+; CHECK-LABEL: release_i64_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i64_generic_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [release_i64_generic_gpu_param_1];
+; CHECK-NEXT: st.release.gpu.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr %addr syncscope("device") release, align 8
+ ret void
+}
+
+define void @release_i64_generic_sys_volatile(i64 %val, ptr %addr) {
+; CHECK-LABEL: release_i64_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i64_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [release_i64_generic_sys_volatile_param_1];
+; CHECK-NEXT: st.release.sys.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr %addr syncscope("") release, align 8
+ ret void
+}
+
+define void @release_i64_generic_cta_volatile(i64 %val, ptr %addr) {
+; CHECK-LABEL: release_i64_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i64_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [release_i64_generic_cta_volatile_param_1];
+; CHECK-NEXT: st.release.sys.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr %addr syncscope("block") release, align 8
+ ret void
+}
+
+define void @release_i64_generic_cluster_volatile(i64 %val, ptr %addr) {
+; CHECK-LABEL: release_i64_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i64_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [release_i64_generic_cluster_volatile_param_1];
+; CHECK-NEXT: st.release.sys.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr %addr syncscope("cluster") release, align 8
+ ret void
+}
+
+define void @release_i64_generic_gpu_volatile(i64 %val, ptr %addr) {
+; CHECK-LABEL: release_i64_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i64_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [release_i64_generic_gpu_volatile_param_1];
+; CHECK-NEXT: st.release.sys.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr %addr syncscope("device") release, align 8
+ ret void
+}
+
+define void @release_half_generic_sys(half %val, ptr %addr) {
+; CHECK-LABEL: release_half_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_half_generic_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_half_generic_sys_param_1];
+; CHECK-NEXT: st.release.sys.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr %addr syncscope("") release, align 2
+ ret void
+}
+
+define void @release_half_generic_cta(half %val, ptr %addr) {
+; CHECK-LABEL: release_half_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_half_generic_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_half_generic_cta_param_1];
+; CHECK-NEXT: st.release.cta.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr %addr syncscope("block") release, align 2
+ ret void
+}
+
+define void @release_half_generic_cluster(half %val, ptr %addr) {
+; CHECK-LABEL: release_half_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_half_generic_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_half_generic_cluster_param_1];
+; CHECK-NEXT: st.release.cta.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr %addr syncscope("cluster") release, align 2
+ ret void
+}
+
+define void @release_half_generic_gpu(half %val, ptr %addr) {
+; CHECK-LABEL: release_half_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_half_generic_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_half_generic_gpu_param_1];
+; CHECK-NEXT: st.release.gpu.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr %addr syncscope("device") release, align 2
+ ret void
+}
+
+define void @release_half_generic_sys_volatile(half %val, ptr %addr) {
+; CHECK-LABEL: release_half_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_half_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_half_generic_sys_volatile_param_1];
+; CHECK-NEXT: st.release.sys.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr %addr syncscope("") release, align 2
+ ret void
+}
+
+define void @release_half_generic_cta_volatile(half %val, ptr %addr) {
+; CHECK-LABEL: release_half_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_half_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_half_generic_cta_volatile_param_1];
+; CHECK-NEXT: st.release.sys.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr %addr syncscope("block") release, align 2
+ ret void
+}
+
+define void @release_half_generic_cluster_volatile(half %val, ptr %addr) {
+; CHECK-LABEL: release_half_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_half_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_half_generic_cluster_volatile_param_1];
+; CHECK-NEXT: st.release.sys.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr %addr syncscope("cluster") release, align 2
+ ret void
+}
+
+define void @release_half_generic_gpu_volatile(half %val, ptr %addr) {
+; CHECK-LABEL: release_half_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_half_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_half_generic_gpu_volatile_param_1];
+; CHECK-NEXT: st.release.sys.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr %addr syncscope("device") release, align 2
+ ret void
+}
+
+define void @release_bfloat_generic_sys(bfloat %val, ptr %addr) {
+; CHECK-LABEL: release_bfloat_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_bfloat_generic_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_bfloat_generic_sys_param_1];
+; CHECK-NEXT: st.release.sys.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr %addr syncscope("") release, align 2
+ ret void
+}
+
+define void @release_bfloat_generic_cta(bfloat %val, ptr %addr) {
+; CHECK-LABEL: release_bfloat_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_bfloat_generic_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_bfloat_generic_cta_param_1];
+; CHECK-NEXT: st.release.cta.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr %addr syncscope("block") release, align 2
+ ret void
+}
+
+define void @release_bfloat_generic_cluster(bfloat %val, ptr %addr) {
+; CHECK-LABEL: release_bfloat_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_bfloat_generic_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_bfloat_generic_cluster_param_1];
+; CHECK-NEXT: st.release.cta.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr %addr syncscope("cluster") release, align 2
+ ret void
+}
+
+define void @release_bfloat_generic_gpu(bfloat %val, ptr %addr) {
+; CHECK-LABEL: release_bfloat_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_bfloat_generic_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_bfloat_generic_gpu_param_1];
+; CHECK-NEXT: st.release.gpu.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr %addr syncscope("device") release, align 2
+ ret void
+}
+
+define void @release_bfloat_generic_sys_volatile(bfloat %val, ptr %addr) {
+; CHECK-LABEL: release_bfloat_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_bfloat_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_bfloat_generic_sys_volatile_param_1];
+; CHECK-NEXT: st.release.sys.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr %addr syncscope("") release, align 2
+ ret void
+}
+
+define void @release_bfloat_generic_cta_volatile(bfloat %val, ptr %addr) {
+; CHECK-LABEL: release_bfloat_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_bfloat_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_bfloat_generic_cta_volatile_param_1];
+; CHECK-NEXT: st.release.sys.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr %addr syncscope("block") release, align 2
+ ret void
+}
+
+define void @release_bfloat_generic_cluster_volatile(bfloat %val, ptr %addr) {
+; CHECK-LABEL: release_bfloat_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_bfloat_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_bfloat_generic_cluster_volatile_param_1];
+; CHECK-NEXT: st.release.sys.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr %addr syncscope("cluster") release, align 2
+ ret void
+}
+
+define void @release_bfloat_generic_gpu_volatile(bfloat %val, ptr %addr) {
+; CHECK-LABEL: release_bfloat_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_bfloat_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_bfloat_generic_gpu_volatile_param_1];
+; CHECK-NEXT: st.release.sys.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr %addr syncscope("device") release, align 2
+ ret void
+}
+
+define void @release_float_generic_sys(float %val, ptr %addr) {
+; CHECK-LABEL: release_float_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_float_generic_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_float_generic_sys_param_1];
+; CHECK-NEXT: st.release.sys.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr %addr syncscope("") release, align 4
+ ret void
+}
+
+define void @release_float_generic_cta(float %val, ptr %addr) {
+; CHECK-LABEL: release_float_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_float_generic_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_float_generic_cta_param_1];
+; CHECK-NEXT: st.release.cta.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr %addr syncscope("block") release, align 4
+ ret void
+}
+
+define void @release_float_generic_cluster(float %val, ptr %addr) {
+; CHECK-LABEL: release_float_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_float_generic_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_float_generic_cluster_param_1];
+; CHECK-NEXT: st.release.cta.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr %addr syncscope("cluster") release, align 4
+ ret void
+}
+
+define void @release_float_generic_gpu(float %val, ptr %addr) {
+; CHECK-LABEL: release_float_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_float_generic_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_float_generic_gpu_param_1];
+; CHECK-NEXT: st.release.gpu.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr %addr syncscope("device") release, align 4
+ ret void
+}
+
+define void @release_float_generic_sys_volatile(float %val, ptr %addr) {
+; CHECK-LABEL: release_float_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_float_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_float_generic_sys_volatile_param_1];
+; CHECK-NEXT: st.release.sys.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr %addr syncscope("") release, align 4
+ ret void
+}
+
+define void @release_float_generic_cta_volatile(float %val, ptr %addr) {
+; CHECK-LABEL: release_float_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_float_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_float_generic_cta_volatile_param_1];
+; CHECK-NEXT: st.release.sys.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr %addr syncscope("block") release, align 4
+ ret void
+}
+
+define void @release_float_generic_cluster_volatile(float %val, ptr %addr) {
+; CHECK-LABEL: release_float_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_float_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_float_generic_cluster_volatile_param_1];
+; CHECK-NEXT: st.release.sys.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr %addr syncscope("cluster") release, align 4
+ ret void
+}
+
+define void @release_float_generic_gpu_volatile(float %val, ptr %addr) {
+; CHECK-LABEL: release_float_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_float_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_float_generic_gpu_volatile_param_1];
+; CHECK-NEXT: st.release.sys.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr %addr syncscope("device") release, align 4
+ ret void
+}
+
+define void @release_double_generic_sys(double %val, ptr %addr) {
+; CHECK-LABEL: release_double_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_double_generic_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [release_double_generic_sys_param_1];
+; CHECK-NEXT: st.release.sys.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr %addr syncscope("") release, align 8
+ ret void
+}
+
+define void @release_double_generic_cta(double %val, ptr %addr) {
+; CHECK-LABEL: release_double_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_double_generic_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [release_double_generic_cta_param_1];
+; CHECK-NEXT: st.release.cta.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr %addr syncscope("block") release, align 8
+ ret void
+}
+
+define void @release_double_generic_cluster(double %val, ptr %addr) {
+; CHECK-LABEL: release_double_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_double_generic_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [release_double_generic_cluster_param_1];
+; CHECK-NEXT: st.release.cta.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr %addr syncscope("cluster") release, align 8
+ ret void
+}
+
+define void @release_double_generic_gpu(double %val, ptr %addr) {
+; CHECK-LABEL: release_double_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_double_generic_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [release_double_generic_gpu_param_1];
+; CHECK-NEXT: st.release.gpu.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr %addr syncscope("device") release, align 8
+ ret void
+}
+
+define void @release_double_generic_sys_volatile(double %val, ptr %addr) {
+; CHECK-LABEL: release_double_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_double_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [release_double_generic_sys_volatile_param_1];
+; CHECK-NEXT: st.release.sys.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr %addr syncscope("") release, align 8
+ ret void
+}
+
+define void @release_double_generic_cta_volatile(double %val, ptr %addr) {
+; CHECK-LABEL: release_double_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_double_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [release_double_generic_cta_volatile_param_1];
+; CHECK-NEXT: st.release.sys.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr %addr syncscope("block") release, align 8
+ ret void
+}
+
+define void @release_double_generic_cluster_volatile(double %val, ptr %addr) {
+; CHECK-LABEL: release_double_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_double_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [release_double_generic_cluster_volatile_param_1];
+; CHECK-NEXT: st.release.sys.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr %addr syncscope("cluster") release, align 8
+ ret void
+}
+
+define void @release_double_generic_gpu_volatile(double %val, ptr %addr) {
+; CHECK-LABEL: release_double_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_double_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [release_double_generic_gpu_volatile_param_1];
+; CHECK-NEXT: st.release.sys.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr %addr syncscope("device") release, align 8
+ ret void
+}
+
+define void @release_i8_global_sys(i8 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_i8_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [release_i8_global_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i8_global_sys_param_1];
+; CHECK-NEXT: st.release.sys.global.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr addrspace(1) %addr syncscope("") release, align 1
+ ret void
+}
+
+define void @release_i8_global_cta(i8 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_i8_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [release_i8_global_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i8_global_cta_param_1];
+; CHECK-NEXT: st.release.cta.global.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr addrspace(1) %addr syncscope("block") release, align 1
+ ret void
+}
+
+define void @release_i8_global_cluster(i8 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_i8_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [release_i8_global_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i8_global_cluster_param_1];
+; CHECK-NEXT: st.release.cta.global.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr addrspace(1) %addr syncscope("cluster") release, align 1
+ ret void
+}
+
+define void @release_i8_global_gpu(i8 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_i8_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [release_i8_global_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i8_global_gpu_param_1];
+; CHECK-NEXT: st.release.gpu.global.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr addrspace(1) %addr syncscope("device") release, align 1
+ ret void
+}
+
+define void @release_i8_global_sys_volatile(i8 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_i8_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [release_i8_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i8_global_sys_volatile_param_1];
+; CHECK-NEXT: st.release.sys.global.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr addrspace(1) %addr syncscope("") release, align 1
+ ret void
+}
+
+define void @release_i8_global_cta_volatile(i8 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_i8_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [release_i8_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i8_global_cta_volatile_param_1];
+; CHECK-NEXT: st.release.sys.global.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr addrspace(1) %addr syncscope("block") release, align 1
+ ret void
+}
+
+define void @release_i8_global_cluster_volatile(i8 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_i8_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [release_i8_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i8_global_cluster_volatile_param_1];
+; CHECK-NEXT: st.release.sys.global.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr addrspace(1) %addr syncscope("cluster") release, align 1
+ ret void
+}
+
+define void @release_i8_global_gpu_volatile(i8 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_i8_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [release_i8_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i8_global_gpu_volatile_param_1];
+; CHECK-NEXT: st.release.sys.global.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr addrspace(1) %addr syncscope("device") release, align 1
+ ret void
+}
+
+define void @release_i16_global_sys(i16 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_i16_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_i16_global_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i16_global_sys_param_1];
+; CHECK-NEXT: st.release.sys.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr addrspace(1) %addr syncscope("") release, align 2
+ ret void
+}
+
+define void @release_i16_global_cta(i16 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_i16_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_i16_global_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i16_global_cta_param_1];
+; CHECK-NEXT: st.release.cta.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr addrspace(1) %addr syncscope("block") release, align 2
+ ret void
+}
+
+define void @release_i16_global_cluster(i16 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_i16_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_i16_global_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i16_global_cluster_param_1];
+; CHECK-NEXT: st.release.cta.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr addrspace(1) %addr syncscope("cluster") release, align 2
+ ret void
+}
+
+define void @release_i16_global_gpu(i16 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_i16_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_i16_global_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i16_global_gpu_param_1];
+; CHECK-NEXT: st.release.gpu.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr addrspace(1) %addr syncscope("device") release, align 2
+ ret void
+}
+
+define void @release_i16_global_sys_volatile(i16 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_i16_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_i16_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i16_global_sys_volatile_param_1];
+; CHECK-NEXT: st.release.sys.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr addrspace(1) %addr syncscope("") release, align 2
+ ret void
+}
+
+define void @release_i16_global_cta_volatile(i16 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_i16_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_i16_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i16_global_cta_volatile_param_1];
+; CHECK-NEXT: st.release.sys.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr addrspace(1) %addr syncscope("block") release, align 2
+ ret void
+}
+
+define void @release_i16_global_cluster_volatile(i16 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_i16_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_i16_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i16_global_cluster_volatile_param_1];
+; CHECK-NEXT: st.release.sys.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr addrspace(1) %addr syncscope("cluster") release, align 2
+ ret void
+}
+
+define void @release_i16_global_gpu_volatile(i16 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_i16_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_i16_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i16_global_gpu_volatile_param_1];
+; CHECK-NEXT: st.release.sys.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr addrspace(1) %addr syncscope("device") release, align 2
+ ret void
+}
+
+define void @release_i32_global_sys(i32 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_i32_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_i32_global_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i32_global_sys_param_1];
+; CHECK-NEXT: st.release.sys.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr addrspace(1) %addr syncscope("") release, align 4
+ ret void
+}
+
+define void @release_i32_global_cta(i32 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_i32_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_i32_global_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i32_global_cta_param_1];
+; CHECK-NEXT: st.release.cta.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr addrspace(1) %addr syncscope("block") release, align 4
+ ret void
+}
+
+define void @release_i32_global_cluster(i32 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_i32_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_i32_global_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i32_global_cluster_param_1];
+; CHECK-NEXT: st.release.cta.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr addrspace(1) %addr syncscope("cluster") release, align 4
+ ret void
+}
+
+define void @release_i32_global_gpu(i32 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_i32_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_i32_global_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i32_global_gpu_param_1];
+; CHECK-NEXT: st.release.gpu.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr addrspace(1) %addr syncscope("device") release, align 4
+ ret void
+}
+
+define void @release_i32_global_sys_volatile(i32 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_i32_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_i32_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i32_global_sys_volatile_param_1];
+; CHECK-NEXT: st.release.sys.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr addrspace(1) %addr syncscope("") release, align 4
+ ret void
+}
+
+define void @release_i32_global_cta_volatile(i32 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_i32_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_i32_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i32_global_cta_volatile_param_1];
+; CHECK-NEXT: st.release.sys.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr addrspace(1) %addr syncscope("block") release, align 4
+ ret void
+}
+
+define void @release_i32_global_cluster_volatile(i32 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_i32_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_i32_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i32_global_cluster_volatile_param_1];
+; CHECK-NEXT: st.release.sys.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr addrspace(1) %addr syncscope("cluster") release, align 4
+ ret void
+}
+
+define void @release_i32_global_gpu_volatile(i32 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_i32_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_i32_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i32_global_gpu_volatile_param_1];
+; CHECK-NEXT: st.release.sys.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr addrspace(1) %addr syncscope("device") release, align 4
+ ret void
+}
+
+define void @release_i64_global_sys(i64 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_i64_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i64_global_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [release_i64_global_sys_param_1];
+; CHECK-NEXT: st.release.sys.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr addrspace(1) %addr syncscope("") release, align 8
+ ret void
+}
+
+define void @release_i64_global_cta(i64 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_i64_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i64_global_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [release_i64_global_cta_param_1];
+; CHECK-NEXT: st.release.cta.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr addrspace(1) %addr syncscope("block") release, align 8
+ ret void
+}
+
+define void @release_i64_global_cluster(i64 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_i64_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i64_global_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [release_i64_global_cluster_param_1];
+; CHECK-NEXT: st.release.cta.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr addrspace(1) %addr syncscope("cluster") release, align 8
+ ret void
+}
+
+define void @release_i64_global_gpu(i64 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_i64_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i64_global_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [release_i64_global_gpu_param_1];
+; CHECK-NEXT: st.release.gpu.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr addrspace(1) %addr syncscope("device") release, align 8
+ ret void
+}
+
+define void @release_i64_global_sys_volatile(i64 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_i64_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i64_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [release_i64_global_sys_volatile_param_1];
+; CHECK-NEXT: st.release.sys.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr addrspace(1) %addr syncscope("") release, align 8
+ ret void
+}
+
+define void @release_i64_global_cta_volatile(i64 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_i64_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i64_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [release_i64_global_cta_volatile_param_1];
+; CHECK-NEXT: st.release.sys.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr addrspace(1) %addr syncscope("block") release, align 8
+ ret void
+}
+
+define void @release_i64_global_cluster_volatile(i64 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_i64_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i64_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [release_i64_global_cluster_volatile_param_1];
+; CHECK-NEXT: st.release.sys.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr addrspace(1) %addr syncscope("cluster") release, align 8
+ ret void
+}
+
+define void @release_i64_global_gpu_volatile(i64 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_i64_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i64_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [release_i64_global_gpu_volatile_param_1];
+; CHECK-NEXT: st.release.sys.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr addrspace(1) %addr syncscope("device") release, align 8
+ ret void
+}
+
+define void @release_half_global_sys(half %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_half_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_half_global_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_half_global_sys_param_1];
+; CHECK-NEXT: st.release.sys.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr addrspace(1) %addr syncscope("") release, align 2
+ ret void
+}
+
+define void @release_half_global_cta(half %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_half_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_half_global_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_half_global_cta_param_1];
+; CHECK-NEXT: st.release.cta.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr addrspace(1) %addr syncscope("block") release, align 2
+ ret void
+}
+
+define void @release_half_global_cluster(half %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_half_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_half_global_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_half_global_cluster_param_1];
+; CHECK-NEXT: st.release.cta.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr addrspace(1) %addr syncscope("cluster") release, align 2
+ ret void
+}
+
+define void @release_half_global_gpu(half %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_half_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_half_global_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_half_global_gpu_param_1];
+; CHECK-NEXT: st.release.gpu.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr addrspace(1) %addr syncscope("device") release, align 2
+ ret void
+}
+
+define void @release_half_global_sys_volatile(half %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_half_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_half_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_half_global_sys_volatile_param_1];
+; CHECK-NEXT: st.release.sys.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr addrspace(1) %addr syncscope("") release, align 2
+ ret void
+}
+
+define void @release_half_global_cta_volatile(half %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_half_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_half_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_half_global_cta_volatile_param_1];
+; CHECK-NEXT: st.release.sys.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr addrspace(1) %addr syncscope("block") release, align 2
+ ret void
+}
+
+define void @release_half_global_cluster_volatile(half %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_half_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_half_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_half_global_cluster_volatile_param_1];
+; CHECK-NEXT: st.release.sys.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr addrspace(1) %addr syncscope("cluster") release, align 2
+ ret void
+}
+
+define void @release_half_global_gpu_volatile(half %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_half_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_half_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_half_global_gpu_volatile_param_1];
+; CHECK-NEXT: st.release.sys.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr addrspace(1) %addr syncscope("device") release, align 2
+ ret void
+}
+
+define void @release_bfloat_global_sys(bfloat %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_bfloat_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_bfloat_global_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_bfloat_global_sys_param_1];
+; CHECK-NEXT: st.release.sys.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr addrspace(1) %addr syncscope("") release, align 2
+ ret void
+}
+
+define void @release_bfloat_global_cta(bfloat %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_bfloat_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_bfloat_global_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_bfloat_global_cta_param_1];
+; CHECK-NEXT: st.release.cta.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr addrspace(1) %addr syncscope("block") release, align 2
+ ret void
+}
+
+define void @release_bfloat_global_cluster(bfloat %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_bfloat_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_bfloat_global_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_bfloat_global_cluster_param_1];
+; CHECK-NEXT: st.release.cta.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr addrspace(1) %addr syncscope("cluster") release, align 2
+ ret void
+}
+
+define void @release_bfloat_global_gpu(bfloat %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_bfloat_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_bfloat_global_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_bfloat_global_gpu_param_1];
+; CHECK-NEXT: st.release.gpu.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr addrspace(1) %addr syncscope("device") release, align 2
+ ret void
+}
+
+define void @release_bfloat_global_sys_volatile(bfloat %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_bfloat_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_bfloat_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_bfloat_global_sys_volatile_param_1];
+; CHECK-NEXT: st.release.sys.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr addrspace(1) %addr syncscope("") release, align 2
+ ret void
+}
+
+define void @release_bfloat_global_cta_volatile(bfloat %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_bfloat_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_bfloat_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_bfloat_global_cta_volatile_param_1];
+; CHECK-NEXT: st.release.sys.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr addrspace(1) %addr syncscope("block") release, align 2
+ ret void
+}
+
+define void @release_bfloat_global_cluster_volatile(bfloat %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_bfloat_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_bfloat_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_bfloat_global_cluster_volatile_param_1];
+; CHECK-NEXT: st.release.sys.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr addrspace(1) %addr syncscope("cluster") release, align 2
+ ret void
+}
+
+define void @release_bfloat_global_gpu_volatile(bfloat %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_bfloat_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_bfloat_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_bfloat_global_gpu_volatile_param_1];
+; CHECK-NEXT: st.release.sys.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr addrspace(1) %addr syncscope("device") release, align 2
+ ret void
+}
+
+define void @release_float_global_sys(float %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_float_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_float_global_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_float_global_sys_param_1];
+; CHECK-NEXT: st.release.sys.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr addrspace(1) %addr syncscope("") release, align 4
+ ret void
+}
+
+define void @release_float_global_cta(float %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_float_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_float_global_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_float_global_cta_param_1];
+; CHECK-NEXT: st.release.cta.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr addrspace(1) %addr syncscope("block") release, align 4
+ ret void
+}
+
+define void @release_float_global_cluster(float %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_float_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_float_global_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_float_global_cluster_param_1];
+; CHECK-NEXT: st.release.cta.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr addrspace(1) %addr syncscope("cluster") release, align 4
+ ret void
+}
+
+define void @release_float_global_gpu(float %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_float_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_float_global_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_float_global_gpu_param_1];
+; CHECK-NEXT: st.release.gpu.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr addrspace(1) %addr syncscope("device") release, align 4
+ ret void
+}
+
+define void @release_float_global_sys_volatile(float %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_float_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_float_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_float_global_sys_volatile_param_1];
+; CHECK-NEXT: st.release.sys.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr addrspace(1) %addr syncscope("") release, align 4
+ ret void
+}
+
+define void @release_float_global_cta_volatile(float %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_float_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_float_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_float_global_cta_volatile_param_1];
+; CHECK-NEXT: st.release.sys.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr addrspace(1) %addr syncscope("block") release, align 4
+ ret void
+}
+
+define void @release_float_global_cluster_volatile(float %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_float_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_float_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_float_global_cluster_volatile_param_1];
+; CHECK-NEXT: st.release.sys.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr addrspace(1) %addr syncscope("cluster") release, align 4
+ ret void
+}
+
+define void @release_float_global_gpu_volatile(float %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_float_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_float_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_float_global_gpu_volatile_param_1];
+; CHECK-NEXT: st.release.sys.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr addrspace(1) %addr syncscope("device") release, align 4
+ ret void
+}
+
+define void @release_double_global_sys(double %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_double_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_double_global_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [release_double_global_sys_param_1];
+; CHECK-NEXT: st.release.sys.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr addrspace(1) %addr syncscope("") release, align 8
+ ret void
+}
+
+define void @release_double_global_cta(double %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_double_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_double_global_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [release_double_global_cta_param_1];
+; CHECK-NEXT: st.release.cta.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr addrspace(1) %addr syncscope("block") release, align 8
+ ret void
+}
+
+define void @release_double_global_cluster(double %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_double_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_double_global_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [release_double_global_cluster_param_1];
+; CHECK-NEXT: st.release.cta.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr addrspace(1) %addr syncscope("cluster") release, align 8
+ ret void
+}
+
+define void @release_double_global_gpu(double %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_double_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_double_global_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [release_double_global_gpu_param_1];
+; CHECK-NEXT: st.release.gpu.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr addrspace(1) %addr syncscope("device") release, align 8
+ ret void
+}
+
+define void @release_double_global_sys_volatile(double %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_double_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_double_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [release_double_global_sys_volatile_param_1];
+; CHECK-NEXT: st.release.sys.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr addrspace(1) %addr syncscope("") release, align 8
+ ret void
+}
+
+define void @release_double_global_cta_volatile(double %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_double_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_double_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [release_double_global_cta_volatile_param_1];
+; CHECK-NEXT: st.release.sys.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr addrspace(1) %addr syncscope("block") release, align 8
+ ret void
+}
+
+define void @release_double_global_cluster_volatile(double %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_double_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_double_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [release_double_global_cluster_volatile_param_1];
+; CHECK-NEXT: st.release.sys.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr addrspace(1) %addr syncscope("cluster") release, align 8
+ ret void
+}
+
+define void @release_double_global_gpu_volatile(double %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_double_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_double_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [release_double_global_gpu_volatile_param_1];
+; CHECK-NEXT: st.release.sys.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr addrspace(1) %addr syncscope("device") release, align 8
+ ret void
+}
+
+define void @release_i8_shared_sys(i8 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_i8_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [release_i8_shared_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i8_shared_sys_param_1];
+; CHECK-NEXT: st.release.sys.shared.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr addrspace(3) %addr syncscope("") release, align 1
+ ret void
+}
+
+define void @release_i8_shared_cta(i8 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_i8_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [release_i8_shared_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i8_shared_cta_param_1];
+; CHECK-NEXT: st.release.cta.shared.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr addrspace(3) %addr syncscope("block") release, align 1
+ ret void
+}
+
+define void @release_i8_shared_cluster(i8 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_i8_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [release_i8_shared_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i8_shared_cluster_param_1];
+; CHECK-NEXT: st.release.cta.shared.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr addrspace(3) %addr syncscope("cluster") release, align 1
+ ret void
+}
+
+define void @release_i8_shared_gpu(i8 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_i8_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [release_i8_shared_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i8_shared_gpu_param_1];
+; CHECK-NEXT: st.release.gpu.shared.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr addrspace(3) %addr syncscope("device") release, align 1
+ ret void
+}
+
+define void @release_i8_shared_sys_volatile(i8 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_i8_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [release_i8_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i8_shared_sys_volatile_param_1];
+; CHECK-NEXT: st.release.sys.shared.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr addrspace(3) %addr syncscope("") release, align 1
+ ret void
+}
+
+define void @release_i8_shared_cta_volatile(i8 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_i8_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [release_i8_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i8_shared_cta_volatile_param_1];
+; CHECK-NEXT: st.release.sys.shared.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr addrspace(3) %addr syncscope("block") release, align 1
+ ret void
+}
+
+define void @release_i8_shared_cluster_volatile(i8 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_i8_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [release_i8_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i8_shared_cluster_volatile_param_1];
+; CHECK-NEXT: st.release.sys.shared.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr addrspace(3) %addr syncscope("cluster") release, align 1
+ ret void
+}
+
+define void @release_i8_shared_gpu_volatile(i8 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_i8_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [release_i8_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i8_shared_gpu_volatile_param_1];
+; CHECK-NEXT: st.release.sys.shared.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr addrspace(3) %addr syncscope("device") release, align 1
+ ret void
+}
+
+define void @release_i16_shared_sys(i16 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_i16_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_i16_shared_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i16_shared_sys_param_1];
+; CHECK-NEXT: st.release.sys.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr addrspace(3) %addr syncscope("") release, align 2
+ ret void
+}
+
+define void @release_i16_shared_cta(i16 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_i16_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_i16_shared_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i16_shared_cta_param_1];
+; CHECK-NEXT: st.release.cta.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr addrspace(3) %addr syncscope("block") release, align 2
+ ret void
+}
+
+define void @release_i16_shared_cluster(i16 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_i16_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_i16_shared_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i16_shared_cluster_param_1];
+; CHECK-NEXT: st.release.cta.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr addrspace(3) %addr syncscope("cluster") release, align 2
+ ret void
+}
+
+define void @release_i16_shared_gpu(i16 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_i16_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_i16_shared_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i16_shared_gpu_param_1];
+; CHECK-NEXT: st.release.gpu.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr addrspace(3) %addr syncscope("device") release, align 2
+ ret void
+}
+
+define void @release_i16_shared_sys_volatile(i16 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_i16_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_i16_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i16_shared_sys_volatile_param_1];
+; CHECK-NEXT: st.release.sys.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr addrspace(3) %addr syncscope("") release, align 2
+ ret void
+}
+
+define void @release_i16_shared_cta_volatile(i16 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_i16_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_i16_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i16_shared_cta_volatile_param_1];
+; CHECK-NEXT: st.release.sys.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr addrspace(3) %addr syncscope("block") release, align 2
+ ret void
+}
+
+define void @release_i16_shared_cluster_volatile(i16 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_i16_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_i16_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i16_shared_cluster_volatile_param_1];
+; CHECK-NEXT: st.release.sys.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr addrspace(3) %addr syncscope("cluster") release, align 2
+ ret void
+}
+
+define void @release_i16_shared_gpu_volatile(i16 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_i16_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_i16_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i16_shared_gpu_volatile_param_1];
+; CHECK-NEXT: st.release.sys.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr addrspace(3) %addr syncscope("device") release, align 2
+ ret void
+}
+
+define void @release_i32_shared_sys(i32 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_i32_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_i32_shared_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i32_shared_sys_param_1];
+; CHECK-NEXT: st.release.sys.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr addrspace(3) %addr syncscope("") release, align 4
+ ret void
+}
+
+define void @release_i32_shared_cta(i32 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_i32_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_i32_shared_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i32_shared_cta_param_1];
+; CHECK-NEXT: st.release.cta.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr addrspace(3) %addr syncscope("block") release, align 4
+ ret void
+}
+
+define void @release_i32_shared_cluster(i32 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_i32_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_i32_shared_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i32_shared_cluster_param_1];
+; CHECK-NEXT: st.release.cta.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr addrspace(3) %addr syncscope("cluster") release, align 4
+ ret void
+}
+
+define void @release_i32_shared_gpu(i32 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_i32_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_i32_shared_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i32_shared_gpu_param_1];
+; CHECK-NEXT: st.release.gpu.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr addrspace(3) %addr syncscope("device") release, align 4
+ ret void
+}
+
+define void @release_i32_shared_sys_volatile(i32 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_i32_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_i32_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i32_shared_sys_volatile_param_1];
+; CHECK-NEXT: st.release.sys.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr addrspace(3) %addr syncscope("") release, align 4
+ ret void
+}
+
+define void @release_i32_shared_cta_volatile(i32 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_i32_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_i32_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i32_shared_cta_volatile_param_1];
+; CHECK-NEXT: st.release.sys.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr addrspace(3) %addr syncscope("block") release, align 4
+ ret void
+}
+
+define void @release_i32_shared_cluster_volatile(i32 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_i32_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_i32_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i32_shared_cluster_volatile_param_1];
+; CHECK-NEXT: st.release.sys.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr addrspace(3) %addr syncscope("cluster") release, align 4
+ ret void
+}
+
+define void @release_i32_shared_gpu_volatile(i32 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_i32_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_i32_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i32_shared_gpu_volatile_param_1];
+; CHECK-NEXT: st.release.sys.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr addrspace(3) %addr syncscope("device") release, align 4
+ ret void
+}
+
+define void @release_i64_shared_sys(i64 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_i64_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i64_shared_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [release_i64_shared_sys_param_1];
+; CHECK-NEXT: st.release.sys.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr addrspace(3) %addr syncscope("") release, align 8
+ ret void
+}
+
+define void @release_i64_shared_cta(i64 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_i64_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i64_shared_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [release_i64_shared_cta_param_1];
+; CHECK-NEXT: st.release.cta.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr addrspace(3) %addr syncscope("block") release, align 8
+ ret void
+}
+
+define void @release_i64_shared_cluster(i64 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_i64_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i64_shared_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [release_i64_shared_cluster_param_1];
+; CHECK-NEXT: st.release.cta.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr addrspace(3) %addr syncscope("cluster") release, align 8
+ ret void
+}
+
+define void @release_i64_shared_gpu(i64 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_i64_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i64_shared_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [release_i64_shared_gpu_param_1];
+; CHECK-NEXT: st.release.gpu.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr addrspace(3) %addr syncscope("device") release, align 8
+ ret void
+}
+
+define void @release_i64_shared_sys_volatile(i64 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_i64_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i64_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [release_i64_shared_sys_volatile_param_1];
+; CHECK-NEXT: st.release.sys.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr addrspace(3) %addr syncscope("") release, align 8
+ ret void
+}
+
+define void @release_i64_shared_cta_volatile(i64 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_i64_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i64_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [release_i64_shared_cta_volatile_param_1];
+; CHECK-NEXT: st.release.sys.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr addrspace(3) %addr syncscope("block") release, align 8
+ ret void
+}
+
+define void @release_i64_shared_cluster_volatile(i64 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_i64_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i64_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [release_i64_shared_cluster_volatile_param_1];
+; CHECK-NEXT: st.release.sys.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr addrspace(3) %addr syncscope("cluster") release, align 8
+ ret void
+}
+
+define void @release_i64_shared_gpu_volatile(i64 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_i64_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i64_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [release_i64_shared_gpu_volatile_param_1];
+; CHECK-NEXT: st.release.sys.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr addrspace(3) %addr syncscope("device") release, align 8
+ ret void
+}
+
+define void @release_half_shared_sys(half %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_half_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_half_shared_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_half_shared_sys_param_1];
+; CHECK-NEXT: st.release.sys.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr addrspace(3) %addr syncscope("") release, align 2
+ ret void
+}
+
+define void @release_half_shared_cta(half %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_half_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_half_shared_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_half_shared_cta_param_1];
+; CHECK-NEXT: st.release.cta.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr addrspace(3) %addr syncscope("block") release, align 2
+ ret void
+}
+
+define void @release_half_shared_cluster(half %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_half_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_half_shared_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_half_shared_cluster_param_1];
+; CHECK-NEXT: st.release.cta.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr addrspace(3) %addr syncscope("cluster") release, align 2
+ ret void
+}
+
+define void @release_half_shared_gpu(half %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_half_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_half_shared_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_half_shared_gpu_param_1];
+; CHECK-NEXT: st.release.gpu.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr addrspace(3) %addr syncscope("device") release, align 2
+ ret void
+}
+
+define void @release_half_shared_sys_volatile(half %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_half_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_half_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_half_shared_sys_volatile_param_1];
+; CHECK-NEXT: st.release.sys.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr addrspace(3) %addr syncscope("") release, align 2
+ ret void
+}
+
+define void @release_half_shared_cta_volatile(half %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_half_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_half_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_half_shared_cta_volatile_param_1];
+; CHECK-NEXT: st.release.sys.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr addrspace(3) %addr syncscope("block") release, align 2
+ ret void
+}
+
+define void @release_half_shared_cluster_volatile(half %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_half_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_half_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_half_shared_cluster_volatile_param_1];
+; CHECK-NEXT: st.release.sys.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr addrspace(3) %addr syncscope("cluster") release, align 2
+ ret void
+}
+
+define void @release_half_shared_gpu_volatile(half %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_half_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_half_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_half_shared_gpu_volatile_param_1];
+; CHECK-NEXT: st.release.sys.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr addrspace(3) %addr syncscope("device") release, align 2
+ ret void
+}
+
+define void @release_bfloat_shared_sys(bfloat %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_bfloat_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_bfloat_shared_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_bfloat_shared_sys_param_1];
+; CHECK-NEXT: st.release.sys.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr addrspace(3) %addr syncscope("") release, align 2
+ ret void
+}
+
+define void @release_bfloat_shared_cta(bfloat %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_bfloat_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_bfloat_shared_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_bfloat_shared_cta_param_1];
+; CHECK-NEXT: st.release.cta.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr addrspace(3) %addr syncscope("block") release, align 2
+ ret void
+}
+
+define void @release_bfloat_shared_cluster(bfloat %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_bfloat_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_bfloat_shared_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_bfloat_shared_cluster_param_1];
+; CHECK-NEXT: st.release.cta.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr addrspace(3) %addr syncscope("cluster") release, align 2
+ ret void
+}
+
+define void @release_bfloat_shared_gpu(bfloat %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_bfloat_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_bfloat_shared_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_bfloat_shared_gpu_param_1];
+; CHECK-NEXT: st.release.gpu.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr addrspace(3) %addr syncscope("device") release, align 2
+ ret void
+}
+
+define void @release_bfloat_shared_sys_volatile(bfloat %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_bfloat_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_bfloat_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_bfloat_shared_sys_volatile_param_1];
+; CHECK-NEXT: st.release.sys.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr addrspace(3) %addr syncscope("") release, align 2
+ ret void
+}
+
+define void @release_bfloat_shared_cta_volatile(bfloat %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_bfloat_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_bfloat_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_bfloat_shared_cta_volatile_param_1];
+; CHECK-NEXT: st.release.sys.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr addrspace(3) %addr syncscope("block") release, align 2
+ ret void
+}
+
+define void @release_bfloat_shared_cluster_volatile(bfloat %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_bfloat_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_bfloat_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_bfloat_shared_cluster_volatile_param_1];
+; CHECK-NEXT: st.release.sys.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr addrspace(3) %addr syncscope("cluster") release, align 2
+ ret void
+}
+
+define void @release_bfloat_shared_gpu_volatile(bfloat %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_bfloat_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_bfloat_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_bfloat_shared_gpu_volatile_param_1];
+; CHECK-NEXT: st.release.sys.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr addrspace(3) %addr syncscope("device") release, align 2
+ ret void
+}
+
+define void @release_float_shared_sys(float %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_float_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_float_shared_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_float_shared_sys_param_1];
+; CHECK-NEXT: st.release.sys.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr addrspace(3) %addr syncscope("") release, align 4
+ ret void
+}
+
+define void @release_float_shared_cta(float %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_float_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_float_shared_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_float_shared_cta_param_1];
+; CHECK-NEXT: st.release.cta.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr addrspace(3) %addr syncscope("block") release, align 4
+ ret void
+}
+
+define void @release_float_shared_cluster(float %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_float_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_float_shared_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_float_shared_cluster_param_1];
+; CHECK-NEXT: st.release.cta.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr addrspace(3) %addr syncscope("cluster") release, align 4
+ ret void
+}
+
+define void @release_float_shared_gpu(float %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_float_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_float_shared_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_float_shared_gpu_param_1];
+; CHECK-NEXT: st.release.gpu.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr addrspace(3) %addr syncscope("device") release, align 4
+ ret void
+}
+
+define void @release_float_shared_sys_volatile(float %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_float_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_float_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_float_shared_sys_volatile_param_1];
+; CHECK-NEXT: st.release.sys.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr addrspace(3) %addr syncscope("") release, align 4
+ ret void
+}
+
+define void @release_float_shared_cta_volatile(float %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_float_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_float_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_float_shared_cta_volatile_param_1];
+; CHECK-NEXT: st.release.sys.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr addrspace(3) %addr syncscope("block") release, align 4
+ ret void
+}
+
+define void @release_float_shared_cluster_volatile(float %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_float_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_float_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_float_shared_cluster_volatile_param_1];
+; CHECK-NEXT: st.release.sys.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr addrspace(3) %addr syncscope("cluster") release, align 4
+ ret void
+}
+
+define void @release_float_shared_gpu_volatile(float %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_float_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_float_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_float_shared_gpu_volatile_param_1];
+; CHECK-NEXT: st.release.sys.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr addrspace(3) %addr syncscope("device") release, align 4
+ ret void
+}
+
+define void @release_double_shared_sys(double %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_double_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_double_shared_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [release_double_shared_sys_param_1];
+; CHECK-NEXT: st.release.sys.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr addrspace(3) %addr syncscope("") release, align 8
+ ret void
+}
+
+define void @release_double_shared_cta(double %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_double_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_double_shared_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [release_double_shared_cta_param_1];
+; CHECK-NEXT: st.release.cta.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr addrspace(3) %addr syncscope("block") release, align 8
+ ret void
+}
+
+define void @release_double_shared_cluster(double %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_double_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_double_shared_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [release_double_shared_cluster_param_1];
+; CHECK-NEXT: st.release.cta.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr addrspace(3) %addr syncscope("cluster") release, align 8
+ ret void
+}
+
+define void @release_double_shared_gpu(double %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_double_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_double_shared_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [release_double_shared_gpu_param_1];
+; CHECK-NEXT: st.release.gpu.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr addrspace(3) %addr syncscope("device") release, align 8
+ ret void
+}
+
+define void @release_double_shared_sys_volatile(double %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_double_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_double_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [release_double_shared_sys_volatile_param_1];
+; CHECK-NEXT: st.release.sys.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr addrspace(3) %addr syncscope("") release, align 8
+ ret void
+}
+
+define void @release_double_shared_cta_volatile(double %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_double_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_double_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [release_double_shared_cta_volatile_param_1];
+; CHECK-NEXT: st.release.sys.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr addrspace(3) %addr syncscope("block") release, align 8
+ ret void
+}
+
+define void @release_double_shared_cluster_volatile(double %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_double_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_double_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [release_double_shared_cluster_volatile_param_1];
+; CHECK-NEXT: st.release.sys.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr addrspace(3) %addr syncscope("cluster") release, align 8
+ ret void
+}
+
+define void @release_double_shared_gpu_volatile(double %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_double_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_double_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [release_double_shared_gpu_volatile_param_1];
+; CHECK-NEXT: st.release.sys.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr addrspace(3) %addr syncscope("device") release, align 8
+ ret void
+}
+
+define void @release_i8_local_sys(i8 %val) {
+; CHECK-LABEL: release_i8_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot768[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot768;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b8 %rs1, [release_i8_local_sys_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b8 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i8 %val, ptr addrspace(5) %slot syncscope("") release, align 1
+ ret void
+}
+
+define void @release_i8_local_cta(i8 %val) {
+; CHECK-LABEL: release_i8_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot769[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot769;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b8 %rs1, [release_i8_local_cta_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b8 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i8 %val, ptr addrspace(5) %slot syncscope("block") release, align 1
+ ret void
+}
+
+define void @release_i8_local_cluster(i8 %val) {
+; CHECK-LABEL: release_i8_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot770[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot770;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b8 %rs1, [release_i8_local_cluster_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b8 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i8 %val, ptr addrspace(5) %slot syncscope("cluster") release, align 1
+ ret void
+}
+
+define void @release_i8_local_gpu(i8 %val) {
+; CHECK-LABEL: release_i8_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot771[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot771;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b8 %rs1, [release_i8_local_gpu_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b8 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i8 %val, ptr addrspace(5) %slot syncscope("device") release, align 1
+ ret void
+}
+
+define void @release_i8_local_sys_volatile(i8 %val) {
+; CHECK-LABEL: release_i8_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot772[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot772;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b8 %rs1, [release_i8_local_sys_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b8 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i8 %val, ptr addrspace(5) %slot syncscope("") release, align 1
+ ret void
+}
+
+define void @release_i8_local_cta_volatile(i8 %val) {
+; CHECK-LABEL: release_i8_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot773[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot773;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b8 %rs1, [release_i8_local_cta_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b8 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i8 %val, ptr addrspace(5) %slot syncscope("block") release, align 1
+ ret void
+}
+
+define void @release_i8_local_cluster_volatile(i8 %val) {
+; CHECK-LABEL: release_i8_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot774[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot774;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b8 %rs1, [release_i8_local_cluster_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b8 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i8 %val, ptr addrspace(5) %slot syncscope("cluster") release, align 1
+ ret void
+}
+
+define void @release_i8_local_gpu_volatile(i8 %val) {
+; CHECK-LABEL: release_i8_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot775[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot775;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b8 %rs1, [release_i8_local_gpu_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b8 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i8 %val, ptr addrspace(5) %slot syncscope("device") release, align 1
+ ret void
+}
+
+define void @release_i16_local_sys(i16 %val) {
+; CHECK-LABEL: release_i16_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot776[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot776;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [release_i16_local_sys_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i16 %val, ptr addrspace(5) %slot syncscope("") release, align 2
+ ret void
+}
+
+define void @release_i16_local_cta(i16 %val) {
+; CHECK-LABEL: release_i16_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot777[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot777;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [release_i16_local_cta_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i16 %val, ptr addrspace(5) %slot syncscope("block") release, align 2
+ ret void
+}
+
+define void @release_i16_local_cluster(i16 %val) {
+; CHECK-LABEL: release_i16_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot778[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot778;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [release_i16_local_cluster_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i16 %val, ptr addrspace(5) %slot syncscope("cluster") release, align 2
+ ret void
+}
+
+define void @release_i16_local_gpu(i16 %val) {
+; CHECK-LABEL: release_i16_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot779[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot779;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [release_i16_local_gpu_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i16 %val, ptr addrspace(5) %slot syncscope("device") release, align 2
+ ret void
+}
+
+define void @release_i16_local_sys_volatile(i16 %val) {
+; CHECK-LABEL: release_i16_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot780[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot780;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [release_i16_local_sys_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i16 %val, ptr addrspace(5) %slot syncscope("") release, align 2
+ ret void
+}
+
+define void @release_i16_local_cta_volatile(i16 %val) {
+; CHECK-LABEL: release_i16_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot781[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot781;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [release_i16_local_cta_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i16 %val, ptr addrspace(5) %slot syncscope("block") release, align 2
+ ret void
+}
+
+define void @release_i16_local_cluster_volatile(i16 %val) {
+; CHECK-LABEL: release_i16_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot782[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot782;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [release_i16_local_cluster_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i16 %val, ptr addrspace(5) %slot syncscope("cluster") release, align 2
+ ret void
+}
+
+define void @release_i16_local_gpu_volatile(i16 %val) {
+; CHECK-LABEL: release_i16_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot783[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot783;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [release_i16_local_gpu_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i16 %val, ptr addrspace(5) %slot syncscope("device") release, align 2
+ ret void
+}
+
+define void @release_i32_local_sys(i32 %val) {
+; CHECK-LABEL: release_i32_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot784[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot784;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [release_i32_local_sys_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i32 %val, ptr addrspace(5) %slot syncscope("") release, align 4
+ ret void
+}
+
+define void @release_i32_local_cta(i32 %val) {
+; CHECK-LABEL: release_i32_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot785[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot785;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [release_i32_local_cta_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i32 %val, ptr addrspace(5) %slot syncscope("block") release, align 4
+ ret void
+}
+
+define void @release_i32_local_cluster(i32 %val) {
+; CHECK-LABEL: release_i32_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot786[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot786;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [release_i32_local_cluster_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i32 %val, ptr addrspace(5) %slot syncscope("cluster") release, align 4
+ ret void
+}
+
+define void @release_i32_local_gpu(i32 %val) {
+; CHECK-LABEL: release_i32_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot787[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot787;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [release_i32_local_gpu_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i32 %val, ptr addrspace(5) %slot syncscope("device") release, align 4
+ ret void
+}
+
+define void @release_i32_local_sys_volatile(i32 %val) {
+; CHECK-LABEL: release_i32_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot788[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot788;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [release_i32_local_sys_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i32 %val, ptr addrspace(5) %slot syncscope("") release, align 4
+ ret void
+}
+
+define void @release_i32_local_cta_volatile(i32 %val) {
+; CHECK-LABEL: release_i32_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot789[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot789;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [release_i32_local_cta_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i32 %val, ptr addrspace(5) %slot syncscope("block") release, align 4
+ ret void
+}
+
+define void @release_i32_local_cluster_volatile(i32 %val) {
+; CHECK-LABEL: release_i32_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot790[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot790;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [release_i32_local_cluster_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i32 %val, ptr addrspace(5) %slot syncscope("cluster") release, align 4
+ ret void
+}
+
+define void @release_i32_local_gpu_volatile(i32 %val) {
+; CHECK-LABEL: release_i32_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot791[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot791;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [release_i32_local_gpu_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i32 %val, ptr addrspace(5) %slot syncscope("device") release, align 4
+ ret void
+}
+
+define void @release_i64_local_sys(i64 %val) {
+; CHECK-LABEL: release_i64_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot792[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot792;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i64_local_sys_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i64 %val, ptr addrspace(5) %slot syncscope("") release, align 8
+ ret void
+}
+
+define void @release_i64_local_cta(i64 %val) {
+; CHECK-LABEL: release_i64_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot793[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot793;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i64_local_cta_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i64 %val, ptr addrspace(5) %slot syncscope("block") release, align 8
+ ret void
+}
+
+define void @release_i64_local_cluster(i64 %val) {
+; CHECK-LABEL: release_i64_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot794[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot794;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i64_local_cluster_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i64 %val, ptr addrspace(5) %slot syncscope("cluster") release, align 8
+ ret void
+}
+
+define void @release_i64_local_gpu(i64 %val) {
+; CHECK-LABEL: release_i64_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot795[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot795;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i64_local_gpu_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i64 %val, ptr addrspace(5) %slot syncscope("device") release, align 8
+ ret void
+}
+
+define void @release_i64_local_sys_volatile(i64 %val) {
+; CHECK-LABEL: release_i64_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot796[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot796;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i64_local_sys_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i64 %val, ptr addrspace(5) %slot syncscope("") release, align 8
+ ret void
+}
+
+define void @release_i64_local_cta_volatile(i64 %val) {
+; CHECK-LABEL: release_i64_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot797[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot797;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i64_local_cta_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i64 %val, ptr addrspace(5) %slot syncscope("block") release, align 8
+ ret void
+}
+
+define void @release_i64_local_cluster_volatile(i64 %val) {
+; CHECK-LABEL: release_i64_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot798[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot798;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i64_local_cluster_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i64 %val, ptr addrspace(5) %slot syncscope("cluster") release, align 8
+ ret void
+}
+
+define void @release_i64_local_gpu_volatile(i64 %val) {
+; CHECK-LABEL: release_i64_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot799[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot799;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i64_local_gpu_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i64 %val, ptr addrspace(5) %slot syncscope("device") release, align 8
+ ret void
+}
+
+define void @release_half_local_sys(half %val) {
+; CHECK-LABEL: release_half_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot800[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot800;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [release_half_local_sys_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic half %val, ptr addrspace(5) %slot syncscope("") release, align 2
+ ret void
+}
+
+define void @release_half_local_cta(half %val) {
+; CHECK-LABEL: release_half_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot801[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot801;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [release_half_local_cta_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic half %val, ptr addrspace(5) %slot syncscope("block") release, align 2
+ ret void
+}
+
+define void @release_half_local_cluster(half %val) {
+; CHECK-LABEL: release_half_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot802[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot802;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [release_half_local_cluster_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic half %val, ptr addrspace(5) %slot syncscope("cluster") release, align 2
+ ret void
+}
+
+define void @release_half_local_gpu(half %val) {
+; CHECK-LABEL: release_half_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot803[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot803;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [release_half_local_gpu_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic half %val, ptr addrspace(5) %slot syncscope("device") release, align 2
+ ret void
+}
+
+define void @release_half_local_sys_volatile(half %val) {
+; CHECK-LABEL: release_half_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot804[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot804;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [release_half_local_sys_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile half %val, ptr addrspace(5) %slot syncscope("") release, align 2
+ ret void
+}
+
+define void @release_half_local_cta_volatile(half %val) {
+; CHECK-LABEL: release_half_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot805[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot805;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [release_half_local_cta_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile half %val, ptr addrspace(5) %slot syncscope("block") release, align 2
+ ret void
+}
+
+define void @release_half_local_cluster_volatile(half %val) {
+; CHECK-LABEL: release_half_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot806[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot806;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [release_half_local_cluster_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile half %val, ptr addrspace(5) %slot syncscope("cluster") release, align 2
+ ret void
+}
+
+define void @release_half_local_gpu_volatile(half %val) {
+; CHECK-LABEL: release_half_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot807[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot807;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [release_half_local_gpu_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile half %val, ptr addrspace(5) %slot syncscope("device") release, align 2
+ ret void
+}
+
+define void @release_bfloat_local_sys(bfloat %val) {
+; CHECK-LABEL: release_bfloat_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot808[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot808;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [release_bfloat_local_sys_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic bfloat %val, ptr addrspace(5) %slot syncscope("") release, align 2
+ ret void
+}
+
+define void @release_bfloat_local_cta(bfloat %val) {
+; CHECK-LABEL: release_bfloat_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot809[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot809;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [release_bfloat_local_cta_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic bfloat %val, ptr addrspace(5) %slot syncscope("block") release, align 2
+ ret void
+}
+
+define void @release_bfloat_local_cluster(bfloat %val) {
+; CHECK-LABEL: release_bfloat_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot810[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot810;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [release_bfloat_local_cluster_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic bfloat %val, ptr addrspace(5) %slot syncscope("cluster") release, align 2
+ ret void
+}
+
+define void @release_bfloat_local_gpu(bfloat %val) {
+; CHECK-LABEL: release_bfloat_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot811[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot811;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [release_bfloat_local_gpu_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic bfloat %val, ptr addrspace(5) %slot syncscope("device") release, align 2
+ ret void
+}
+
+define void @release_bfloat_local_sys_volatile(bfloat %val) {
+; CHECK-LABEL: release_bfloat_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot812[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot812;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [release_bfloat_local_sys_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile bfloat %val, ptr addrspace(5) %slot syncscope("") release, align 2
+ ret void
+}
+
+define void @release_bfloat_local_cta_volatile(bfloat %val) {
+; CHECK-LABEL: release_bfloat_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot813[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot813;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [release_bfloat_local_cta_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile bfloat %val, ptr addrspace(5) %slot syncscope("block") release, align 2
+ ret void
+}
+
+define void @release_bfloat_local_cluster_volatile(bfloat %val) {
+; CHECK-LABEL: release_bfloat_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot814[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot814;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [release_bfloat_local_cluster_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile bfloat %val, ptr addrspace(5) %slot syncscope("cluster") release, align 2
+ ret void
+}
+
+define void @release_bfloat_local_gpu_volatile(bfloat %val) {
+; CHECK-LABEL: release_bfloat_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot815[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot815;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [release_bfloat_local_gpu_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile bfloat %val, ptr addrspace(5) %slot syncscope("device") release, align 2
+ ret void
+}
+
+define void @release_float_local_sys(float %val) {
+; CHECK-LABEL: release_float_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot816[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot816;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [release_float_local_sys_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic float %val, ptr addrspace(5) %slot syncscope("") release, align 4
+ ret void
+}
+
+define void @release_float_local_cta(float %val) {
+; CHECK-LABEL: release_float_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot817[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot817;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [release_float_local_cta_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic float %val, ptr addrspace(5) %slot syncscope("block") release, align 4
+ ret void
+}
+
+define void @release_float_local_cluster(float %val) {
+; CHECK-LABEL: release_float_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot818[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot818;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [release_float_local_cluster_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic float %val, ptr addrspace(5) %slot syncscope("cluster") release, align 4
+ ret void
+}
+
+define void @release_float_local_gpu(float %val) {
+; CHECK-LABEL: release_float_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot819[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot819;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [release_float_local_gpu_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic float %val, ptr addrspace(5) %slot syncscope("device") release, align 4
+ ret void
+}
+
+define void @release_float_local_sys_volatile(float %val) {
+; CHECK-LABEL: release_float_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot820[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot820;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [release_float_local_sys_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile float %val, ptr addrspace(5) %slot syncscope("") release, align 4
+ ret void
+}
+
+define void @release_float_local_cta_volatile(float %val) {
+; CHECK-LABEL: release_float_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot821[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot821;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [release_float_local_cta_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile float %val, ptr addrspace(5) %slot syncscope("block") release, align 4
+ ret void
+}
+
+define void @release_float_local_cluster_volatile(float %val) {
+; CHECK-LABEL: release_float_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot822[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot822;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [release_float_local_cluster_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile float %val, ptr addrspace(5) %slot syncscope("cluster") release, align 4
+ ret void
+}
+
+define void @release_float_local_gpu_volatile(float %val) {
+; CHECK-LABEL: release_float_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot823[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot823;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [release_float_local_gpu_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile float %val, ptr addrspace(5) %slot syncscope("device") release, align 4
+ ret void
+}
+
+define void @release_double_local_sys(double %val) {
+; CHECK-LABEL: release_double_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot824[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot824;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_double_local_sys_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic double %val, ptr addrspace(5) %slot syncscope("") release, align 8
+ ret void
+}
+
+define void @release_double_local_cta(double %val) {
+; CHECK-LABEL: release_double_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot825[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot825;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_double_local_cta_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic double %val, ptr addrspace(5) %slot syncscope("block") release, align 8
+ ret void
+}
+
+define void @release_double_local_cluster(double %val) {
+; CHECK-LABEL: release_double_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot826[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot826;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_double_local_cluster_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic double %val, ptr addrspace(5) %slot syncscope("cluster") release, align 8
+ ret void
+}
+
+define void @release_double_local_gpu(double %val) {
+; CHECK-LABEL: release_double_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot827[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot827;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_double_local_gpu_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic double %val, ptr addrspace(5) %slot syncscope("device") release, align 8
+ ret void
+}
+
+define void @release_double_local_sys_volatile(double %val) {
+; CHECK-LABEL: release_double_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot828[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot828;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_double_local_sys_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile double %val, ptr addrspace(5) %slot syncscope("") release, align 8
+ ret void
+}
+
+define void @release_double_local_cta_volatile(double %val) {
+; CHECK-LABEL: release_double_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot829[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot829;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_double_local_cta_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile double %val, ptr addrspace(5) %slot syncscope("block") release, align 8
+ ret void
+}
+
+define void @release_double_local_cluster_volatile(double %val) {
+; CHECK-LABEL: release_double_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot830[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot830;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_double_local_cluster_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile double %val, ptr addrspace(5) %slot syncscope("cluster") release, align 8
+ ret void
+}
+
+define void @release_double_local_gpu_volatile(double %val) {
+; CHECK-LABEL: release_double_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot831[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot831;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_double_local_gpu_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile double %val, ptr addrspace(5) %slot syncscope("device") release, align 8
+ ret void
+}
+
+define void @seq_cst_i8_generic_sys(i8 %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_i8_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [seq_cst_i8_generic_sys_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_generic_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr %addr syncscope("") seq_cst, align 1
+ ret void
+}
+
+define void @seq_cst_i8_generic_cta(i8 %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_i8_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [seq_cst_i8_generic_cta_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_generic_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr %addr syncscope("block") seq_cst, align 1
+ ret void
+}
+
+define void @seq_cst_i8_generic_cluster(i8 %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_i8_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [seq_cst_i8_generic_cluster_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_generic_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cta.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr %addr syncscope("cluster") seq_cst, align 1
+ ret void
+}
+
+define void @seq_cst_i8_generic_gpu(i8 %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_i8_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [seq_cst_i8_generic_gpu_param_0];
+; CHECK-NEXT: fence.sc.gpu;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_generic_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr %addr syncscope("device") seq_cst, align 1
+ ret void
+}
+
+define void @seq_cst_i8_generic_sys_volatile(i8 %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_i8_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [seq_cst_i8_generic_sys_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_generic_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr %addr syncscope("") seq_cst, align 1
+ ret void
+}
+
+define void @seq_cst_i8_generic_cta_volatile(i8 %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_i8_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [seq_cst_i8_generic_cta_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_generic_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr %addr syncscope("block") seq_cst, align 1
+ ret void
+}
+
+define void @seq_cst_i8_generic_cluster_volatile(i8 %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_i8_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [seq_cst_i8_generic_cluster_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_generic_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr %addr syncscope("cluster") seq_cst, align 1
+ ret void
+}
+
+define void @seq_cst_i8_generic_gpu_volatile(i8 %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_i8_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [seq_cst_i8_generic_gpu_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_generic_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr %addr syncscope("device") seq_cst, align 1
+ ret void
+}
+
+define void @seq_cst_i16_generic_sys(i16 %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_i16_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_i16_generic_sys_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_generic_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr %addr syncscope("") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_i16_generic_cta(i16 %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_i16_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_i16_generic_cta_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_generic_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr %addr syncscope("block") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_i16_generic_cluster(i16 %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_i16_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_i16_generic_cluster_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_generic_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cta.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr %addr syncscope("cluster") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_i16_generic_gpu(i16 %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_i16_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_i16_generic_gpu_param_0];
+; CHECK-NEXT: fence.sc.gpu;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_generic_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr %addr syncscope("device") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_i16_generic_sys_volatile(i16 %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_i16_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_i16_generic_sys_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_generic_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr %addr syncscope("") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_i16_generic_cta_volatile(i16 %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_i16_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_i16_generic_cta_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_generic_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr %addr syncscope("block") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_i16_generic_cluster_volatile(i16 %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_i16_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_i16_generic_cluster_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_generic_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr %addr syncscope("cluster") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_i16_generic_gpu_volatile(i16 %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_i16_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_i16_generic_gpu_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_generic_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr %addr syncscope("device") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_i32_generic_sys(i32 %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_i32_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_i32_generic_sys_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_generic_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr %addr syncscope("") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_i32_generic_cta(i32 %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_i32_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_i32_generic_cta_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_generic_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr %addr syncscope("block") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_i32_generic_cluster(i32 %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_i32_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_i32_generic_cluster_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_generic_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cta.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr %addr syncscope("cluster") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_i32_generic_gpu(i32 %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_i32_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_i32_generic_gpu_param_0];
+; CHECK-NEXT: fence.sc.gpu;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_generic_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr %addr syncscope("device") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_i32_generic_sys_volatile(i32 %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_i32_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_i32_generic_sys_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_generic_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr %addr syncscope("") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_i32_generic_cta_volatile(i32 %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_i32_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_i32_generic_cta_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_generic_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr %addr syncscope("block") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_i32_generic_cluster_volatile(i32 %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_i32_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_i32_generic_cluster_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_generic_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr %addr syncscope("cluster") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_i32_generic_gpu_volatile(i32 %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_i32_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_i32_generic_gpu_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_generic_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr %addr syncscope("device") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_i64_generic_sys(i64 %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_i64_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_generic_sys_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_i64_generic_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr %addr syncscope("") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_i64_generic_cta(i64 %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_i64_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_generic_cta_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_i64_generic_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr %addr syncscope("block") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_i64_generic_cluster(i64 %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_i64_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_generic_cluster_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_i64_generic_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cta.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr %addr syncscope("cluster") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_i64_generic_gpu(i64 %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_i64_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_generic_gpu_param_0];
+; CHECK-NEXT: fence.sc.gpu;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_i64_generic_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr %addr syncscope("device") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_i64_generic_sys_volatile(i64 %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_i64_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_generic_sys_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_i64_generic_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr %addr syncscope("") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_i64_generic_cta_volatile(i64 %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_i64_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_generic_cta_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_i64_generic_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr %addr syncscope("block") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_i64_generic_cluster_volatile(i64 %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_i64_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_generic_cluster_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_i64_generic_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr %addr syncscope("cluster") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_i64_generic_gpu_volatile(i64 %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_i64_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_generic_gpu_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_i64_generic_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr %addr syncscope("device") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_half_generic_sys(half %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_half_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_half_generic_sys_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_generic_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr %addr syncscope("") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_half_generic_cta(half %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_half_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_half_generic_cta_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_generic_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr %addr syncscope("block") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_half_generic_cluster(half %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_half_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_half_generic_cluster_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_generic_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cta.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr %addr syncscope("cluster") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_half_generic_gpu(half %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_half_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_half_generic_gpu_param_0];
+; CHECK-NEXT: fence.sc.gpu;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_generic_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr %addr syncscope("device") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_half_generic_sys_volatile(half %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_half_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_half_generic_sys_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_generic_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr %addr syncscope("") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_half_generic_cta_volatile(half %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_half_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_half_generic_cta_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_generic_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr %addr syncscope("block") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_half_generic_cluster_volatile(half %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_half_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_half_generic_cluster_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_generic_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr %addr syncscope("cluster") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_half_generic_gpu_volatile(half %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_half_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_half_generic_gpu_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_generic_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr %addr syncscope("device") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_bfloat_generic_sys(bfloat %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_bfloat_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_bfloat_generic_sys_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_generic_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr %addr syncscope("") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_bfloat_generic_cta(bfloat %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_bfloat_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_bfloat_generic_cta_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_generic_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr %addr syncscope("block") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_bfloat_generic_cluster(bfloat %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_bfloat_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_bfloat_generic_cluster_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_generic_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cta.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr %addr syncscope("cluster") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_bfloat_generic_gpu(bfloat %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_bfloat_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_bfloat_generic_gpu_param_0];
+; CHECK-NEXT: fence.sc.gpu;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_generic_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr %addr syncscope("device") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_bfloat_generic_sys_volatile(bfloat %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_bfloat_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_bfloat_generic_sys_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_generic_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr %addr syncscope("") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_bfloat_generic_cta_volatile(bfloat %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_bfloat_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_bfloat_generic_cta_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_generic_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr %addr syncscope("block") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_bfloat_generic_cluster_volatile(bfloat %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_bfloat_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_bfloat_generic_cluster_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_generic_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr %addr syncscope("cluster") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_bfloat_generic_gpu_volatile(bfloat %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_bfloat_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_bfloat_generic_gpu_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_generic_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr %addr syncscope("device") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_float_generic_sys(float %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_float_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_float_generic_sys_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_generic_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr %addr syncscope("") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_float_generic_cta(float %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_float_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_float_generic_cta_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_generic_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr %addr syncscope("block") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_float_generic_cluster(float %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_float_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_float_generic_cluster_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_generic_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cta.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr %addr syncscope("cluster") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_float_generic_gpu(float %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_float_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_float_generic_gpu_param_0];
+; CHECK-NEXT: fence.sc.gpu;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_generic_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr %addr syncscope("device") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_float_generic_sys_volatile(float %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_float_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_float_generic_sys_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_generic_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr %addr syncscope("") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_float_generic_cta_volatile(float %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_float_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_float_generic_cta_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_generic_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr %addr syncscope("block") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_float_generic_cluster_volatile(float %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_float_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_float_generic_cluster_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_generic_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr %addr syncscope("cluster") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_float_generic_gpu_volatile(float %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_float_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_float_generic_gpu_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_generic_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr %addr syncscope("device") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_double_generic_sys(double %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_double_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_generic_sys_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_double_generic_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr %addr syncscope("") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_double_generic_cta(double %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_double_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_generic_cta_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_double_generic_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr %addr syncscope("block") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_double_generic_cluster(double %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_double_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_generic_cluster_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_double_generic_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cta.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr %addr syncscope("cluster") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_double_generic_gpu(double %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_double_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_generic_gpu_param_0];
+; CHECK-NEXT: fence.sc.gpu;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_double_generic_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr %addr syncscope("device") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_double_generic_sys_volatile(double %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_double_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_generic_sys_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_double_generic_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr %addr syncscope("") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_double_generic_cta_volatile(double %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_double_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_generic_cta_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_double_generic_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr %addr syncscope("block") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_double_generic_cluster_volatile(double %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_double_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_generic_cluster_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_double_generic_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr %addr syncscope("cluster") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_double_generic_gpu_volatile(double %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_double_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_generic_gpu_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_double_generic_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr %addr syncscope("device") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_i8_global_sys(i8 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i8_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [seq_cst_i8_global_sys_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_global_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.global.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr addrspace(1) %addr syncscope("") seq_cst, align 1
+ ret void
+}
+
+define void @seq_cst_i8_global_cta(i8 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i8_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [seq_cst_i8_global_cta_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_global_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.global.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr addrspace(1) %addr syncscope("block") seq_cst, align 1
+ ret void
+}
+
+define void @seq_cst_i8_global_cluster(i8 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i8_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [seq_cst_i8_global_cluster_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_global_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cta.global.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr addrspace(1) %addr syncscope("cluster") seq_cst, align 1
+ ret void
+}
+
+define void @seq_cst_i8_global_gpu(i8 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i8_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [seq_cst_i8_global_gpu_param_0];
+; CHECK-NEXT: fence.sc.gpu;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_global_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.global.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr addrspace(1) %addr syncscope("device") seq_cst, align 1
+ ret void
+}
+
+define void @seq_cst_i8_global_sys_volatile(i8 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i8_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [seq_cst_i8_global_sys_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_global_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr addrspace(1) %addr syncscope("") seq_cst, align 1
+ ret void
+}
+
+define void @seq_cst_i8_global_cta_volatile(i8 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i8_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [seq_cst_i8_global_cta_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_global_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr addrspace(1) %addr syncscope("block") seq_cst, align 1
+ ret void
+}
+
+define void @seq_cst_i8_global_cluster_volatile(i8 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i8_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [seq_cst_i8_global_cluster_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_global_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr addrspace(1) %addr syncscope("cluster") seq_cst, align 1
+ ret void
+}
+
+define void @seq_cst_i8_global_gpu_volatile(i8 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i8_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [seq_cst_i8_global_gpu_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_global_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr addrspace(1) %addr syncscope("device") seq_cst, align 1
+ ret void
+}
+
+define void @seq_cst_i16_global_sys(i16 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i16_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_i16_global_sys_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_global_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr addrspace(1) %addr syncscope("") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_i16_global_cta(i16 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i16_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_i16_global_cta_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_global_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr addrspace(1) %addr syncscope("block") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_i16_global_cluster(i16 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i16_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_i16_global_cluster_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_global_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cta.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr addrspace(1) %addr syncscope("cluster") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_i16_global_gpu(i16 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i16_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_i16_global_gpu_param_0];
+; CHECK-NEXT: fence.sc.gpu;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_global_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr addrspace(1) %addr syncscope("device") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_i16_global_sys_volatile(i16 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i16_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_i16_global_sys_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_global_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr addrspace(1) %addr syncscope("") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_i16_global_cta_volatile(i16 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i16_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_i16_global_cta_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_global_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr addrspace(1) %addr syncscope("block") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_i16_global_cluster_volatile(i16 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i16_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_i16_global_cluster_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_global_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr addrspace(1) %addr syncscope("cluster") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_i16_global_gpu_volatile(i16 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i16_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_i16_global_gpu_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_global_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr addrspace(1) %addr syncscope("device") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_i32_global_sys(i32 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i32_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_i32_global_sys_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_global_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr addrspace(1) %addr syncscope("") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_i32_global_cta(i32 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i32_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_i32_global_cta_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_global_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr addrspace(1) %addr syncscope("block") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_i32_global_cluster(i32 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i32_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_i32_global_cluster_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_global_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cta.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr addrspace(1) %addr syncscope("cluster") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_i32_global_gpu(i32 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i32_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_i32_global_gpu_param_0];
+; CHECK-NEXT: fence.sc.gpu;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_global_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr addrspace(1) %addr syncscope("device") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_i32_global_sys_volatile(i32 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i32_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_i32_global_sys_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_global_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr addrspace(1) %addr syncscope("") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_i32_global_cta_volatile(i32 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i32_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_i32_global_cta_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_global_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr addrspace(1) %addr syncscope("block") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_i32_global_cluster_volatile(i32 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i32_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_i32_global_cluster_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_global_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr addrspace(1) %addr syncscope("cluster") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_i32_global_gpu_volatile(i32 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i32_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_i32_global_gpu_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_global_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr addrspace(1) %addr syncscope("device") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_i64_global_sys(i64 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i64_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_global_sys_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_i64_global_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr addrspace(1) %addr syncscope("") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_i64_global_cta(i64 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i64_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_global_cta_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_i64_global_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr addrspace(1) %addr syncscope("block") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_i64_global_cluster(i64 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i64_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_global_cluster_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_i64_global_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cta.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr addrspace(1) %addr syncscope("cluster") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_i64_global_gpu(i64 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i64_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_global_gpu_param_0];
+; CHECK-NEXT: fence.sc.gpu;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_i64_global_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr addrspace(1) %addr syncscope("device") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_i64_global_sys_volatile(i64 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i64_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_global_sys_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_i64_global_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr addrspace(1) %addr syncscope("") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_i64_global_cta_volatile(i64 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i64_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_global_cta_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_i64_global_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr addrspace(1) %addr syncscope("block") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_i64_global_cluster_volatile(i64 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i64_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_global_cluster_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_i64_global_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr addrspace(1) %addr syncscope("cluster") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_i64_global_gpu_volatile(i64 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i64_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_global_gpu_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_i64_global_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr addrspace(1) %addr syncscope("device") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_half_global_sys(half %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_half_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_half_global_sys_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_global_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr addrspace(1) %addr syncscope("") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_half_global_cta(half %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_half_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_half_global_cta_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_global_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr addrspace(1) %addr syncscope("block") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_half_global_cluster(half %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_half_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_half_global_cluster_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_global_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cta.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr addrspace(1) %addr syncscope("cluster") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_half_global_gpu(half %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_half_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_half_global_gpu_param_0];
+; CHECK-NEXT: fence.sc.gpu;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_global_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr addrspace(1) %addr syncscope("device") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_half_global_sys_volatile(half %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_half_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_half_global_sys_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_global_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr addrspace(1) %addr syncscope("") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_half_global_cta_volatile(half %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_half_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_half_global_cta_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_global_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr addrspace(1) %addr syncscope("block") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_half_global_cluster_volatile(half %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_half_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_half_global_cluster_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_global_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr addrspace(1) %addr syncscope("cluster") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_half_global_gpu_volatile(half %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_half_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_half_global_gpu_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_global_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr addrspace(1) %addr syncscope("device") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_bfloat_global_sys(bfloat %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_bfloat_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_bfloat_global_sys_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_global_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr addrspace(1) %addr syncscope("") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_bfloat_global_cta(bfloat %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_bfloat_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_bfloat_global_cta_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_global_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr addrspace(1) %addr syncscope("block") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_bfloat_global_cluster(bfloat %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_bfloat_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_bfloat_global_cluster_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_global_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cta.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr addrspace(1) %addr syncscope("cluster") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_bfloat_global_gpu(bfloat %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_bfloat_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_bfloat_global_gpu_param_0];
+; CHECK-NEXT: fence.sc.gpu;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_global_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr addrspace(1) %addr syncscope("device") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_bfloat_global_sys_volatile(bfloat %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_bfloat_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_bfloat_global_sys_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_global_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr addrspace(1) %addr syncscope("") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_bfloat_global_cta_volatile(bfloat %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_bfloat_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_bfloat_global_cta_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_global_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr addrspace(1) %addr syncscope("block") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_bfloat_global_cluster_volatile(bfloat %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_bfloat_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_bfloat_global_cluster_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_global_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr addrspace(1) %addr syncscope("cluster") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_bfloat_global_gpu_volatile(bfloat %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_bfloat_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_bfloat_global_gpu_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_global_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr addrspace(1) %addr syncscope("device") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_float_global_sys(float %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_float_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_float_global_sys_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_global_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr addrspace(1) %addr syncscope("") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_float_global_cta(float %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_float_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_float_global_cta_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_global_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr addrspace(1) %addr syncscope("block") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_float_global_cluster(float %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_float_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_float_global_cluster_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_global_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cta.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr addrspace(1) %addr syncscope("cluster") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_float_global_gpu(float %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_float_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_float_global_gpu_param_0];
+; CHECK-NEXT: fence.sc.gpu;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_global_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr addrspace(1) %addr syncscope("device") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_float_global_sys_volatile(float %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_float_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_float_global_sys_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_global_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr addrspace(1) %addr syncscope("") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_float_global_cta_volatile(float %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_float_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_float_global_cta_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_global_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr addrspace(1) %addr syncscope("block") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_float_global_cluster_volatile(float %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_float_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_float_global_cluster_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_global_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr addrspace(1) %addr syncscope("cluster") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_float_global_gpu_volatile(float %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_float_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_float_global_gpu_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_global_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr addrspace(1) %addr syncscope("device") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_double_global_sys(double %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_double_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_global_sys_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_double_global_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr addrspace(1) %addr syncscope("") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_double_global_cta(double %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_double_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_global_cta_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_double_global_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr addrspace(1) %addr syncscope("block") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_double_global_cluster(double %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_double_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_global_cluster_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_double_global_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cta.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr addrspace(1) %addr syncscope("cluster") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_double_global_gpu(double %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_double_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_global_gpu_param_0];
+; CHECK-NEXT: fence.sc.gpu;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_double_global_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr addrspace(1) %addr syncscope("device") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_double_global_sys_volatile(double %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_double_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_global_sys_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_double_global_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr addrspace(1) %addr syncscope("") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_double_global_cta_volatile(double %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_double_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_global_cta_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_double_global_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr addrspace(1) %addr syncscope("block") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_double_global_cluster_volatile(double %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_double_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_global_cluster_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_double_global_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr addrspace(1) %addr syncscope("cluster") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_double_global_gpu_volatile(double %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_double_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_global_gpu_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_double_global_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr addrspace(1) %addr syncscope("device") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_i8_shared_sys(i8 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i8_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [seq_cst_i8_shared_sys_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_shared_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.shared.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr addrspace(3) %addr syncscope("") seq_cst, align 1
+ ret void
+}
+
+define void @seq_cst_i8_shared_cta(i8 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i8_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [seq_cst_i8_shared_cta_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_shared_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.shared.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr addrspace(3) %addr syncscope("block") seq_cst, align 1
+ ret void
+}
+
+define void @seq_cst_i8_shared_cluster(i8 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i8_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [seq_cst_i8_shared_cluster_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_shared_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cta.shared.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr addrspace(3) %addr syncscope("cluster") seq_cst, align 1
+ ret void
+}
+
+define void @seq_cst_i8_shared_gpu(i8 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i8_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [seq_cst_i8_shared_gpu_param_0];
+; CHECK-NEXT: fence.sc.gpu;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_shared_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.shared.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr addrspace(3) %addr syncscope("device") seq_cst, align 1
+ ret void
+}
+
+define void @seq_cst_i8_shared_sys_volatile(i8 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i8_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [seq_cst_i8_shared_sys_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_shared_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr addrspace(3) %addr syncscope("") seq_cst, align 1
+ ret void
+}
+
+define void @seq_cst_i8_shared_cta_volatile(i8 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i8_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [seq_cst_i8_shared_cta_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_shared_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr addrspace(3) %addr syncscope("block") seq_cst, align 1
+ ret void
+}
+
+define void @seq_cst_i8_shared_cluster_volatile(i8 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i8_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [seq_cst_i8_shared_cluster_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_shared_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr addrspace(3) %addr syncscope("cluster") seq_cst, align 1
+ ret void
+}
+
+define void @seq_cst_i8_shared_gpu_volatile(i8 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i8_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [seq_cst_i8_shared_gpu_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_shared_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr addrspace(3) %addr syncscope("device") seq_cst, align 1
+ ret void
+}
+
+define void @seq_cst_i16_shared_sys(i16 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i16_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_i16_shared_sys_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_shared_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr addrspace(3) %addr syncscope("") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_i16_shared_cta(i16 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i16_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_i16_shared_cta_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_shared_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr addrspace(3) %addr syncscope("block") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_i16_shared_cluster(i16 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i16_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_i16_shared_cluster_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_shared_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cta.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr addrspace(3) %addr syncscope("cluster") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_i16_shared_gpu(i16 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i16_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_i16_shared_gpu_param_0];
+; CHECK-NEXT: fence.sc.gpu;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_shared_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr addrspace(3) %addr syncscope("device") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_i16_shared_sys_volatile(i16 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i16_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_i16_shared_sys_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_shared_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr addrspace(3) %addr syncscope("") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_i16_shared_cta_volatile(i16 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i16_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_i16_shared_cta_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_shared_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr addrspace(3) %addr syncscope("block") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_i16_shared_cluster_volatile(i16 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i16_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_i16_shared_cluster_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_shared_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr addrspace(3) %addr syncscope("cluster") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_i16_shared_gpu_volatile(i16 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i16_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_i16_shared_gpu_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_shared_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr addrspace(3) %addr syncscope("device") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_i32_shared_sys(i32 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i32_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_i32_shared_sys_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_shared_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr addrspace(3) %addr syncscope("") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_i32_shared_cta(i32 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i32_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_i32_shared_cta_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_shared_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr addrspace(3) %addr syncscope("block") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_i32_shared_cluster(i32 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i32_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_i32_shared_cluster_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_shared_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cta.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr addrspace(3) %addr syncscope("cluster") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_i32_shared_gpu(i32 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i32_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_i32_shared_gpu_param_0];
+; CHECK-NEXT: fence.sc.gpu;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_shared_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr addrspace(3) %addr syncscope("device") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_i32_shared_sys_volatile(i32 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i32_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_i32_shared_sys_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_shared_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr addrspace(3) %addr syncscope("") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_i32_shared_cta_volatile(i32 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i32_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_i32_shared_cta_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_shared_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr addrspace(3) %addr syncscope("block") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_i32_shared_cluster_volatile(i32 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i32_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_i32_shared_cluster_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_shared_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr addrspace(3) %addr syncscope("cluster") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_i32_shared_gpu_volatile(i32 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i32_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_i32_shared_gpu_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_shared_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr addrspace(3) %addr syncscope("device") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_i64_shared_sys(i64 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i64_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_shared_sys_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_i64_shared_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr addrspace(3) %addr syncscope("") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_i64_shared_cta(i64 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i64_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_shared_cta_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_i64_shared_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr addrspace(3) %addr syncscope("block") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_i64_shared_cluster(i64 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i64_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_shared_cluster_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_i64_shared_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cta.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr addrspace(3) %addr syncscope("cluster") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_i64_shared_gpu(i64 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i64_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_shared_gpu_param_0];
+; CHECK-NEXT: fence.sc.gpu;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_i64_shared_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr addrspace(3) %addr syncscope("device") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_i64_shared_sys_volatile(i64 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i64_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_shared_sys_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_i64_shared_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr addrspace(3) %addr syncscope("") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_i64_shared_cta_volatile(i64 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i64_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_shared_cta_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_i64_shared_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr addrspace(3) %addr syncscope("block") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_i64_shared_cluster_volatile(i64 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i64_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_shared_cluster_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_i64_shared_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr addrspace(3) %addr syncscope("cluster") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_i64_shared_gpu_volatile(i64 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i64_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_shared_gpu_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_i64_shared_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr addrspace(3) %addr syncscope("device") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_half_shared_sys(half %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_half_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_half_shared_sys_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_shared_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr addrspace(3) %addr syncscope("") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_half_shared_cta(half %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_half_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_half_shared_cta_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_shared_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr addrspace(3) %addr syncscope("block") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_half_shared_cluster(half %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_half_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_half_shared_cluster_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_shared_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cta.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr addrspace(3) %addr syncscope("cluster") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_half_shared_gpu(half %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_half_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_half_shared_gpu_param_0];
+; CHECK-NEXT: fence.sc.gpu;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_shared_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr addrspace(3) %addr syncscope("device") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_half_shared_sys_volatile(half %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_half_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_half_shared_sys_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_shared_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr addrspace(3) %addr syncscope("") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_half_shared_cta_volatile(half %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_half_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_half_shared_cta_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_shared_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr addrspace(3) %addr syncscope("block") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_half_shared_cluster_volatile(half %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_half_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_half_shared_cluster_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_shared_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr addrspace(3) %addr syncscope("cluster") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_half_shared_gpu_volatile(half %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_half_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_half_shared_gpu_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_shared_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr addrspace(3) %addr syncscope("device") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_bfloat_shared_sys(bfloat %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_bfloat_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_bfloat_shared_sys_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_shared_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr addrspace(3) %addr syncscope("") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_bfloat_shared_cta(bfloat %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_bfloat_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_bfloat_shared_cta_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_shared_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr addrspace(3) %addr syncscope("block") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_bfloat_shared_cluster(bfloat %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_bfloat_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_bfloat_shared_cluster_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_shared_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cta.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr addrspace(3) %addr syncscope("cluster") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_bfloat_shared_gpu(bfloat %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_bfloat_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_bfloat_shared_gpu_param_0];
+; CHECK-NEXT: fence.sc.gpu;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_shared_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr addrspace(3) %addr syncscope("device") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_bfloat_shared_sys_volatile(bfloat %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_bfloat_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_bfloat_shared_sys_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_shared_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr addrspace(3) %addr syncscope("") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_bfloat_shared_cta_volatile(bfloat %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_bfloat_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_bfloat_shared_cta_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_shared_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr addrspace(3) %addr syncscope("block") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_bfloat_shared_cluster_volatile(bfloat %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_bfloat_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_bfloat_shared_cluster_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_shared_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr addrspace(3) %addr syncscope("cluster") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_bfloat_shared_gpu_volatile(bfloat %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_bfloat_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_bfloat_shared_gpu_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_shared_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr addrspace(3) %addr syncscope("device") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_float_shared_sys(float %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_float_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_float_shared_sys_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_shared_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr addrspace(3) %addr syncscope("") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_float_shared_cta(float %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_float_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_float_shared_cta_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_shared_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr addrspace(3) %addr syncscope("block") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_float_shared_cluster(float %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_float_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_float_shared_cluster_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_shared_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cta.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr addrspace(3) %addr syncscope("cluster") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_float_shared_gpu(float %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_float_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_float_shared_gpu_param_0];
+; CHECK-NEXT: fence.sc.gpu;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_shared_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr addrspace(3) %addr syncscope("device") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_float_shared_sys_volatile(float %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_float_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_float_shared_sys_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_shared_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr addrspace(3) %addr syncscope("") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_float_shared_cta_volatile(float %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_float_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_float_shared_cta_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_shared_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr addrspace(3) %addr syncscope("block") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_float_shared_cluster_volatile(float %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_float_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_float_shared_cluster_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_shared_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr addrspace(3) %addr syncscope("cluster") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_float_shared_gpu_volatile(float %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_float_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_float_shared_gpu_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_shared_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr addrspace(3) %addr syncscope("device") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_double_shared_sys(double %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_double_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_shared_sys_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_double_shared_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr addrspace(3) %addr syncscope("") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_double_shared_cta(double %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_double_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_shared_cta_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_double_shared_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr addrspace(3) %addr syncscope("block") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_double_shared_cluster(double %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_double_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_shared_cluster_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_double_shared_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cta.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr addrspace(3) %addr syncscope("cluster") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_double_shared_gpu(double %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_double_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_shared_gpu_param_0];
+; CHECK-NEXT: fence.sc.gpu;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_double_shared_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr addrspace(3) %addr syncscope("device") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_double_shared_sys_volatile(double %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_double_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_shared_sys_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_double_shared_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr addrspace(3) %addr syncscope("") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_double_shared_cta_volatile(double %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_double_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_shared_cta_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_double_shared_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr addrspace(3) %addr syncscope("block") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_double_shared_cluster_volatile(double %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_double_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_shared_cluster_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_double_shared_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr addrspace(3) %addr syncscope("cluster") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_double_shared_gpu_volatile(double %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_double_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_shared_gpu_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_double_shared_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr addrspace(3) %addr syncscope("device") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_i8_local_sys(i8 %val) {
+; CHECK-LABEL: seq_cst_i8_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot1024[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1024;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b8 %rs1, [seq_cst_i8_local_sys_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b8 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i8 %val, ptr addrspace(5) %slot syncscope("") seq_cst, align 1
+ ret void
+}
+
+define void @seq_cst_i8_local_cta(i8 %val) {
+; CHECK-LABEL: seq_cst_i8_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot1025[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1025;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b8 %rs1, [seq_cst_i8_local_cta_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b8 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i8 %val, ptr addrspace(5) %slot syncscope("block") seq_cst, align 1
+ ret void
+}
+
+define void @seq_cst_i8_local_cluster(i8 %val) {
+; CHECK-LABEL: seq_cst_i8_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot1026[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1026;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b8 %rs1, [seq_cst_i8_local_cluster_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b8 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i8 %val, ptr addrspace(5) %slot syncscope("cluster") seq_cst, align 1
+ ret void
+}
+
+define void @seq_cst_i8_local_gpu(i8 %val) {
+; CHECK-LABEL: seq_cst_i8_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot1027[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1027;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b8 %rs1, [seq_cst_i8_local_gpu_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b8 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i8 %val, ptr addrspace(5) %slot syncscope("device") seq_cst, align 1
+ ret void
+}
+
+define void @seq_cst_i8_local_sys_volatile(i8 %val) {
+; CHECK-LABEL: seq_cst_i8_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot1028[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1028;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b8 %rs1, [seq_cst_i8_local_sys_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b8 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i8 %val, ptr addrspace(5) %slot syncscope("") seq_cst, align 1
+ ret void
+}
+
+define void @seq_cst_i8_local_cta_volatile(i8 %val) {
+; CHECK-LABEL: seq_cst_i8_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot1029[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1029;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b8 %rs1, [seq_cst_i8_local_cta_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b8 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i8 %val, ptr addrspace(5) %slot syncscope("block") seq_cst, align 1
+ ret void
+}
+
+define void @seq_cst_i8_local_cluster_volatile(i8 %val) {
+; CHECK-LABEL: seq_cst_i8_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot1030[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1030;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b8 %rs1, [seq_cst_i8_local_cluster_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b8 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i8 %val, ptr addrspace(5) %slot syncscope("cluster") seq_cst, align 1
+ ret void
+}
+
+define void @seq_cst_i8_local_gpu_volatile(i8 %val) {
+; CHECK-LABEL: seq_cst_i8_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot1031[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1031;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b8 %rs1, [seq_cst_i8_local_gpu_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b8 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i8 %val, ptr addrspace(5) %slot syncscope("device") seq_cst, align 1
+ ret void
+}
+
+define void @seq_cst_i16_local_sys(i16 %val) {
+; CHECK-LABEL: seq_cst_i16_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1032[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1032;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_i16_local_sys_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i16 %val, ptr addrspace(5) %slot syncscope("") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_i16_local_cta(i16 %val) {
+; CHECK-LABEL: seq_cst_i16_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1033[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1033;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_i16_local_cta_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i16 %val, ptr addrspace(5) %slot syncscope("block") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_i16_local_cluster(i16 %val) {
+; CHECK-LABEL: seq_cst_i16_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1034[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1034;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_i16_local_cluster_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i16 %val, ptr addrspace(5) %slot syncscope("cluster") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_i16_local_gpu(i16 %val) {
+; CHECK-LABEL: seq_cst_i16_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1035[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1035;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_i16_local_gpu_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i16 %val, ptr addrspace(5) %slot syncscope("device") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_i16_local_sys_volatile(i16 %val) {
+; CHECK-LABEL: seq_cst_i16_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1036[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1036;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_i16_local_sys_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i16 %val, ptr addrspace(5) %slot syncscope("") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_i16_local_cta_volatile(i16 %val) {
+; CHECK-LABEL: seq_cst_i16_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1037[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1037;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_i16_local_cta_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i16 %val, ptr addrspace(5) %slot syncscope("block") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_i16_local_cluster_volatile(i16 %val) {
+; CHECK-LABEL: seq_cst_i16_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1038[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1038;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_i16_local_cluster_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i16 %val, ptr addrspace(5) %slot syncscope("cluster") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_i16_local_gpu_volatile(i16 %val) {
+; CHECK-LABEL: seq_cst_i16_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1039[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1039;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_i16_local_gpu_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i16 %val, ptr addrspace(5) %slot syncscope("device") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_i32_local_sys(i32 %val) {
+; CHECK-LABEL: seq_cst_i32_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot1040[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1040;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [seq_cst_i32_local_sys_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i32 %val, ptr addrspace(5) %slot syncscope("") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_i32_local_cta(i32 %val) {
+; CHECK-LABEL: seq_cst_i32_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot1041[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1041;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [seq_cst_i32_local_cta_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i32 %val, ptr addrspace(5) %slot syncscope("block") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_i32_local_cluster(i32 %val) {
+; CHECK-LABEL: seq_cst_i32_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot1042[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1042;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [seq_cst_i32_local_cluster_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i32 %val, ptr addrspace(5) %slot syncscope("cluster") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_i32_local_gpu(i32 %val) {
+; CHECK-LABEL: seq_cst_i32_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot1043[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1043;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [seq_cst_i32_local_gpu_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i32 %val, ptr addrspace(5) %slot syncscope("device") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_i32_local_sys_volatile(i32 %val) {
+; CHECK-LABEL: seq_cst_i32_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot1044[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1044;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [seq_cst_i32_local_sys_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i32 %val, ptr addrspace(5) %slot syncscope("") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_i32_local_cta_volatile(i32 %val) {
+; CHECK-LABEL: seq_cst_i32_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot1045[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1045;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [seq_cst_i32_local_cta_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i32 %val, ptr addrspace(5) %slot syncscope("block") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_i32_local_cluster_volatile(i32 %val) {
+; CHECK-LABEL: seq_cst_i32_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot1046[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1046;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [seq_cst_i32_local_cluster_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i32 %val, ptr addrspace(5) %slot syncscope("cluster") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_i32_local_gpu_volatile(i32 %val) {
+; CHECK-LABEL: seq_cst_i32_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot1047[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1047;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [seq_cst_i32_local_gpu_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i32 %val, ptr addrspace(5) %slot syncscope("device") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_i64_local_sys(i64 %val) {
+; CHECK-LABEL: seq_cst_i64_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot1048[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1048;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_local_sys_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i64 %val, ptr addrspace(5) %slot syncscope("") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_i64_local_cta(i64 %val) {
+; CHECK-LABEL: seq_cst_i64_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot1049[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1049;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_local_cta_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i64 %val, ptr addrspace(5) %slot syncscope("block") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_i64_local_cluster(i64 %val) {
+; CHECK-LABEL: seq_cst_i64_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot1050[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1050;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_local_cluster_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i64 %val, ptr addrspace(5) %slot syncscope("cluster") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_i64_local_gpu(i64 %val) {
+; CHECK-LABEL: seq_cst_i64_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot1051[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1051;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_local_gpu_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i64 %val, ptr addrspace(5) %slot syncscope("device") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_i64_local_sys_volatile(i64 %val) {
+; CHECK-LABEL: seq_cst_i64_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot1052[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1052;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_local_sys_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i64 %val, ptr addrspace(5) %slot syncscope("") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_i64_local_cta_volatile(i64 %val) {
+; CHECK-LABEL: seq_cst_i64_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot1053[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1053;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_local_cta_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i64 %val, ptr addrspace(5) %slot syncscope("block") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_i64_local_cluster_volatile(i64 %val) {
+; CHECK-LABEL: seq_cst_i64_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot1054[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1054;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_local_cluster_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i64 %val, ptr addrspace(5) %slot syncscope("cluster") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_i64_local_gpu_volatile(i64 %val) {
+; CHECK-LABEL: seq_cst_i64_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot1055[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1055;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_local_gpu_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i64 %val, ptr addrspace(5) %slot syncscope("device") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_half_local_sys(half %val) {
+; CHECK-LABEL: seq_cst_half_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1056[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1056;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_half_local_sys_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic half %val, ptr addrspace(5) %slot syncscope("") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_half_local_cta(half %val) {
+; CHECK-LABEL: seq_cst_half_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1057[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1057;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_half_local_cta_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic half %val, ptr addrspace(5) %slot syncscope("block") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_half_local_cluster(half %val) {
+; CHECK-LABEL: seq_cst_half_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1058[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1058;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_half_local_cluster_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic half %val, ptr addrspace(5) %slot syncscope("cluster") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_half_local_gpu(half %val) {
+; CHECK-LABEL: seq_cst_half_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1059[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1059;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_half_local_gpu_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic half %val, ptr addrspace(5) %slot syncscope("device") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_half_local_sys_volatile(half %val) {
+; CHECK-LABEL: seq_cst_half_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1060[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1060;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_half_local_sys_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile half %val, ptr addrspace(5) %slot syncscope("") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_half_local_cta_volatile(half %val) {
+; CHECK-LABEL: seq_cst_half_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1061[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1061;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_half_local_cta_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile half %val, ptr addrspace(5) %slot syncscope("block") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_half_local_cluster_volatile(half %val) {
+; CHECK-LABEL: seq_cst_half_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1062[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1062;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_half_local_cluster_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile half %val, ptr addrspace(5) %slot syncscope("cluster") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_half_local_gpu_volatile(half %val) {
+; CHECK-LABEL: seq_cst_half_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1063[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1063;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_half_local_gpu_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile half %val, ptr addrspace(5) %slot syncscope("device") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_bfloat_local_sys(bfloat %val) {
+; CHECK-LABEL: seq_cst_bfloat_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1064[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1064;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_bfloat_local_sys_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic bfloat %val, ptr addrspace(5) %slot syncscope("") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_bfloat_local_cta(bfloat %val) {
+; CHECK-LABEL: seq_cst_bfloat_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1065[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1065;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_bfloat_local_cta_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic bfloat %val, ptr addrspace(5) %slot syncscope("block") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_bfloat_local_cluster(bfloat %val) {
+; CHECK-LABEL: seq_cst_bfloat_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1066[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1066;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_bfloat_local_cluster_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic bfloat %val, ptr addrspace(5) %slot syncscope("cluster") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_bfloat_local_gpu(bfloat %val) {
+; CHECK-LABEL: seq_cst_bfloat_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1067[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1067;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_bfloat_local_gpu_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic bfloat %val, ptr addrspace(5) %slot syncscope("device") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_bfloat_local_sys_volatile(bfloat %val) {
+; CHECK-LABEL: seq_cst_bfloat_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1068[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1068;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_bfloat_local_sys_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile bfloat %val, ptr addrspace(5) %slot syncscope("") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_bfloat_local_cta_volatile(bfloat %val) {
+; CHECK-LABEL: seq_cst_bfloat_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1069[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1069;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_bfloat_local_cta_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile bfloat %val, ptr addrspace(5) %slot syncscope("block") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_bfloat_local_cluster_volatile(bfloat %val) {
+; CHECK-LABEL: seq_cst_bfloat_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1070[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1070;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_bfloat_local_cluster_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile bfloat %val, ptr addrspace(5) %slot syncscope("cluster") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_bfloat_local_gpu_volatile(bfloat %val) {
+; CHECK-LABEL: seq_cst_bfloat_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1071[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1071;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_bfloat_local_gpu_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile bfloat %val, ptr addrspace(5) %slot syncscope("device") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_float_local_sys(float %val) {
+; CHECK-LABEL: seq_cst_float_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot1072[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1072;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [seq_cst_float_local_sys_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic float %val, ptr addrspace(5) %slot syncscope("") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_float_local_cta(float %val) {
+; CHECK-LABEL: seq_cst_float_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot1073[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1073;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [seq_cst_float_local_cta_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic float %val, ptr addrspace(5) %slot syncscope("block") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_float_local_cluster(float %val) {
+; CHECK-LABEL: seq_cst_float_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot1074[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1074;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [seq_cst_float_local_cluster_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic float %val, ptr addrspace(5) %slot syncscope("cluster") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_float_local_gpu(float %val) {
+; CHECK-LABEL: seq_cst_float_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot1075[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1075;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [seq_cst_float_local_gpu_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic float %val, ptr addrspace(5) %slot syncscope("device") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_float_local_sys_volatile(float %val) {
+; CHECK-LABEL: seq_cst_float_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot1076[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1076;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [seq_cst_float_local_sys_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile float %val, ptr addrspace(5) %slot syncscope("") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_float_local_cta_volatile(float %val) {
+; CHECK-LABEL: seq_cst_float_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot1077[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1077;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [seq_cst_float_local_cta_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile float %val, ptr addrspace(5) %slot syncscope("block") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_float_local_cluster_volatile(float %val) {
+; CHECK-LABEL: seq_cst_float_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot1078[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1078;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [seq_cst_float_local_cluster_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile float %val, ptr addrspace(5) %slot syncscope("cluster") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_float_local_gpu_volatile(float %val) {
+; CHECK-LABEL: seq_cst_float_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot1079[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1079;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [seq_cst_float_local_gpu_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile float %val, ptr addrspace(5) %slot syncscope("device") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_double_local_sys(double %val) {
+; CHECK-LABEL: seq_cst_double_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot1080[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1080;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_local_sys_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic double %val, ptr addrspace(5) %slot syncscope("") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_double_local_cta(double %val) {
+; CHECK-LABEL: seq_cst_double_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot1081[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1081;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_local_cta_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic double %val, ptr addrspace(5) %slot syncscope("block") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_double_local_cluster(double %val) {
+; CHECK-LABEL: seq_cst_double_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot1082[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1082;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_local_cluster_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic double %val, ptr addrspace(5) %slot syncscope("cluster") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_double_local_gpu(double %val) {
+; CHECK-LABEL: seq_cst_double_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot1083[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1083;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_local_gpu_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic double %val, ptr addrspace(5) %slot syncscope("device") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_double_local_sys_volatile(double %val) {
+; CHECK-LABEL: seq_cst_double_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot1084[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1084;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_local_sys_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile double %val, ptr addrspace(5) %slot syncscope("") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_double_local_cta_volatile(double %val) {
+; CHECK-LABEL: seq_cst_double_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot1085[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1085;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_local_cta_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile double %val, ptr addrspace(5) %slot syncscope("block") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_double_local_cluster_volatile(double %val) {
+; CHECK-LABEL: seq_cst_double_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot1086[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1086;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_local_cluster_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile double %val, ptr addrspace(5) %slot syncscope("cluster") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_double_local_gpu_volatile(double %val) {
+; CHECK-LABEL: seq_cst_double_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot1087[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1087;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_local_gpu_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile double %val, ptr addrspace(5) %slot syncscope("device") seq_cst, align 8
+ ret void
+}
+
diff --git a/llvm/test/CodeGen/NVPTX/store-sm90.ll b/llvm/test/CodeGen/NVPTX/store-sm90.ll
new file mode 100644
index 0000000000000..4e5123448f35a
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/store-sm90.ll
@@ -0,0 +1,21950 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_90 -mattr=+ptx87 | FileCheck %s
+; RUN: %if ptxas-sm_90 && ptxas-isa-8.7 %{ llc < %s -march=nvptx64 -mcpu=sm_90 -mattr=+ptx87 | %ptxas-verify -arch=sm_90 %}
+
+define void @notatomic_i8_generic(i8 %val, ptr %addr) {
+; CHECK-LABEL: notatomic_i8_generic(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [notatomic_i8_generic_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i8_generic_param_1];
+; CHECK-NEXT: st.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store i8 %val, ptr %addr
+ ret void
+}
+
+define void @notatomic_i8_generic_volatile(i8 %val, ptr %addr) {
+; CHECK-LABEL: notatomic_i8_generic_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [notatomic_i8_generic_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i8_generic_volatile_param_1];
+; CHECK-NEXT: st.volatile.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store volatile i8 %val, ptr %addr
+ ret void
+}
+
+define void @notatomic_i16_generic(i16 %val, ptr %addr) {
+; CHECK-LABEL: notatomic_i16_generic(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [notatomic_i16_generic_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i16_generic_param_1];
+; CHECK-NEXT: st.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store i16 %val, ptr %addr
+ ret void
+}
+
+define void @notatomic_i16_generic_volatile(i16 %val, ptr %addr) {
+; CHECK-LABEL: notatomic_i16_generic_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [notatomic_i16_generic_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i16_generic_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store volatile i16 %val, ptr %addr
+ ret void
+}
+
+define void @notatomic_i32_generic(i32 %val, ptr %addr) {
+; CHECK-LABEL: notatomic_i32_generic(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [notatomic_i32_generic_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i32_generic_param_1];
+; CHECK-NEXT: st.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store i32 %val, ptr %addr
+ ret void
+}
+
+define void @notatomic_i32_generic_volatile(i32 %val, ptr %addr) {
+; CHECK-LABEL: notatomic_i32_generic_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [notatomic_i32_generic_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i32_generic_volatile_param_1];
+; CHECK-NEXT: st.volatile.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store volatile i32 %val, ptr %addr
+ ret void
+}
+
+define void @notatomic_i64_generic(i64 %val, ptr %addr) {
+; CHECK-LABEL: notatomic_i64_generic(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i64_generic_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [notatomic_i64_generic_param_1];
+; CHECK-NEXT: st.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store i64 %val, ptr %addr
+ ret void
+}
+
+define void @notatomic_i64_generic_volatile(i64 %val, ptr %addr) {
+; CHECK-LABEL: notatomic_i64_generic_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i64_generic_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [notatomic_i64_generic_volatile_param_1];
+; CHECK-NEXT: st.volatile.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store volatile i64 %val, ptr %addr
+ ret void
+}
+
+define void @notatomic_i128_generic(i128 %val, ptr %addr) {
+; CHECK-LABEL: notatomic_i128_generic(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [notatomic_i128_generic_param_0];
+; CHECK-NEXT: ld.param.b64 %rd3, [notatomic_i128_generic_param_1];
+; CHECK-NEXT: st.v2.b64 [%rd3], {%rd1, %rd2};
+; CHECK-NEXT: ret;
+ store i128 %val, ptr %addr
+ ret void
+}
+
+define void @notatomic_i128_generic_volatile(i128 %val, ptr %addr) {
+; CHECK-LABEL: notatomic_i128_generic_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [notatomic_i128_generic_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd3, [notatomic_i128_generic_volatile_param_1];
+; CHECK-NEXT: st.volatile.v2.b64 [%rd3], {%rd1, %rd2};
+; CHECK-NEXT: ret;
+ store volatile i128 %val, ptr %addr
+ ret void
+}
+
+define void @notatomic_half_generic(half %val, ptr %addr) {
+; CHECK-LABEL: notatomic_half_generic(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [notatomic_half_generic_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_half_generic_param_1];
+; CHECK-NEXT: st.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store half %val, ptr %addr
+ ret void
+}
+
+define void @notatomic_half_generic_volatile(half %val, ptr %addr) {
+; CHECK-LABEL: notatomic_half_generic_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [notatomic_half_generic_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_half_generic_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store volatile half %val, ptr %addr
+ ret void
+}
+
+define void @notatomic_bfloat_generic(bfloat %val, ptr %addr) {
+; CHECK-LABEL: notatomic_bfloat_generic(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [notatomic_bfloat_generic_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_bfloat_generic_param_1];
+; CHECK-NEXT: st.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store bfloat %val, ptr %addr
+ ret void
+}
+
+define void @notatomic_bfloat_generic_volatile(bfloat %val, ptr %addr) {
+; CHECK-LABEL: notatomic_bfloat_generic_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [notatomic_bfloat_generic_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_bfloat_generic_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store volatile bfloat %val, ptr %addr
+ ret void
+}
+
+define void @notatomic_float_generic(float %val, ptr %addr) {
+; CHECK-LABEL: notatomic_float_generic(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [notatomic_float_generic_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_float_generic_param_1];
+; CHECK-NEXT: st.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store float %val, ptr %addr
+ ret void
+}
+
+define void @notatomic_float_generic_volatile(float %val, ptr %addr) {
+; CHECK-LABEL: notatomic_float_generic_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [notatomic_float_generic_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_float_generic_volatile_param_1];
+; CHECK-NEXT: st.volatile.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store volatile float %val, ptr %addr
+ ret void
+}
+
+define void @notatomic_double_generic(double %val, ptr %addr) {
+; CHECK-LABEL: notatomic_double_generic(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_double_generic_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [notatomic_double_generic_param_1];
+; CHECK-NEXT: st.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store double %val, ptr %addr
+ ret void
+}
+
+define void @notatomic_double_generic_volatile(double %val, ptr %addr) {
+; CHECK-LABEL: notatomic_double_generic_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_double_generic_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [notatomic_double_generic_volatile_param_1];
+; CHECK-NEXT: st.volatile.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store volatile double %val, ptr %addr
+ ret void
+}
+
+define void @notatomic_i8_global(i8 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: notatomic_i8_global(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [notatomic_i8_global_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i8_global_param_1];
+; CHECK-NEXT: st.global.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store i8 %val, ptr addrspace(1) %addr
+ ret void
+}
+
+define void @notatomic_i8_global_volatile(i8 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: notatomic_i8_global_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [notatomic_i8_global_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i8_global_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store volatile i8 %val, ptr addrspace(1) %addr
+ ret void
+}
+
+define void @notatomic_i16_global(i16 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: notatomic_i16_global(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [notatomic_i16_global_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i16_global_param_1];
+; CHECK-NEXT: st.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store i16 %val, ptr addrspace(1) %addr
+ ret void
+}
+
+define void @notatomic_i16_global_volatile(i16 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: notatomic_i16_global_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [notatomic_i16_global_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i16_global_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store volatile i16 %val, ptr addrspace(1) %addr
+ ret void
+}
+
+define void @notatomic_i32_global(i32 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: notatomic_i32_global(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [notatomic_i32_global_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i32_global_param_1];
+; CHECK-NEXT: st.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store i32 %val, ptr addrspace(1) %addr
+ ret void
+}
+
+define void @notatomic_i32_global_volatile(i32 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: notatomic_i32_global_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [notatomic_i32_global_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i32_global_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store volatile i32 %val, ptr addrspace(1) %addr
+ ret void
+}
+
+define void @notatomic_i64_global(i64 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: notatomic_i64_global(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i64_global_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [notatomic_i64_global_param_1];
+; CHECK-NEXT: st.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store i64 %val, ptr addrspace(1) %addr
+ ret void
+}
+
+define void @notatomic_i64_global_volatile(i64 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: notatomic_i64_global_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i64_global_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [notatomic_i64_global_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store volatile i64 %val, ptr addrspace(1) %addr
+ ret void
+}
+
+define void @notatomic_i128_global(i128 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: notatomic_i128_global(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [notatomic_i128_global_param_0];
+; CHECK-NEXT: ld.param.b64 %rd3, [notatomic_i128_global_param_1];
+; CHECK-NEXT: st.global.v2.b64 [%rd3], {%rd1, %rd2};
+; CHECK-NEXT: ret;
+ store i128 %val, ptr addrspace(1) %addr
+ ret void
+}
+
+define void @notatomic_i128_global_volatile(i128 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: notatomic_i128_global_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [notatomic_i128_global_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd3, [notatomic_i128_global_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.v2.b64 [%rd3], {%rd1, %rd2};
+; CHECK-NEXT: ret;
+ store volatile i128 %val, ptr addrspace(1) %addr
+ ret void
+}
+
+define void @notatomic_half_global(half %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: notatomic_half_global(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [notatomic_half_global_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_half_global_param_1];
+; CHECK-NEXT: st.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store half %val, ptr addrspace(1) %addr
+ ret void
+}
+
+define void @notatomic_half_global_volatile(half %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: notatomic_half_global_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [notatomic_half_global_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_half_global_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store volatile half %val, ptr addrspace(1) %addr
+ ret void
+}
+
+define void @notatomic_bfloat_global(bfloat %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: notatomic_bfloat_global(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [notatomic_bfloat_global_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_bfloat_global_param_1];
+; CHECK-NEXT: st.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store bfloat %val, ptr addrspace(1) %addr
+ ret void
+}
+
+define void @notatomic_bfloat_global_volatile(bfloat %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: notatomic_bfloat_global_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [notatomic_bfloat_global_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_bfloat_global_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store volatile bfloat %val, ptr addrspace(1) %addr
+ ret void
+}
+
+define void @notatomic_float_global(float %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: notatomic_float_global(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [notatomic_float_global_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_float_global_param_1];
+; CHECK-NEXT: st.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store float %val, ptr addrspace(1) %addr
+ ret void
+}
+
+define void @notatomic_float_global_volatile(float %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: notatomic_float_global_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [notatomic_float_global_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_float_global_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store volatile float %val, ptr addrspace(1) %addr
+ ret void
+}
+
+define void @notatomic_double_global(double %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: notatomic_double_global(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_double_global_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [notatomic_double_global_param_1];
+; CHECK-NEXT: st.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store double %val, ptr addrspace(1) %addr
+ ret void
+}
+
+define void @notatomic_double_global_volatile(double %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: notatomic_double_global_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_double_global_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [notatomic_double_global_volatile_param_1];
+; CHECK-NEXT: st.volatile.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store volatile double %val, ptr addrspace(1) %addr
+ ret void
+}
+
+define void @notatomic_i8_shared(i8 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: notatomic_i8_shared(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [notatomic_i8_shared_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i8_shared_param_1];
+; CHECK-NEXT: st.shared.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store i8 %val, ptr addrspace(3) %addr
+ ret void
+}
+
+define void @notatomic_i8_shared_volatile(i8 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: notatomic_i8_shared_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [notatomic_i8_shared_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i8_shared_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store volatile i8 %val, ptr addrspace(3) %addr
+ ret void
+}
+
+define void @notatomic_i16_shared(i16 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: notatomic_i16_shared(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [notatomic_i16_shared_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i16_shared_param_1];
+; CHECK-NEXT: st.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store i16 %val, ptr addrspace(3) %addr
+ ret void
+}
+
+define void @notatomic_i16_shared_volatile(i16 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: notatomic_i16_shared_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [notatomic_i16_shared_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i16_shared_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store volatile i16 %val, ptr addrspace(3) %addr
+ ret void
+}
+
+define void @notatomic_i32_shared(i32 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: notatomic_i32_shared(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [notatomic_i32_shared_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i32_shared_param_1];
+; CHECK-NEXT: st.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store i32 %val, ptr addrspace(3) %addr
+ ret void
+}
+
+define void @notatomic_i32_shared_volatile(i32 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: notatomic_i32_shared_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [notatomic_i32_shared_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i32_shared_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store volatile i32 %val, ptr addrspace(3) %addr
+ ret void
+}
+
+define void @notatomic_i64_shared(i64 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: notatomic_i64_shared(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i64_shared_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [notatomic_i64_shared_param_1];
+; CHECK-NEXT: st.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store i64 %val, ptr addrspace(3) %addr
+ ret void
+}
+
+define void @notatomic_i64_shared_volatile(i64 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: notatomic_i64_shared_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i64_shared_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [notatomic_i64_shared_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store volatile i64 %val, ptr addrspace(3) %addr
+ ret void
+}
+
+define void @notatomic_i128_shared(i128 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: notatomic_i128_shared(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [notatomic_i128_shared_param_0];
+; CHECK-NEXT: ld.param.b64 %rd3, [notatomic_i128_shared_param_1];
+; CHECK-NEXT: st.shared.v2.b64 [%rd3], {%rd1, %rd2};
+; CHECK-NEXT: ret;
+ store i128 %val, ptr addrspace(3) %addr
+ ret void
+}
+
+define void @notatomic_i128_shared_volatile(i128 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: notatomic_i128_shared_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [notatomic_i128_shared_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd3, [notatomic_i128_shared_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.v2.b64 [%rd3], {%rd1, %rd2};
+; CHECK-NEXT: ret;
+ store volatile i128 %val, ptr addrspace(3) %addr
+ ret void
+}
+
+define void @notatomic_half_shared(half %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: notatomic_half_shared(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [notatomic_half_shared_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_half_shared_param_1];
+; CHECK-NEXT: st.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store half %val, ptr addrspace(3) %addr
+ ret void
+}
+
+define void @notatomic_half_shared_volatile(half %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: notatomic_half_shared_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [notatomic_half_shared_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_half_shared_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store volatile half %val, ptr addrspace(3) %addr
+ ret void
+}
+
+define void @notatomic_bfloat_shared(bfloat %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: notatomic_bfloat_shared(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [notatomic_bfloat_shared_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_bfloat_shared_param_1];
+; CHECK-NEXT: st.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store bfloat %val, ptr addrspace(3) %addr
+ ret void
+}
+
+define void @notatomic_bfloat_shared_volatile(bfloat %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: notatomic_bfloat_shared_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [notatomic_bfloat_shared_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_bfloat_shared_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store volatile bfloat %val, ptr addrspace(3) %addr
+ ret void
+}
+
+define void @notatomic_float_shared(float %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: notatomic_float_shared(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [notatomic_float_shared_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_float_shared_param_1];
+; CHECK-NEXT: st.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store float %val, ptr addrspace(3) %addr
+ ret void
+}
+
+define void @notatomic_float_shared_volatile(float %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: notatomic_float_shared_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [notatomic_float_shared_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_float_shared_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store volatile float %val, ptr addrspace(3) %addr
+ ret void
+}
+
+define void @notatomic_double_shared(double %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: notatomic_double_shared(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_double_shared_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [notatomic_double_shared_param_1];
+; CHECK-NEXT: st.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store double %val, ptr addrspace(3) %addr
+ ret void
+}
+
+define void @notatomic_double_shared_volatile(double %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: notatomic_double_shared_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_double_shared_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [notatomic_double_shared_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store volatile double %val, ptr addrspace(3) %addr
+ ret void
+}
+
+define void @notatomic_i8_local(i8 %val) {
+; CHECK-LABEL: notatomic_i8_local(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot54[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot54;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b8 %rs1, [notatomic_i8_local_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b8 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store i8 %val, ptr addrspace(5) %slot
+ ret void
+}
+
+define void @notatomic_i8_local_volatile(i8 %val) {
+; CHECK-LABEL: notatomic_i8_local_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot55[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot55;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b8 %rs1, [notatomic_i8_local_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b8 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store volatile i8 %val, ptr addrspace(5) %slot
+ ret void
+}
+
+define void @notatomic_i16_local(i16 %val) {
+; CHECK-LABEL: notatomic_i16_local(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot56[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot56;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [notatomic_i16_local_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store i16 %val, ptr addrspace(5) %slot
+ ret void
+}
+
+define void @notatomic_i16_local_volatile(i16 %val) {
+; CHECK-LABEL: notatomic_i16_local_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot57[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot57;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [notatomic_i16_local_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store volatile i16 %val, ptr addrspace(5) %slot
+ ret void
+}
+
+define void @notatomic_i32_local(i32 %val) {
+; CHECK-LABEL: notatomic_i32_local(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot58[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot58;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [notatomic_i32_local_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store i32 %val, ptr addrspace(5) %slot
+ ret void
+}
+
+define void @notatomic_i32_local_volatile(i32 %val) {
+; CHECK-LABEL: notatomic_i32_local_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot59[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot59;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [notatomic_i32_local_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store volatile i32 %val, ptr addrspace(5) %slot
+ ret void
+}
+
+define void @notatomic_i64_local(i64 %val) {
+; CHECK-LABEL: notatomic_i64_local(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot60[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot60;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i64_local_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store i64 %val, ptr addrspace(5) %slot
+ ret void
+}
+
+define void @notatomic_i64_local_volatile(i64 %val) {
+; CHECK-LABEL: notatomic_i64_local_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot61[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot61;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_i64_local_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store volatile i64 %val, ptr addrspace(5) %slot
+ ret void
+}
+
+define void @notatomic_i128_local(i128 %val) {
+; CHECK-LABEL: notatomic_i128_local(
+; CHECK: {
+; CHECK-NEXT: .local .align 16 .b8 __local_depot62[16];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot62;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [notatomic_i128_local_param_0];
+; CHECK-NEXT: add.u64 %rd3, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd3;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.v2.b64 [%SP], {%rd1, %rd2};
+; CHECK-NEXT: ret;
+ %slot = alloca i128, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store i128 %val, ptr addrspace(5) %slot
+ ret void
+}
+
+define void @notatomic_i128_local_volatile(i128 %val) {
+; CHECK-LABEL: notatomic_i128_local_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 16 .b8 __local_depot63[16];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot63;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [notatomic_i128_local_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd3, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd3;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.v2.b64 [%SP], {%rd1, %rd2};
+; CHECK-NEXT: ret;
+ %slot = alloca i128, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store volatile i128 %val, ptr addrspace(5) %slot
+ ret void
+}
+
+define void @notatomic_half_local(half %val) {
+; CHECK-LABEL: notatomic_half_local(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot64[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot64;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [notatomic_half_local_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store half %val, ptr addrspace(5) %slot
+ ret void
+}
+
+define void @notatomic_half_local_volatile(half %val) {
+; CHECK-LABEL: notatomic_half_local_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot65[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot65;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [notatomic_half_local_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store volatile half %val, ptr addrspace(5) %slot
+ ret void
+}
+
+define void @notatomic_bfloat_local(bfloat %val) {
+; CHECK-LABEL: notatomic_bfloat_local(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot66[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot66;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [notatomic_bfloat_local_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store bfloat %val, ptr addrspace(5) %slot
+ ret void
+}
+
+define void @notatomic_bfloat_local_volatile(bfloat %val) {
+; CHECK-LABEL: notatomic_bfloat_local_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot67[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot67;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [notatomic_bfloat_local_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store volatile bfloat %val, ptr addrspace(5) %slot
+ ret void
+}
+
+define void @notatomic_float_local(float %val) {
+; CHECK-LABEL: notatomic_float_local(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot68[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot68;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [notatomic_float_local_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store float %val, ptr addrspace(5) %slot
+ ret void
+}
+
+define void @notatomic_float_local_volatile(float %val) {
+; CHECK-LABEL: notatomic_float_local_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot69[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot69;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [notatomic_float_local_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store volatile float %val, ptr addrspace(5) %slot
+ ret void
+}
+
+define void @notatomic_double_local(double %val) {
+; CHECK-LABEL: notatomic_double_local(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot70[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot70;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_double_local_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca double, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store double %val, ptr addrspace(5) %slot
+ ret void
+}
+
+define void @notatomic_double_local_volatile(double %val) {
+; CHECK-LABEL: notatomic_double_local_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot71[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot71;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [notatomic_double_local_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca double, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store volatile double %val, ptr addrspace(5) %slot
+ ret void
+}
+
+define void @unordered_i8_generic_sys(i8 %val, ptr %addr) {
+; CHECK-LABEL: unordered_i8_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [unordered_i8_generic_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_generic_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr %addr syncscope("") unordered, align 1
+ ret void
+}
+
+define void @unordered_i8_generic_cta(i8 %val, ptr %addr) {
+; CHECK-LABEL: unordered_i8_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [unordered_i8_generic_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_generic_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr %addr syncscope("block") unordered, align 1
+ ret void
+}
+
+define void @unordered_i8_generic_cluster(i8 %val, ptr %addr) {
+; CHECK-LABEL: unordered_i8_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [unordered_i8_generic_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_generic_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cluster.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr %addr syncscope("cluster") unordered, align 1
+ ret void
+}
+
+define void @unordered_i8_generic_gpu(i8 %val, ptr %addr) {
+; CHECK-LABEL: unordered_i8_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [unordered_i8_generic_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_generic_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr %addr syncscope("device") unordered, align 1
+ ret void
+}
+
+define void @unordered_i8_generic_sys_volatile(i8 %val, ptr %addr) {
+; CHECK-LABEL: unordered_i8_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [unordered_i8_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_generic_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr %addr syncscope("") unordered, align 1
+ ret void
+}
+
+define void @unordered_i8_generic_cta_volatile(i8 %val, ptr %addr) {
+; CHECK-LABEL: unordered_i8_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [unordered_i8_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_generic_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr %addr syncscope("block") unordered, align 1
+ ret void
+}
+
+define void @unordered_i8_generic_cluster_volatile(i8 %val, ptr %addr) {
+; CHECK-LABEL: unordered_i8_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [unordered_i8_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_generic_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr %addr syncscope("cluster") unordered, align 1
+ ret void
+}
+
+define void @unordered_i8_generic_gpu_volatile(i8 %val, ptr %addr) {
+; CHECK-LABEL: unordered_i8_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [unordered_i8_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_generic_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr %addr syncscope("device") unordered, align 1
+ ret void
+}
+
+define void @unordered_i16_generic_sys(i16 %val, ptr %addr) {
+; CHECK-LABEL: unordered_i16_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_i16_generic_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_generic_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr %addr syncscope("") unordered, align 2
+ ret void
+}
+
+define void @unordered_i16_generic_cta(i16 %val, ptr %addr) {
+; CHECK-LABEL: unordered_i16_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_i16_generic_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_generic_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr %addr syncscope("block") unordered, align 2
+ ret void
+}
+
+define void @unordered_i16_generic_cluster(i16 %val, ptr %addr) {
+; CHECK-LABEL: unordered_i16_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_i16_generic_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_generic_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cluster.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr %addr syncscope("cluster") unordered, align 2
+ ret void
+}
+
+define void @unordered_i16_generic_gpu(i16 %val, ptr %addr) {
+; CHECK-LABEL: unordered_i16_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_i16_generic_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_generic_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr %addr syncscope("device") unordered, align 2
+ ret void
+}
+
+define void @unordered_i16_generic_sys_volatile(i16 %val, ptr %addr) {
+; CHECK-LABEL: unordered_i16_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_i16_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_generic_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr %addr syncscope("") unordered, align 2
+ ret void
+}
+
+define void @unordered_i16_generic_cta_volatile(i16 %val, ptr %addr) {
+; CHECK-LABEL: unordered_i16_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_i16_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_generic_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr %addr syncscope("block") unordered, align 2
+ ret void
+}
+
+define void @unordered_i16_generic_cluster_volatile(i16 %val, ptr %addr) {
+; CHECK-LABEL: unordered_i16_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_i16_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_generic_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr %addr syncscope("cluster") unordered, align 2
+ ret void
+}
+
+define void @unordered_i16_generic_gpu_volatile(i16 %val, ptr %addr) {
+; CHECK-LABEL: unordered_i16_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_i16_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_generic_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr %addr syncscope("device") unordered, align 2
+ ret void
+}
+
+define void @unordered_i32_generic_sys(i32 %val, ptr %addr) {
+; CHECK-LABEL: unordered_i32_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_i32_generic_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_generic_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr %addr syncscope("") unordered, align 4
+ ret void
+}
+
+define void @unordered_i32_generic_cta(i32 %val, ptr %addr) {
+; CHECK-LABEL: unordered_i32_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_i32_generic_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_generic_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr %addr syncscope("block") unordered, align 4
+ ret void
+}
+
+define void @unordered_i32_generic_cluster(i32 %val, ptr %addr) {
+; CHECK-LABEL: unordered_i32_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_i32_generic_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_generic_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cluster.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr %addr syncscope("cluster") unordered, align 4
+ ret void
+}
+
+define void @unordered_i32_generic_gpu(i32 %val, ptr %addr) {
+; CHECK-LABEL: unordered_i32_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_i32_generic_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_generic_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr %addr syncscope("device") unordered, align 4
+ ret void
+}
+
+define void @unordered_i32_generic_sys_volatile(i32 %val, ptr %addr) {
+; CHECK-LABEL: unordered_i32_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_i32_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_generic_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr %addr syncscope("") unordered, align 4
+ ret void
+}
+
+define void @unordered_i32_generic_cta_volatile(i32 %val, ptr %addr) {
+; CHECK-LABEL: unordered_i32_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_i32_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_generic_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr %addr syncscope("block") unordered, align 4
+ ret void
+}
+
+define void @unordered_i32_generic_cluster_volatile(i32 %val, ptr %addr) {
+; CHECK-LABEL: unordered_i32_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_i32_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_generic_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr %addr syncscope("cluster") unordered, align 4
+ ret void
+}
+
+define void @unordered_i32_generic_gpu_volatile(i32 %val, ptr %addr) {
+; CHECK-LABEL: unordered_i32_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_i32_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_generic_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr %addr syncscope("device") unordered, align 4
+ ret void
+}
+
+define void @unordered_i64_generic_sys(i64 %val, ptr %addr) {
+; CHECK-LABEL: unordered_i64_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_generic_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_i64_generic_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr %addr syncscope("") unordered, align 8
+ ret void
+}
+
+define void @unordered_i64_generic_cta(i64 %val, ptr %addr) {
+; CHECK-LABEL: unordered_i64_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_generic_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_i64_generic_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr %addr syncscope("block") unordered, align 8
+ ret void
+}
+
+define void @unordered_i64_generic_cluster(i64 %val, ptr %addr) {
+; CHECK-LABEL: unordered_i64_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_generic_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_i64_generic_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cluster.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr %addr syncscope("cluster") unordered, align 8
+ ret void
+}
+
+define void @unordered_i64_generic_gpu(i64 %val, ptr %addr) {
+; CHECK-LABEL: unordered_i64_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_generic_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_i64_generic_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr %addr syncscope("device") unordered, align 8
+ ret void
+}
+
+define void @unordered_i64_generic_sys_volatile(i64 %val, ptr %addr) {
+; CHECK-LABEL: unordered_i64_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_i64_generic_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr %addr syncscope("") unordered, align 8
+ ret void
+}
+
+define void @unordered_i64_generic_cta_volatile(i64 %val, ptr %addr) {
+; CHECK-LABEL: unordered_i64_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_i64_generic_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr %addr syncscope("block") unordered, align 8
+ ret void
+}
+
+define void @unordered_i64_generic_cluster_volatile(i64 %val, ptr %addr) {
+; CHECK-LABEL: unordered_i64_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_i64_generic_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr %addr syncscope("cluster") unordered, align 8
+ ret void
+}
+
+define void @unordered_i64_generic_gpu_volatile(i64 %val, ptr %addr) {
+; CHECK-LABEL: unordered_i64_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_i64_generic_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr %addr syncscope("device") unordered, align 8
+ ret void
+}
+
+define void @unordered_i128_generic_sys(i128 %val, ptr %addr) {
+; CHECK-LABEL: unordered_i128_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [unordered_i128_generic_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd3, [unordered_i128_generic_sys_param_1];
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 amt, dst;
+; CHECK-NEXT: mov.b128 amt, {%rd1, %rd2};
+; CHECK-NEXT: atom.relaxed.sys.exch.b128 dst, [%rd3], amt;
+; CHECK-NEXT: mov.b128 {%rd4, %rd5}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: ret;
+ store atomic i128 %val, ptr %addr syncscope("") unordered, align 16
+ ret void
+}
+
+define void @unordered_i128_generic_cta(i128 %val, ptr %addr) {
+; CHECK-LABEL: unordered_i128_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [unordered_i128_generic_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd3, [unordered_i128_generic_cta_param_1];
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 amt, dst;
+; CHECK-NEXT: mov.b128 amt, {%rd1, %rd2};
+; CHECK-NEXT: atom.relaxed.cta.exch.b128 dst, [%rd3], amt;
+; CHECK-NEXT: mov.b128 {%rd4, %rd5}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: ret;
+ store atomic i128 %val, ptr %addr syncscope("block") unordered, align 16
+ ret void
+}
+
+define void @unordered_i128_generic_cluster(i128 %val, ptr %addr) {
+; CHECK-LABEL: unordered_i128_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [unordered_i128_generic_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd3, [unordered_i128_generic_cluster_param_1];
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 amt, dst;
+; CHECK-NEXT: mov.b128 amt, {%rd1, %rd2};
+; CHECK-NEXT: atom.relaxed.cluster.exch.b128 dst, [%rd3], amt;
+; CHECK-NEXT: mov.b128 {%rd4, %rd5}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: ret;
+ store atomic i128 %val, ptr %addr syncscope("cluster") unordered, align 16
+ ret void
+}
+
+define void @unordered_i128_generic_gpu(i128 %val, ptr %addr) {
+; CHECK-LABEL: unordered_i128_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [unordered_i128_generic_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd3, [unordered_i128_generic_gpu_param_1];
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 amt, dst;
+; CHECK-NEXT: mov.b128 amt, {%rd1, %rd2};
+; CHECK-NEXT: atom.relaxed.gpu.exch.b128 dst, [%rd3], amt;
+; CHECK-NEXT: mov.b128 {%rd4, %rd5}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: ret;
+ store atomic i128 %val, ptr %addr syncscope("device") unordered, align 16
+ ret void
+}
+
+define void @unordered_i128_generic_sys_volatile(i128 %val, ptr %addr) {
+; CHECK-LABEL: unordered_i128_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [unordered_i128_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd3, [unordered_i128_generic_sys_volatile_param_1];
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 amt, dst;
+; CHECK-NEXT: mov.b128 amt, {%rd1, %rd2};
+; CHECK-NEXT: atom.relaxed.sys.exch.b128 dst, [%rd3], amt;
+; CHECK-NEXT: mov.b128 {%rd4, %rd5}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: ret;
+ store atomic volatile i128 %val, ptr %addr syncscope("") unordered, align 16
+ ret void
+}
+
+define void @unordered_i128_generic_cta_volatile(i128 %val, ptr %addr) {
+; CHECK-LABEL: unordered_i128_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [unordered_i128_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd3, [unordered_i128_generic_cta_volatile_param_1];
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 amt, dst;
+; CHECK-NEXT: mov.b128 amt, {%rd1, %rd2};
+; CHECK-NEXT: atom.relaxed.cta.exch.b128 dst, [%rd3], amt;
+; CHECK-NEXT: mov.b128 {%rd4, %rd5}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: ret;
+ store atomic volatile i128 %val, ptr %addr syncscope("block") unordered, align 16
+ ret void
+}
+
+define void @unordered_i128_generic_cluster_volatile(i128 %val, ptr %addr) {
+; CHECK-LABEL: unordered_i128_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [unordered_i128_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd3, [unordered_i128_generic_cluster_volatile_param_1];
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 amt, dst;
+; CHECK-NEXT: mov.b128 amt, {%rd1, %rd2};
+; CHECK-NEXT: atom.relaxed.cluster.exch.b128 dst, [%rd3], amt;
+; CHECK-NEXT: mov.b128 {%rd4, %rd5}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: ret;
+ store atomic volatile i128 %val, ptr %addr syncscope("cluster") unordered, align 16
+ ret void
+}
+
+define void @unordered_i128_generic_gpu_volatile(i128 %val, ptr %addr) {
+; CHECK-LABEL: unordered_i128_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [unordered_i128_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd3, [unordered_i128_generic_gpu_volatile_param_1];
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 amt, dst;
+; CHECK-NEXT: mov.b128 amt, {%rd1, %rd2};
+; CHECK-NEXT: atom.relaxed.gpu.exch.b128 dst, [%rd3], amt;
+; CHECK-NEXT: mov.b128 {%rd4, %rd5}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: ret;
+ store atomic volatile i128 %val, ptr %addr syncscope("device") unordered, align 16
+ ret void
+}
+
+define void @unordered_half_generic_sys(half %val, ptr %addr) {
+; CHECK-LABEL: unordered_half_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_half_generic_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_generic_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr %addr syncscope("") unordered, align 2
+ ret void
+}
+
+define void @unordered_half_generic_cta(half %val, ptr %addr) {
+; CHECK-LABEL: unordered_half_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_half_generic_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_generic_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr %addr syncscope("block") unordered, align 2
+ ret void
+}
+
+define void @unordered_half_generic_cluster(half %val, ptr %addr) {
+; CHECK-LABEL: unordered_half_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_half_generic_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_generic_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cluster.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr %addr syncscope("cluster") unordered, align 2
+ ret void
+}
+
+define void @unordered_half_generic_gpu(half %val, ptr %addr) {
+; CHECK-LABEL: unordered_half_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_half_generic_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_generic_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr %addr syncscope("device") unordered, align 2
+ ret void
+}
+
+define void @unordered_half_generic_sys_volatile(half %val, ptr %addr) {
+; CHECK-LABEL: unordered_half_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_half_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_generic_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr %addr syncscope("") unordered, align 2
+ ret void
+}
+
+define void @unordered_half_generic_cta_volatile(half %val, ptr %addr) {
+; CHECK-LABEL: unordered_half_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_half_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_generic_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr %addr syncscope("block") unordered, align 2
+ ret void
+}
+
+define void @unordered_half_generic_cluster_volatile(half %val, ptr %addr) {
+; CHECK-LABEL: unordered_half_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_half_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_generic_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr %addr syncscope("cluster") unordered, align 2
+ ret void
+}
+
+define void @unordered_half_generic_gpu_volatile(half %val, ptr %addr) {
+; CHECK-LABEL: unordered_half_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_half_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_generic_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr %addr syncscope("device") unordered, align 2
+ ret void
+}
+
+define void @unordered_bfloat_generic_sys(bfloat %val, ptr %addr) {
+; CHECK-LABEL: unordered_bfloat_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_bfloat_generic_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_generic_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr %addr syncscope("") unordered, align 2
+ ret void
+}
+
+define void @unordered_bfloat_generic_cta(bfloat %val, ptr %addr) {
+; CHECK-LABEL: unordered_bfloat_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_bfloat_generic_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_generic_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr %addr syncscope("block") unordered, align 2
+ ret void
+}
+
+define void @unordered_bfloat_generic_cluster(bfloat %val, ptr %addr) {
+; CHECK-LABEL: unordered_bfloat_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_bfloat_generic_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_generic_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cluster.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr %addr syncscope("cluster") unordered, align 2
+ ret void
+}
+
+define void @unordered_bfloat_generic_gpu(bfloat %val, ptr %addr) {
+; CHECK-LABEL: unordered_bfloat_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_bfloat_generic_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_generic_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr %addr syncscope("device") unordered, align 2
+ ret void
+}
+
+define void @unordered_bfloat_generic_sys_volatile(bfloat %val, ptr %addr) {
+; CHECK-LABEL: unordered_bfloat_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_bfloat_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_generic_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr %addr syncscope("") unordered, align 2
+ ret void
+}
+
+define void @unordered_bfloat_generic_cta_volatile(bfloat %val, ptr %addr) {
+; CHECK-LABEL: unordered_bfloat_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_bfloat_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_generic_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr %addr syncscope("block") unordered, align 2
+ ret void
+}
+
+define void @unordered_bfloat_generic_cluster_volatile(bfloat %val, ptr %addr) {
+; CHECK-LABEL: unordered_bfloat_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_bfloat_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_generic_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr %addr syncscope("cluster") unordered, align 2
+ ret void
+}
+
+define void @unordered_bfloat_generic_gpu_volatile(bfloat %val, ptr %addr) {
+; CHECK-LABEL: unordered_bfloat_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_bfloat_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_generic_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr %addr syncscope("device") unordered, align 2
+ ret void
+}
+
+define void @unordered_float_generic_sys(float %val, ptr %addr) {
+; CHECK-LABEL: unordered_float_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_float_generic_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_generic_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr %addr syncscope("") unordered, align 4
+ ret void
+}
+
+define void @unordered_float_generic_cta(float %val, ptr %addr) {
+; CHECK-LABEL: unordered_float_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_float_generic_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_generic_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr %addr syncscope("block") unordered, align 4
+ ret void
+}
+
+define void @unordered_float_generic_cluster(float %val, ptr %addr) {
+; CHECK-LABEL: unordered_float_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_float_generic_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_generic_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cluster.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr %addr syncscope("cluster") unordered, align 4
+ ret void
+}
+
+define void @unordered_float_generic_gpu(float %val, ptr %addr) {
+; CHECK-LABEL: unordered_float_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_float_generic_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_generic_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr %addr syncscope("device") unordered, align 4
+ ret void
+}
+
+define void @unordered_float_generic_sys_volatile(float %val, ptr %addr) {
+; CHECK-LABEL: unordered_float_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_float_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_generic_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr %addr syncscope("") unordered, align 4
+ ret void
+}
+
+define void @unordered_float_generic_cta_volatile(float %val, ptr %addr) {
+; CHECK-LABEL: unordered_float_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_float_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_generic_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr %addr syncscope("block") unordered, align 4
+ ret void
+}
+
+define void @unordered_float_generic_cluster_volatile(float %val, ptr %addr) {
+; CHECK-LABEL: unordered_float_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_float_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_generic_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr %addr syncscope("cluster") unordered, align 4
+ ret void
+}
+
+define void @unordered_float_generic_gpu_volatile(float %val, ptr %addr) {
+; CHECK-LABEL: unordered_float_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_float_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_generic_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr %addr syncscope("device") unordered, align 4
+ ret void
+}
+
+define void @unordered_double_generic_sys(double %val, ptr %addr) {
+; CHECK-LABEL: unordered_double_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_generic_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_double_generic_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr %addr syncscope("") unordered, align 8
+ ret void
+}
+
+define void @unordered_double_generic_cta(double %val, ptr %addr) {
+; CHECK-LABEL: unordered_double_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_generic_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_double_generic_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr %addr syncscope("block") unordered, align 8
+ ret void
+}
+
+define void @unordered_double_generic_cluster(double %val, ptr %addr) {
+; CHECK-LABEL: unordered_double_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_generic_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_double_generic_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cluster.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr %addr syncscope("cluster") unordered, align 8
+ ret void
+}
+
+define void @unordered_double_generic_gpu(double %val, ptr %addr) {
+; CHECK-LABEL: unordered_double_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_generic_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_double_generic_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr %addr syncscope("device") unordered, align 8
+ ret void
+}
+
+define void @unordered_double_generic_sys_volatile(double %val, ptr %addr) {
+; CHECK-LABEL: unordered_double_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_double_generic_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr %addr syncscope("") unordered, align 8
+ ret void
+}
+
+define void @unordered_double_generic_cta_volatile(double %val, ptr %addr) {
+; CHECK-LABEL: unordered_double_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_double_generic_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr %addr syncscope("block") unordered, align 8
+ ret void
+}
+
+define void @unordered_double_generic_cluster_volatile(double %val, ptr %addr) {
+; CHECK-LABEL: unordered_double_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_double_generic_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr %addr syncscope("cluster") unordered, align 8
+ ret void
+}
+
+define void @unordered_double_generic_gpu_volatile(double %val, ptr %addr) {
+; CHECK-LABEL: unordered_double_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_double_generic_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr %addr syncscope("device") unordered, align 8
+ ret void
+}
+
+define void @unordered_i8_global_sys(i8 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i8_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [unordered_i8_global_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_global_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.global.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr addrspace(1) %addr syncscope("") unordered, align 1
+ ret void
+}
+
+define void @unordered_i8_global_cta(i8 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i8_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [unordered_i8_global_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_global_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.global.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr addrspace(1) %addr syncscope("block") unordered, align 1
+ ret void
+}
+
+define void @unordered_i8_global_cluster(i8 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i8_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [unordered_i8_global_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_global_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cluster.global.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr addrspace(1) %addr syncscope("cluster") unordered, align 1
+ ret void
+}
+
+define void @unordered_i8_global_gpu(i8 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i8_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [unordered_i8_global_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_global_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.global.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr addrspace(1) %addr syncscope("device") unordered, align 1
+ ret void
+}
+
+define void @unordered_i8_global_sys_volatile(i8 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i8_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [unordered_i8_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_global_sys_volatile_param_1];
+; CHECK-NEXT: st.mmio.relaxed.sys.global.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr addrspace(1) %addr syncscope("") unordered, align 1
+ ret void
+}
+
+define void @unordered_i8_global_cta_volatile(i8 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i8_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [unordered_i8_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_global_cta_volatile_param_1];
+; CHECK-NEXT: st.mmio.relaxed.sys.global.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr addrspace(1) %addr syncscope("block") unordered, align 1
+ ret void
+}
+
+define void @unordered_i8_global_cluster_volatile(i8 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i8_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [unordered_i8_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_global_cluster_volatile_param_1];
+; CHECK-NEXT: st.mmio.relaxed.sys.global.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr addrspace(1) %addr syncscope("cluster") unordered, align 1
+ ret void
+}
+
+define void @unordered_i8_global_gpu_volatile(i8 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i8_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [unordered_i8_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_global_gpu_volatile_param_1];
+; CHECK-NEXT: st.mmio.relaxed.sys.global.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr addrspace(1) %addr syncscope("device") unordered, align 1
+ ret void
+}
+
+define void @unordered_i16_global_sys(i16 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i16_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_i16_global_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_global_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr addrspace(1) %addr syncscope("") unordered, align 2
+ ret void
+}
+
+define void @unordered_i16_global_cta(i16 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i16_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_i16_global_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_global_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr addrspace(1) %addr syncscope("block") unordered, align 2
+ ret void
+}
+
+define void @unordered_i16_global_cluster(i16 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i16_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_i16_global_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_global_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cluster.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr addrspace(1) %addr syncscope("cluster") unordered, align 2
+ ret void
+}
+
+define void @unordered_i16_global_gpu(i16 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i16_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_i16_global_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_global_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr addrspace(1) %addr syncscope("device") unordered, align 2
+ ret void
+}
+
+define void @unordered_i16_global_sys_volatile(i16 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i16_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_i16_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_global_sys_volatile_param_1];
+; CHECK-NEXT: st.mmio.relaxed.sys.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr addrspace(1) %addr syncscope("") unordered, align 2
+ ret void
+}
+
+define void @unordered_i16_global_cta_volatile(i16 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i16_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_i16_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_global_cta_volatile_param_1];
+; CHECK-NEXT: st.mmio.relaxed.sys.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr addrspace(1) %addr syncscope("block") unordered, align 2
+ ret void
+}
+
+define void @unordered_i16_global_cluster_volatile(i16 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i16_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_i16_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_global_cluster_volatile_param_1];
+; CHECK-NEXT: st.mmio.relaxed.sys.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr addrspace(1) %addr syncscope("cluster") unordered, align 2
+ ret void
+}
+
+define void @unordered_i16_global_gpu_volatile(i16 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i16_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_i16_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_global_gpu_volatile_param_1];
+; CHECK-NEXT: st.mmio.relaxed.sys.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr addrspace(1) %addr syncscope("device") unordered, align 2
+ ret void
+}
+
+define void @unordered_i32_global_sys(i32 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i32_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_i32_global_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_global_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr addrspace(1) %addr syncscope("") unordered, align 4
+ ret void
+}
+
+define void @unordered_i32_global_cta(i32 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i32_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_i32_global_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_global_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr addrspace(1) %addr syncscope("block") unordered, align 4
+ ret void
+}
+
+define void @unordered_i32_global_cluster(i32 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i32_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_i32_global_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_global_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cluster.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr addrspace(1) %addr syncscope("cluster") unordered, align 4
+ ret void
+}
+
+define void @unordered_i32_global_gpu(i32 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i32_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_i32_global_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_global_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr addrspace(1) %addr syncscope("device") unordered, align 4
+ ret void
+}
+
+define void @unordered_i32_global_sys_volatile(i32 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i32_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_i32_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_global_sys_volatile_param_1];
+; CHECK-NEXT: st.mmio.relaxed.sys.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr addrspace(1) %addr syncscope("") unordered, align 4
+ ret void
+}
+
+define void @unordered_i32_global_cta_volatile(i32 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i32_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_i32_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_global_cta_volatile_param_1];
+; CHECK-NEXT: st.mmio.relaxed.sys.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr addrspace(1) %addr syncscope("block") unordered, align 4
+ ret void
+}
+
+define void @unordered_i32_global_cluster_volatile(i32 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i32_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_i32_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_global_cluster_volatile_param_1];
+; CHECK-NEXT: st.mmio.relaxed.sys.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr addrspace(1) %addr syncscope("cluster") unordered, align 4
+ ret void
+}
+
+define void @unordered_i32_global_gpu_volatile(i32 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i32_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_i32_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_global_gpu_volatile_param_1];
+; CHECK-NEXT: st.mmio.relaxed.sys.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr addrspace(1) %addr syncscope("device") unordered, align 4
+ ret void
+}
+
+define void @unordered_i64_global_sys(i64 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i64_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_global_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_i64_global_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr addrspace(1) %addr syncscope("") unordered, align 8
+ ret void
+}
+
+define void @unordered_i64_global_cta(i64 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i64_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_global_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_i64_global_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr addrspace(1) %addr syncscope("block") unordered, align 8
+ ret void
+}
+
+define void @unordered_i64_global_cluster(i64 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i64_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_global_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_i64_global_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cluster.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr addrspace(1) %addr syncscope("cluster") unordered, align 8
+ ret void
+}
+
+define void @unordered_i64_global_gpu(i64 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i64_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_global_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_i64_global_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr addrspace(1) %addr syncscope("device") unordered, align 8
+ ret void
+}
+
+define void @unordered_i64_global_sys_volatile(i64 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i64_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_i64_global_sys_volatile_param_1];
+; CHECK-NEXT: st.mmio.relaxed.sys.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr addrspace(1) %addr syncscope("") unordered, align 8
+ ret void
+}
+
+define void @unordered_i64_global_cta_volatile(i64 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i64_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_i64_global_cta_volatile_param_1];
+; CHECK-NEXT: st.mmio.relaxed.sys.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr addrspace(1) %addr syncscope("block") unordered, align 8
+ ret void
+}
+
+define void @unordered_i64_global_cluster_volatile(i64 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i64_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_i64_global_cluster_volatile_param_1];
+; CHECK-NEXT: st.mmio.relaxed.sys.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr addrspace(1) %addr syncscope("cluster") unordered, align 8
+ ret void
+}
+
+define void @unordered_i64_global_gpu_volatile(i64 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i64_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_i64_global_gpu_volatile_param_1];
+; CHECK-NEXT: st.mmio.relaxed.sys.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr addrspace(1) %addr syncscope("device") unordered, align 8
+ ret void
+}
+
+define void @unordered_i128_global_sys(i128 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i128_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [unordered_i128_global_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd3, [unordered_i128_global_sys_param_1];
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 amt, dst;
+; CHECK-NEXT: mov.b128 amt, {%rd1, %rd2};
+; CHECK-NEXT: atom.relaxed.sys.global.exch.b128 dst, [%rd3], amt;
+; CHECK-NEXT: mov.b128 {%rd4, %rd5}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: ret;
+ store atomic i128 %val, ptr addrspace(1) %addr syncscope("") unordered, align 16
+ ret void
+}
+
+define void @unordered_i128_global_cta(i128 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i128_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [unordered_i128_global_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd3, [unordered_i128_global_cta_param_1];
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 amt, dst;
+; CHECK-NEXT: mov.b128 amt, {%rd1, %rd2};
+; CHECK-NEXT: atom.relaxed.cta.global.exch.b128 dst, [%rd3], amt;
+; CHECK-NEXT: mov.b128 {%rd4, %rd5}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: ret;
+ store atomic i128 %val, ptr addrspace(1) %addr syncscope("block") unordered, align 16
+ ret void
+}
+
+define void @unordered_i128_global_cluster(i128 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i128_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [unordered_i128_global_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd3, [unordered_i128_global_cluster_param_1];
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 amt, dst;
+; CHECK-NEXT: mov.b128 amt, {%rd1, %rd2};
+; CHECK-NEXT: atom.relaxed.cluster.global.exch.b128 dst, [%rd3], amt;
+; CHECK-NEXT: mov.b128 {%rd4, %rd5}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: ret;
+ store atomic i128 %val, ptr addrspace(1) %addr syncscope("cluster") unordered, align 16
+ ret void
+}
+
+define void @unordered_i128_global_gpu(i128 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i128_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [unordered_i128_global_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd3, [unordered_i128_global_gpu_param_1];
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 amt, dst;
+; CHECK-NEXT: mov.b128 amt, {%rd1, %rd2};
+; CHECK-NEXT: atom.relaxed.gpu.global.exch.b128 dst, [%rd3], amt;
+; CHECK-NEXT: mov.b128 {%rd4, %rd5}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: ret;
+ store atomic i128 %val, ptr addrspace(1) %addr syncscope("device") unordered, align 16
+ ret void
+}
+
+define void @unordered_i128_global_sys_volatile(i128 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i128_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [unordered_i128_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd3, [unordered_i128_global_sys_volatile_param_1];
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 amt, dst;
+; CHECK-NEXT: mov.b128 amt, {%rd1, %rd2};
+; CHECK-NEXT: atom.relaxed.sys.global.exch.b128 dst, [%rd3], amt;
+; CHECK-NEXT: mov.b128 {%rd4, %rd5}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: ret;
+ store atomic volatile i128 %val, ptr addrspace(1) %addr syncscope("") unordered, align 16
+ ret void
+}
+
+define void @unordered_i128_global_cta_volatile(i128 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i128_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [unordered_i128_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd3, [unordered_i128_global_cta_volatile_param_1];
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 amt, dst;
+; CHECK-NEXT: mov.b128 amt, {%rd1, %rd2};
+; CHECK-NEXT: atom.relaxed.cta.global.exch.b128 dst, [%rd3], amt;
+; CHECK-NEXT: mov.b128 {%rd4, %rd5}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: ret;
+ store atomic volatile i128 %val, ptr addrspace(1) %addr syncscope("block") unordered, align 16
+ ret void
+}
+
+define void @unordered_i128_global_cluster_volatile(i128 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i128_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [unordered_i128_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd3, [unordered_i128_global_cluster_volatile_param_1];
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 amt, dst;
+; CHECK-NEXT: mov.b128 amt, {%rd1, %rd2};
+; CHECK-NEXT: atom.relaxed.cluster.global.exch.b128 dst, [%rd3], amt;
+; CHECK-NEXT: mov.b128 {%rd4, %rd5}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: ret;
+ store atomic volatile i128 %val, ptr addrspace(1) %addr syncscope("cluster") unordered, align 16
+ ret void
+}
+
+define void @unordered_i128_global_gpu_volatile(i128 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_i128_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [unordered_i128_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd3, [unordered_i128_global_gpu_volatile_param_1];
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 amt, dst;
+; CHECK-NEXT: mov.b128 amt, {%rd1, %rd2};
+; CHECK-NEXT: atom.relaxed.gpu.global.exch.b128 dst, [%rd3], amt;
+; CHECK-NEXT: mov.b128 {%rd4, %rd5}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: ret;
+ store atomic volatile i128 %val, ptr addrspace(1) %addr syncscope("device") unordered, align 16
+ ret void
+}
+
+define void @unordered_half_global_sys(half %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_half_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_half_global_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_global_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr addrspace(1) %addr syncscope("") unordered, align 2
+ ret void
+}
+
+define void @unordered_half_global_cta(half %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_half_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_half_global_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_global_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr addrspace(1) %addr syncscope("block") unordered, align 2
+ ret void
+}
+
+define void @unordered_half_global_cluster(half %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_half_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_half_global_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_global_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cluster.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr addrspace(1) %addr syncscope("cluster") unordered, align 2
+ ret void
+}
+
+define void @unordered_half_global_gpu(half %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_half_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_half_global_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_global_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr addrspace(1) %addr syncscope("device") unordered, align 2
+ ret void
+}
+
+define void @unordered_half_global_sys_volatile(half %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_half_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_half_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_global_sys_volatile_param_1];
+; CHECK-NEXT: st.mmio.relaxed.sys.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr addrspace(1) %addr syncscope("") unordered, align 2
+ ret void
+}
+
+define void @unordered_half_global_cta_volatile(half %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_half_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_half_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_global_cta_volatile_param_1];
+; CHECK-NEXT: st.mmio.relaxed.sys.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr addrspace(1) %addr syncscope("block") unordered, align 2
+ ret void
+}
+
+define void @unordered_half_global_cluster_volatile(half %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_half_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_half_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_global_cluster_volatile_param_1];
+; CHECK-NEXT: st.mmio.relaxed.sys.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr addrspace(1) %addr syncscope("cluster") unordered, align 2
+ ret void
+}
+
+define void @unordered_half_global_gpu_volatile(half %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_half_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_half_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_global_gpu_volatile_param_1];
+; CHECK-NEXT: st.mmio.relaxed.sys.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr addrspace(1) %addr syncscope("device") unordered, align 2
+ ret void
+}
+
+define void @unordered_bfloat_global_sys(bfloat %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_bfloat_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_bfloat_global_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_global_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr addrspace(1) %addr syncscope("") unordered, align 2
+ ret void
+}
+
+define void @unordered_bfloat_global_cta(bfloat %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_bfloat_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_bfloat_global_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_global_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr addrspace(1) %addr syncscope("block") unordered, align 2
+ ret void
+}
+
+define void @unordered_bfloat_global_cluster(bfloat %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_bfloat_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_bfloat_global_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_global_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cluster.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr addrspace(1) %addr syncscope("cluster") unordered, align 2
+ ret void
+}
+
+define void @unordered_bfloat_global_gpu(bfloat %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_bfloat_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_bfloat_global_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_global_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr addrspace(1) %addr syncscope("device") unordered, align 2
+ ret void
+}
+
+define void @unordered_bfloat_global_sys_volatile(bfloat %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_bfloat_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_bfloat_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_global_sys_volatile_param_1];
+; CHECK-NEXT: st.mmio.relaxed.sys.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr addrspace(1) %addr syncscope("") unordered, align 2
+ ret void
+}
+
+define void @unordered_bfloat_global_cta_volatile(bfloat %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_bfloat_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_bfloat_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_global_cta_volatile_param_1];
+; CHECK-NEXT: st.mmio.relaxed.sys.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr addrspace(1) %addr syncscope("block") unordered, align 2
+ ret void
+}
+
+define void @unordered_bfloat_global_cluster_volatile(bfloat %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_bfloat_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_bfloat_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_global_cluster_volatile_param_1];
+; CHECK-NEXT: st.mmio.relaxed.sys.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr addrspace(1) %addr syncscope("cluster") unordered, align 2
+ ret void
+}
+
+define void @unordered_bfloat_global_gpu_volatile(bfloat %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_bfloat_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_bfloat_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_global_gpu_volatile_param_1];
+; CHECK-NEXT: st.mmio.relaxed.sys.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr addrspace(1) %addr syncscope("device") unordered, align 2
+ ret void
+}
+
+define void @unordered_float_global_sys(float %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_float_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_float_global_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_global_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr addrspace(1) %addr syncscope("") unordered, align 4
+ ret void
+}
+
+define void @unordered_float_global_cta(float %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_float_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_float_global_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_global_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr addrspace(1) %addr syncscope("block") unordered, align 4
+ ret void
+}
+
+define void @unordered_float_global_cluster(float %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_float_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_float_global_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_global_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cluster.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr addrspace(1) %addr syncscope("cluster") unordered, align 4
+ ret void
+}
+
+define void @unordered_float_global_gpu(float %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_float_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_float_global_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_global_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr addrspace(1) %addr syncscope("device") unordered, align 4
+ ret void
+}
+
+define void @unordered_float_global_sys_volatile(float %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_float_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_float_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_global_sys_volatile_param_1];
+; CHECK-NEXT: st.mmio.relaxed.sys.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr addrspace(1) %addr syncscope("") unordered, align 4
+ ret void
+}
+
+define void @unordered_float_global_cta_volatile(float %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_float_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_float_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_global_cta_volatile_param_1];
+; CHECK-NEXT: st.mmio.relaxed.sys.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr addrspace(1) %addr syncscope("block") unordered, align 4
+ ret void
+}
+
+define void @unordered_float_global_cluster_volatile(float %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_float_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_float_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_global_cluster_volatile_param_1];
+; CHECK-NEXT: st.mmio.relaxed.sys.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr addrspace(1) %addr syncscope("cluster") unordered, align 4
+ ret void
+}
+
+define void @unordered_float_global_gpu_volatile(float %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_float_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_float_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_global_gpu_volatile_param_1];
+; CHECK-NEXT: st.mmio.relaxed.sys.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr addrspace(1) %addr syncscope("device") unordered, align 4
+ ret void
+}
+
+define void @unordered_double_global_sys(double %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_double_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_global_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_double_global_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr addrspace(1) %addr syncscope("") unordered, align 8
+ ret void
+}
+
+define void @unordered_double_global_cta(double %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_double_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_global_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_double_global_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr addrspace(1) %addr syncscope("block") unordered, align 8
+ ret void
+}
+
+define void @unordered_double_global_cluster(double %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_double_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_global_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_double_global_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cluster.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr addrspace(1) %addr syncscope("cluster") unordered, align 8
+ ret void
+}
+
+define void @unordered_double_global_gpu(double %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_double_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_global_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_double_global_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr addrspace(1) %addr syncscope("device") unordered, align 8
+ ret void
+}
+
+define void @unordered_double_global_sys_volatile(double %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_double_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_double_global_sys_volatile_param_1];
+; CHECK-NEXT: st.mmio.relaxed.sys.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr addrspace(1) %addr syncscope("") unordered, align 8
+ ret void
+}
+
+define void @unordered_double_global_cta_volatile(double %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_double_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_double_global_cta_volatile_param_1];
+; CHECK-NEXT: st.mmio.relaxed.sys.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr addrspace(1) %addr syncscope("block") unordered, align 8
+ ret void
+}
+
+define void @unordered_double_global_cluster_volatile(double %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_double_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_double_global_cluster_volatile_param_1];
+; CHECK-NEXT: st.mmio.relaxed.sys.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr addrspace(1) %addr syncscope("cluster") unordered, align 8
+ ret void
+}
+
+define void @unordered_double_global_gpu_volatile(double %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: unordered_double_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_double_global_gpu_volatile_param_1];
+; CHECK-NEXT: st.mmio.relaxed.sys.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr addrspace(1) %addr syncscope("device") unordered, align 8
+ ret void
+}
+
+define void @unordered_i8_shared_sys(i8 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i8_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [unordered_i8_shared_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_shared_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.shared.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr addrspace(3) %addr syncscope("") unordered, align 1
+ ret void
+}
+
+define void @unordered_i8_shared_cta(i8 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i8_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [unordered_i8_shared_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_shared_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.shared.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr addrspace(3) %addr syncscope("block") unordered, align 1
+ ret void
+}
+
+define void @unordered_i8_shared_cluster(i8 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i8_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [unordered_i8_shared_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_shared_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cluster.shared.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr addrspace(3) %addr syncscope("cluster") unordered, align 1
+ ret void
+}
+
+define void @unordered_i8_shared_gpu(i8 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i8_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [unordered_i8_shared_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_shared_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.shared.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr addrspace(3) %addr syncscope("device") unordered, align 1
+ ret void
+}
+
+define void @unordered_i8_shared_sys_volatile(i8 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i8_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [unordered_i8_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_shared_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr addrspace(3) %addr syncscope("") unordered, align 1
+ ret void
+}
+
+define void @unordered_i8_shared_cta_volatile(i8 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i8_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [unordered_i8_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_shared_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr addrspace(3) %addr syncscope("block") unordered, align 1
+ ret void
+}
+
+define void @unordered_i8_shared_cluster_volatile(i8 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i8_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [unordered_i8_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_shared_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr addrspace(3) %addr syncscope("cluster") unordered, align 1
+ ret void
+}
+
+define void @unordered_i8_shared_gpu_volatile(i8 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i8_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [unordered_i8_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i8_shared_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr addrspace(3) %addr syncscope("device") unordered, align 1
+ ret void
+}
+
+define void @unordered_i16_shared_sys(i16 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i16_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_i16_shared_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_shared_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr addrspace(3) %addr syncscope("") unordered, align 2
+ ret void
+}
+
+define void @unordered_i16_shared_cta(i16 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i16_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_i16_shared_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_shared_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr addrspace(3) %addr syncscope("block") unordered, align 2
+ ret void
+}
+
+define void @unordered_i16_shared_cluster(i16 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i16_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_i16_shared_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_shared_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cluster.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr addrspace(3) %addr syncscope("cluster") unordered, align 2
+ ret void
+}
+
+define void @unordered_i16_shared_gpu(i16 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i16_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_i16_shared_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_shared_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr addrspace(3) %addr syncscope("device") unordered, align 2
+ ret void
+}
+
+define void @unordered_i16_shared_sys_volatile(i16 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i16_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_i16_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_shared_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr addrspace(3) %addr syncscope("") unordered, align 2
+ ret void
+}
+
+define void @unordered_i16_shared_cta_volatile(i16 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i16_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_i16_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_shared_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr addrspace(3) %addr syncscope("block") unordered, align 2
+ ret void
+}
+
+define void @unordered_i16_shared_cluster_volatile(i16 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i16_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_i16_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_shared_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr addrspace(3) %addr syncscope("cluster") unordered, align 2
+ ret void
+}
+
+define void @unordered_i16_shared_gpu_volatile(i16 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i16_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_i16_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i16_shared_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr addrspace(3) %addr syncscope("device") unordered, align 2
+ ret void
+}
+
+define void @unordered_i32_shared_sys(i32 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i32_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_i32_shared_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_shared_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr addrspace(3) %addr syncscope("") unordered, align 4
+ ret void
+}
+
+define void @unordered_i32_shared_cta(i32 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i32_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_i32_shared_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_shared_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr addrspace(3) %addr syncscope("block") unordered, align 4
+ ret void
+}
+
+define void @unordered_i32_shared_cluster(i32 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i32_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_i32_shared_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_shared_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cluster.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr addrspace(3) %addr syncscope("cluster") unordered, align 4
+ ret void
+}
+
+define void @unordered_i32_shared_gpu(i32 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i32_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_i32_shared_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_shared_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr addrspace(3) %addr syncscope("device") unordered, align 4
+ ret void
+}
+
+define void @unordered_i32_shared_sys_volatile(i32 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i32_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_i32_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_shared_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr addrspace(3) %addr syncscope("") unordered, align 4
+ ret void
+}
+
+define void @unordered_i32_shared_cta_volatile(i32 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i32_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_i32_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_shared_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr addrspace(3) %addr syncscope("block") unordered, align 4
+ ret void
+}
+
+define void @unordered_i32_shared_cluster_volatile(i32 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i32_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_i32_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_shared_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr addrspace(3) %addr syncscope("cluster") unordered, align 4
+ ret void
+}
+
+define void @unordered_i32_shared_gpu_volatile(i32 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i32_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_i32_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i32_shared_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr addrspace(3) %addr syncscope("device") unordered, align 4
+ ret void
+}
+
+define void @unordered_i64_shared_sys(i64 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i64_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_shared_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_i64_shared_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr addrspace(3) %addr syncscope("") unordered, align 8
+ ret void
+}
+
+define void @unordered_i64_shared_cta(i64 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i64_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_shared_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_i64_shared_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr addrspace(3) %addr syncscope("block") unordered, align 8
+ ret void
+}
+
+define void @unordered_i64_shared_cluster(i64 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i64_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_shared_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_i64_shared_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cluster.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr addrspace(3) %addr syncscope("cluster") unordered, align 8
+ ret void
+}
+
+define void @unordered_i64_shared_gpu(i64 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i64_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_shared_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_i64_shared_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr addrspace(3) %addr syncscope("device") unordered, align 8
+ ret void
+}
+
+define void @unordered_i64_shared_sys_volatile(i64 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i64_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_i64_shared_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr addrspace(3) %addr syncscope("") unordered, align 8
+ ret void
+}
+
+define void @unordered_i64_shared_cta_volatile(i64 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i64_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_i64_shared_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr addrspace(3) %addr syncscope("block") unordered, align 8
+ ret void
+}
+
+define void @unordered_i64_shared_cluster_volatile(i64 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i64_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_i64_shared_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr addrspace(3) %addr syncscope("cluster") unordered, align 8
+ ret void
+}
+
+define void @unordered_i64_shared_gpu_volatile(i64 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i64_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_i64_shared_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr addrspace(3) %addr syncscope("device") unordered, align 8
+ ret void
+}
+
+define void @unordered_i128_shared_sys(i128 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i128_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [unordered_i128_shared_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd3, [unordered_i128_shared_sys_param_1];
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 amt, dst;
+; CHECK-NEXT: mov.b128 amt, {%rd1, %rd2};
+; CHECK-NEXT: atom.relaxed.sys.shared.exch.b128 dst, [%rd3], amt;
+; CHECK-NEXT: mov.b128 {%rd4, %rd5}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: ret;
+ store atomic i128 %val, ptr addrspace(3) %addr syncscope("") unordered, align 16
+ ret void
+}
+
+define void @unordered_i128_shared_cta(i128 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i128_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [unordered_i128_shared_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd3, [unordered_i128_shared_cta_param_1];
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 amt, dst;
+; CHECK-NEXT: mov.b128 amt, {%rd1, %rd2};
+; CHECK-NEXT: atom.relaxed.cta.shared.exch.b128 dst, [%rd3], amt;
+; CHECK-NEXT: mov.b128 {%rd4, %rd5}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: ret;
+ store atomic i128 %val, ptr addrspace(3) %addr syncscope("block") unordered, align 16
+ ret void
+}
+
+define void @unordered_i128_shared_cluster(i128 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i128_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [unordered_i128_shared_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd3, [unordered_i128_shared_cluster_param_1];
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 amt, dst;
+; CHECK-NEXT: mov.b128 amt, {%rd1, %rd2};
+; CHECK-NEXT: atom.relaxed.cluster.shared.exch.b128 dst, [%rd3], amt;
+; CHECK-NEXT: mov.b128 {%rd4, %rd5}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: ret;
+ store atomic i128 %val, ptr addrspace(3) %addr syncscope("cluster") unordered, align 16
+ ret void
+}
+
+define void @unordered_i128_shared_gpu(i128 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i128_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [unordered_i128_shared_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd3, [unordered_i128_shared_gpu_param_1];
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 amt, dst;
+; CHECK-NEXT: mov.b128 amt, {%rd1, %rd2};
+; CHECK-NEXT: atom.relaxed.gpu.shared.exch.b128 dst, [%rd3], amt;
+; CHECK-NEXT: mov.b128 {%rd4, %rd5}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: ret;
+ store atomic i128 %val, ptr addrspace(3) %addr syncscope("device") unordered, align 16
+ ret void
+}
+
+define void @unordered_i128_shared_sys_volatile(i128 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i128_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [unordered_i128_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd3, [unordered_i128_shared_sys_volatile_param_1];
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 amt, dst;
+; CHECK-NEXT: mov.b128 amt, {%rd1, %rd2};
+; CHECK-NEXT: atom.relaxed.sys.shared.exch.b128 dst, [%rd3], amt;
+; CHECK-NEXT: mov.b128 {%rd4, %rd5}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: ret;
+ store atomic volatile i128 %val, ptr addrspace(3) %addr syncscope("") unordered, align 16
+ ret void
+}
+
+define void @unordered_i128_shared_cta_volatile(i128 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i128_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [unordered_i128_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd3, [unordered_i128_shared_cta_volatile_param_1];
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 amt, dst;
+; CHECK-NEXT: mov.b128 amt, {%rd1, %rd2};
+; CHECK-NEXT: atom.relaxed.cta.shared.exch.b128 dst, [%rd3], amt;
+; CHECK-NEXT: mov.b128 {%rd4, %rd5}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: ret;
+ store atomic volatile i128 %val, ptr addrspace(3) %addr syncscope("block") unordered, align 16
+ ret void
+}
+
+define void @unordered_i128_shared_cluster_volatile(i128 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i128_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [unordered_i128_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd3, [unordered_i128_shared_cluster_volatile_param_1];
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 amt, dst;
+; CHECK-NEXT: mov.b128 amt, {%rd1, %rd2};
+; CHECK-NEXT: atom.relaxed.cluster.shared.exch.b128 dst, [%rd3], amt;
+; CHECK-NEXT: mov.b128 {%rd4, %rd5}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: ret;
+ store atomic volatile i128 %val, ptr addrspace(3) %addr syncscope("cluster") unordered, align 16
+ ret void
+}
+
+define void @unordered_i128_shared_gpu_volatile(i128 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_i128_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [unordered_i128_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd3, [unordered_i128_shared_gpu_volatile_param_1];
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 amt, dst;
+; CHECK-NEXT: mov.b128 amt, {%rd1, %rd2};
+; CHECK-NEXT: atom.relaxed.gpu.shared.exch.b128 dst, [%rd3], amt;
+; CHECK-NEXT: mov.b128 {%rd4, %rd5}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: ret;
+ store atomic volatile i128 %val, ptr addrspace(3) %addr syncscope("device") unordered, align 16
+ ret void
+}
+
+define void @unordered_half_shared_sys(half %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_half_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_half_shared_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_shared_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr addrspace(3) %addr syncscope("") unordered, align 2
+ ret void
+}
+
+define void @unordered_half_shared_cta(half %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_half_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_half_shared_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_shared_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr addrspace(3) %addr syncscope("block") unordered, align 2
+ ret void
+}
+
+define void @unordered_half_shared_cluster(half %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_half_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_half_shared_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_shared_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cluster.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr addrspace(3) %addr syncscope("cluster") unordered, align 2
+ ret void
+}
+
+define void @unordered_half_shared_gpu(half %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_half_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_half_shared_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_shared_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr addrspace(3) %addr syncscope("device") unordered, align 2
+ ret void
+}
+
+define void @unordered_half_shared_sys_volatile(half %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_half_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_half_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_shared_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr addrspace(3) %addr syncscope("") unordered, align 2
+ ret void
+}
+
+define void @unordered_half_shared_cta_volatile(half %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_half_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_half_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_shared_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr addrspace(3) %addr syncscope("block") unordered, align 2
+ ret void
+}
+
+define void @unordered_half_shared_cluster_volatile(half %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_half_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_half_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_shared_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr addrspace(3) %addr syncscope("cluster") unordered, align 2
+ ret void
+}
+
+define void @unordered_half_shared_gpu_volatile(half %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_half_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_half_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_half_shared_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr addrspace(3) %addr syncscope("device") unordered, align 2
+ ret void
+}
+
+define void @unordered_bfloat_shared_sys(bfloat %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_bfloat_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_bfloat_shared_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_shared_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr addrspace(3) %addr syncscope("") unordered, align 2
+ ret void
+}
+
+define void @unordered_bfloat_shared_cta(bfloat %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_bfloat_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_bfloat_shared_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_shared_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr addrspace(3) %addr syncscope("block") unordered, align 2
+ ret void
+}
+
+define void @unordered_bfloat_shared_cluster(bfloat %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_bfloat_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_bfloat_shared_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_shared_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cluster.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr addrspace(3) %addr syncscope("cluster") unordered, align 2
+ ret void
+}
+
+define void @unordered_bfloat_shared_gpu(bfloat %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_bfloat_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_bfloat_shared_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_shared_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr addrspace(3) %addr syncscope("device") unordered, align 2
+ ret void
+}
+
+define void @unordered_bfloat_shared_sys_volatile(bfloat %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_bfloat_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_bfloat_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_shared_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr addrspace(3) %addr syncscope("") unordered, align 2
+ ret void
+}
+
+define void @unordered_bfloat_shared_cta_volatile(bfloat %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_bfloat_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_bfloat_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_shared_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr addrspace(3) %addr syncscope("block") unordered, align 2
+ ret void
+}
+
+define void @unordered_bfloat_shared_cluster_volatile(bfloat %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_bfloat_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_bfloat_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_shared_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr addrspace(3) %addr syncscope("cluster") unordered, align 2
+ ret void
+}
+
+define void @unordered_bfloat_shared_gpu_volatile(bfloat %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_bfloat_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_bfloat_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_bfloat_shared_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr addrspace(3) %addr syncscope("device") unordered, align 2
+ ret void
+}
+
+define void @unordered_float_shared_sys(float %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_float_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_float_shared_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_shared_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr addrspace(3) %addr syncscope("") unordered, align 4
+ ret void
+}
+
+define void @unordered_float_shared_cta(float %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_float_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_float_shared_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_shared_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr addrspace(3) %addr syncscope("block") unordered, align 4
+ ret void
+}
+
+define void @unordered_float_shared_cluster(float %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_float_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_float_shared_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_shared_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cluster.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr addrspace(3) %addr syncscope("cluster") unordered, align 4
+ ret void
+}
+
+define void @unordered_float_shared_gpu(float %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_float_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_float_shared_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_shared_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr addrspace(3) %addr syncscope("device") unordered, align 4
+ ret void
+}
+
+define void @unordered_float_shared_sys_volatile(float %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_float_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_float_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_shared_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr addrspace(3) %addr syncscope("") unordered, align 4
+ ret void
+}
+
+define void @unordered_float_shared_cta_volatile(float %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_float_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_float_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_shared_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr addrspace(3) %addr syncscope("block") unordered, align 4
+ ret void
+}
+
+define void @unordered_float_shared_cluster_volatile(float %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_float_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_float_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_shared_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr addrspace(3) %addr syncscope("cluster") unordered, align 4
+ ret void
+}
+
+define void @unordered_float_shared_gpu_volatile(float %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_float_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [unordered_float_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_float_shared_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr addrspace(3) %addr syncscope("device") unordered, align 4
+ ret void
+}
+
+define void @unordered_double_shared_sys(double %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_double_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_shared_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_double_shared_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr addrspace(3) %addr syncscope("") unordered, align 8
+ ret void
+}
+
+define void @unordered_double_shared_cta(double %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_double_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_shared_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_double_shared_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr addrspace(3) %addr syncscope("block") unordered, align 8
+ ret void
+}
+
+define void @unordered_double_shared_cluster(double %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_double_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_shared_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_double_shared_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cluster.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr addrspace(3) %addr syncscope("cluster") unordered, align 8
+ ret void
+}
+
+define void @unordered_double_shared_gpu(double %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_double_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_shared_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_double_shared_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr addrspace(3) %addr syncscope("device") unordered, align 8
+ ret void
+}
+
+define void @unordered_double_shared_sys_volatile(double %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_double_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_double_shared_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr addrspace(3) %addr syncscope("") unordered, align 8
+ ret void
+}
+
+define void @unordered_double_shared_cta_volatile(double %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_double_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_double_shared_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr addrspace(3) %addr syncscope("block") unordered, align 8
+ ret void
+}
+
+define void @unordered_double_shared_cluster_volatile(double %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_double_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_double_shared_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr addrspace(3) %addr syncscope("cluster") unordered, align 8
+ ret void
+}
+
+define void @unordered_double_shared_gpu_volatile(double %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: unordered_double_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [unordered_double_shared_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr addrspace(3) %addr syncscope("device") unordered, align 8
+ ret void
+}
+
+define void @unordered_i8_local_sys(i8 %val) {
+; CHECK-LABEL: unordered_i8_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot288[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot288;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b8 %rs1, [unordered_i8_local_sys_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b8 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i8 %val, ptr addrspace(5) %slot syncscope("") unordered, align 1
+ ret void
+}
+
+define void @unordered_i8_local_cta(i8 %val) {
+; CHECK-LABEL: unordered_i8_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot289[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot289;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b8 %rs1, [unordered_i8_local_cta_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b8 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i8 %val, ptr addrspace(5) %slot syncscope("block") unordered, align 1
+ ret void
+}
+
+define void @unordered_i8_local_cluster(i8 %val) {
+; CHECK-LABEL: unordered_i8_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot290[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot290;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b8 %rs1, [unordered_i8_local_cluster_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b8 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i8 %val, ptr addrspace(5) %slot syncscope("cluster") unordered, align 1
+ ret void
+}
+
+define void @unordered_i8_local_gpu(i8 %val) {
+; CHECK-LABEL: unordered_i8_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot291[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot291;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b8 %rs1, [unordered_i8_local_gpu_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b8 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i8 %val, ptr addrspace(5) %slot syncscope("device") unordered, align 1
+ ret void
+}
+
+define void @unordered_i8_local_sys_volatile(i8 %val) {
+; CHECK-LABEL: unordered_i8_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot292[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot292;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b8 %rs1, [unordered_i8_local_sys_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b8 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i8 %val, ptr addrspace(5) %slot syncscope("") unordered, align 1
+ ret void
+}
+
+define void @unordered_i8_local_cta_volatile(i8 %val) {
+; CHECK-LABEL: unordered_i8_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot293[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot293;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b8 %rs1, [unordered_i8_local_cta_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b8 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i8 %val, ptr addrspace(5) %slot syncscope("block") unordered, align 1
+ ret void
+}
+
+define void @unordered_i8_local_cluster_volatile(i8 %val) {
+; CHECK-LABEL: unordered_i8_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot294[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot294;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b8 %rs1, [unordered_i8_local_cluster_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b8 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i8 %val, ptr addrspace(5) %slot syncscope("cluster") unordered, align 1
+ ret void
+}
+
+define void @unordered_i8_local_gpu_volatile(i8 %val) {
+; CHECK-LABEL: unordered_i8_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot295[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot295;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b8 %rs1, [unordered_i8_local_gpu_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b8 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i8 %val, ptr addrspace(5) %slot syncscope("device") unordered, align 1
+ ret void
+}
+
+define void @unordered_i16_local_sys(i16 %val) {
+; CHECK-LABEL: unordered_i16_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot296[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot296;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_i16_local_sys_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i16 %val, ptr addrspace(5) %slot syncscope("") unordered, align 2
+ ret void
+}
+
+define void @unordered_i16_local_cta(i16 %val) {
+; CHECK-LABEL: unordered_i16_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot297[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot297;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_i16_local_cta_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i16 %val, ptr addrspace(5) %slot syncscope("block") unordered, align 2
+ ret void
+}
+
+define void @unordered_i16_local_cluster(i16 %val) {
+; CHECK-LABEL: unordered_i16_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot298[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot298;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_i16_local_cluster_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i16 %val, ptr addrspace(5) %slot syncscope("cluster") unordered, align 2
+ ret void
+}
+
+define void @unordered_i16_local_gpu(i16 %val) {
+; CHECK-LABEL: unordered_i16_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot299[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot299;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_i16_local_gpu_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i16 %val, ptr addrspace(5) %slot syncscope("device") unordered, align 2
+ ret void
+}
+
+define void @unordered_i16_local_sys_volatile(i16 %val) {
+; CHECK-LABEL: unordered_i16_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot300[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot300;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_i16_local_sys_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i16 %val, ptr addrspace(5) %slot syncscope("") unordered, align 2
+ ret void
+}
+
+define void @unordered_i16_local_cta_volatile(i16 %val) {
+; CHECK-LABEL: unordered_i16_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot301[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot301;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_i16_local_cta_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i16 %val, ptr addrspace(5) %slot syncscope("block") unordered, align 2
+ ret void
+}
+
+define void @unordered_i16_local_cluster_volatile(i16 %val) {
+; CHECK-LABEL: unordered_i16_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot302[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot302;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_i16_local_cluster_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i16 %val, ptr addrspace(5) %slot syncscope("cluster") unordered, align 2
+ ret void
+}
+
+define void @unordered_i16_local_gpu_volatile(i16 %val) {
+; CHECK-LABEL: unordered_i16_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot303[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot303;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_i16_local_gpu_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i16 %val, ptr addrspace(5) %slot syncscope("device") unordered, align 2
+ ret void
+}
+
+define void @unordered_i32_local_sys(i32 %val) {
+; CHECK-LABEL: unordered_i32_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot304[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot304;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [unordered_i32_local_sys_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i32 %val, ptr addrspace(5) %slot syncscope("") unordered, align 4
+ ret void
+}
+
+define void @unordered_i32_local_cta(i32 %val) {
+; CHECK-LABEL: unordered_i32_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot305[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot305;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [unordered_i32_local_cta_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i32 %val, ptr addrspace(5) %slot syncscope("block") unordered, align 4
+ ret void
+}
+
+define void @unordered_i32_local_cluster(i32 %val) {
+; CHECK-LABEL: unordered_i32_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot306[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot306;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [unordered_i32_local_cluster_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i32 %val, ptr addrspace(5) %slot syncscope("cluster") unordered, align 4
+ ret void
+}
+
+define void @unordered_i32_local_gpu(i32 %val) {
+; CHECK-LABEL: unordered_i32_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot307[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot307;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [unordered_i32_local_gpu_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i32 %val, ptr addrspace(5) %slot syncscope("device") unordered, align 4
+ ret void
+}
+
+define void @unordered_i32_local_sys_volatile(i32 %val) {
+; CHECK-LABEL: unordered_i32_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot308[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot308;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [unordered_i32_local_sys_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i32 %val, ptr addrspace(5) %slot syncscope("") unordered, align 4
+ ret void
+}
+
+define void @unordered_i32_local_cta_volatile(i32 %val) {
+; CHECK-LABEL: unordered_i32_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot309[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot309;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [unordered_i32_local_cta_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i32 %val, ptr addrspace(5) %slot syncscope("block") unordered, align 4
+ ret void
+}
+
+define void @unordered_i32_local_cluster_volatile(i32 %val) {
+; CHECK-LABEL: unordered_i32_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot310[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot310;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [unordered_i32_local_cluster_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i32 %val, ptr addrspace(5) %slot syncscope("cluster") unordered, align 4
+ ret void
+}
+
+define void @unordered_i32_local_gpu_volatile(i32 %val) {
+; CHECK-LABEL: unordered_i32_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot311[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot311;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [unordered_i32_local_gpu_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i32 %val, ptr addrspace(5) %slot syncscope("device") unordered, align 4
+ ret void
+}
+
+define void @unordered_i64_local_sys(i64 %val) {
+; CHECK-LABEL: unordered_i64_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot312[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot312;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_local_sys_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i64 %val, ptr addrspace(5) %slot syncscope("") unordered, align 8
+ ret void
+}
+
+define void @unordered_i64_local_cta(i64 %val) {
+; CHECK-LABEL: unordered_i64_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot313[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot313;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_local_cta_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i64 %val, ptr addrspace(5) %slot syncscope("block") unordered, align 8
+ ret void
+}
+
+define void @unordered_i64_local_cluster(i64 %val) {
+; CHECK-LABEL: unordered_i64_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot314[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot314;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_local_cluster_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i64 %val, ptr addrspace(5) %slot syncscope("cluster") unordered, align 8
+ ret void
+}
+
+define void @unordered_i64_local_gpu(i64 %val) {
+; CHECK-LABEL: unordered_i64_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot315[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot315;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_local_gpu_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i64 %val, ptr addrspace(5) %slot syncscope("device") unordered, align 8
+ ret void
+}
+
+define void @unordered_i64_local_sys_volatile(i64 %val) {
+; CHECK-LABEL: unordered_i64_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot316[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot316;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_local_sys_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i64 %val, ptr addrspace(5) %slot syncscope("") unordered, align 8
+ ret void
+}
+
+define void @unordered_i64_local_cta_volatile(i64 %val) {
+; CHECK-LABEL: unordered_i64_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot317[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot317;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_local_cta_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i64 %val, ptr addrspace(5) %slot syncscope("block") unordered, align 8
+ ret void
+}
+
+define void @unordered_i64_local_cluster_volatile(i64 %val) {
+; CHECK-LABEL: unordered_i64_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot318[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot318;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_local_cluster_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i64 %val, ptr addrspace(5) %slot syncscope("cluster") unordered, align 8
+ ret void
+}
+
+define void @unordered_i64_local_gpu_volatile(i64 %val) {
+; CHECK-LABEL: unordered_i64_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot319[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot319;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_i64_local_gpu_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i64 %val, ptr addrspace(5) %slot syncscope("device") unordered, align 8
+ ret void
+}
+
+define void @unordered_i128_local_sys(i128 %val) {
+; CHECK-LABEL: unordered_i128_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 16 .b8 __local_depot320[16];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot320;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [unordered_i128_local_sys_param_0];
+; CHECK-NEXT: add.u64 %rd3, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd3;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.v2.b64 [%SP], {%rd1, %rd2};
+; CHECK-NEXT: ret;
+ %slot = alloca i128, align 16, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i128 %val, ptr addrspace(5) %slot syncscope("") unordered, align 16
+ ret void
+}
+
+define void @unordered_i128_local_cta(i128 %val) {
+; CHECK-LABEL: unordered_i128_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 16 .b8 __local_depot321[16];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot321;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [unordered_i128_local_cta_param_0];
+; CHECK-NEXT: add.u64 %rd3, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd3;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.v2.b64 [%SP], {%rd1, %rd2};
+; CHECK-NEXT: ret;
+ %slot = alloca i128, align 16, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i128 %val, ptr addrspace(5) %slot syncscope("block") unordered, align 16
+ ret void
+}
+
+define void @unordered_i128_local_cluster(i128 %val) {
+; CHECK-LABEL: unordered_i128_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 16 .b8 __local_depot322[16];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot322;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [unordered_i128_local_cluster_param_0];
+; CHECK-NEXT: add.u64 %rd3, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd3;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.v2.b64 [%SP], {%rd1, %rd2};
+; CHECK-NEXT: ret;
+ %slot = alloca i128, align 16, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i128 %val, ptr addrspace(5) %slot syncscope("cluster") unordered, align 16
+ ret void
+}
+
+define void @unordered_i128_local_gpu(i128 %val) {
+; CHECK-LABEL: unordered_i128_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 16 .b8 __local_depot323[16];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot323;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [unordered_i128_local_gpu_param_0];
+; CHECK-NEXT: add.u64 %rd3, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd3;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.v2.b64 [%SP], {%rd1, %rd2};
+; CHECK-NEXT: ret;
+ %slot = alloca i128, align 16, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i128 %val, ptr addrspace(5) %slot syncscope("device") unordered, align 16
+ ret void
+}
+
+define void @unordered_i128_local_sys_volatile(i128 %val) {
+; CHECK-LABEL: unordered_i128_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 16 .b8 __local_depot324[16];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot324;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [unordered_i128_local_sys_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd3, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd3;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.v2.b64 [%SP], {%rd1, %rd2};
+; CHECK-NEXT: ret;
+ %slot = alloca i128, align 16, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i128 %val, ptr addrspace(5) %slot syncscope("") unordered, align 16
+ ret void
+}
+
+define void @unordered_i128_local_cta_volatile(i128 %val) {
+; CHECK-LABEL: unordered_i128_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 16 .b8 __local_depot325[16];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot325;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [unordered_i128_local_cta_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd3, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd3;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.v2.b64 [%SP], {%rd1, %rd2};
+; CHECK-NEXT: ret;
+ %slot = alloca i128, align 16, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i128 %val, ptr addrspace(5) %slot syncscope("block") unordered, align 16
+ ret void
+}
+
+define void @unordered_i128_local_cluster_volatile(i128 %val) {
+; CHECK-LABEL: unordered_i128_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 16 .b8 __local_depot326[16];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot326;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [unordered_i128_local_cluster_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd3, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd3;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.v2.b64 [%SP], {%rd1, %rd2};
+; CHECK-NEXT: ret;
+ %slot = alloca i128, align 16, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i128 %val, ptr addrspace(5) %slot syncscope("cluster") unordered, align 16
+ ret void
+}
+
+define void @unordered_i128_local_gpu_volatile(i128 %val) {
+; CHECK-LABEL: unordered_i128_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 16 .b8 __local_depot327[16];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot327;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [unordered_i128_local_gpu_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd3, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd3;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.v2.b64 [%SP], {%rd1, %rd2};
+; CHECK-NEXT: ret;
+ %slot = alloca i128, align 16, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i128 %val, ptr addrspace(5) %slot syncscope("device") unordered, align 16
+ ret void
+}
+
+define void @unordered_half_local_sys(half %val) {
+; CHECK-LABEL: unordered_half_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot328[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot328;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_half_local_sys_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic half %val, ptr addrspace(5) %slot syncscope("") unordered, align 2
+ ret void
+}
+
+define void @unordered_half_local_cta(half %val) {
+; CHECK-LABEL: unordered_half_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot329[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot329;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_half_local_cta_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic half %val, ptr addrspace(5) %slot syncscope("block") unordered, align 2
+ ret void
+}
+
+define void @unordered_half_local_cluster(half %val) {
+; CHECK-LABEL: unordered_half_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot330[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot330;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_half_local_cluster_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic half %val, ptr addrspace(5) %slot syncscope("cluster") unordered, align 2
+ ret void
+}
+
+define void @unordered_half_local_gpu(half %val) {
+; CHECK-LABEL: unordered_half_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot331[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot331;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_half_local_gpu_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic half %val, ptr addrspace(5) %slot syncscope("device") unordered, align 2
+ ret void
+}
+
+define void @unordered_half_local_sys_volatile(half %val) {
+; CHECK-LABEL: unordered_half_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot332[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot332;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_half_local_sys_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile half %val, ptr addrspace(5) %slot syncscope("") unordered, align 2
+ ret void
+}
+
+define void @unordered_half_local_cta_volatile(half %val) {
+; CHECK-LABEL: unordered_half_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot333[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot333;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_half_local_cta_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile half %val, ptr addrspace(5) %slot syncscope("block") unordered, align 2
+ ret void
+}
+
+define void @unordered_half_local_cluster_volatile(half %val) {
+; CHECK-LABEL: unordered_half_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot334[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot334;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_half_local_cluster_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile half %val, ptr addrspace(5) %slot syncscope("cluster") unordered, align 2
+ ret void
+}
+
+define void @unordered_half_local_gpu_volatile(half %val) {
+; CHECK-LABEL: unordered_half_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot335[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot335;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_half_local_gpu_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile half %val, ptr addrspace(5) %slot syncscope("device") unordered, align 2
+ ret void
+}
+
+define void @unordered_bfloat_local_sys(bfloat %val) {
+; CHECK-LABEL: unordered_bfloat_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot336[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot336;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_bfloat_local_sys_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic bfloat %val, ptr addrspace(5) %slot syncscope("") unordered, align 2
+ ret void
+}
+
+define void @unordered_bfloat_local_cta(bfloat %val) {
+; CHECK-LABEL: unordered_bfloat_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot337[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot337;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_bfloat_local_cta_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic bfloat %val, ptr addrspace(5) %slot syncscope("block") unordered, align 2
+ ret void
+}
+
+define void @unordered_bfloat_local_cluster(bfloat %val) {
+; CHECK-LABEL: unordered_bfloat_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot338[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot338;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_bfloat_local_cluster_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic bfloat %val, ptr addrspace(5) %slot syncscope("cluster") unordered, align 2
+ ret void
+}
+
+define void @unordered_bfloat_local_gpu(bfloat %val) {
+; CHECK-LABEL: unordered_bfloat_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot339[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot339;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_bfloat_local_gpu_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic bfloat %val, ptr addrspace(5) %slot syncscope("device") unordered, align 2
+ ret void
+}
+
+define void @unordered_bfloat_local_sys_volatile(bfloat %val) {
+; CHECK-LABEL: unordered_bfloat_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot340[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot340;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_bfloat_local_sys_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile bfloat %val, ptr addrspace(5) %slot syncscope("") unordered, align 2
+ ret void
+}
+
+define void @unordered_bfloat_local_cta_volatile(bfloat %val) {
+; CHECK-LABEL: unordered_bfloat_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot341[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot341;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_bfloat_local_cta_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile bfloat %val, ptr addrspace(5) %slot syncscope("block") unordered, align 2
+ ret void
+}
+
+define void @unordered_bfloat_local_cluster_volatile(bfloat %val) {
+; CHECK-LABEL: unordered_bfloat_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot342[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot342;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_bfloat_local_cluster_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile bfloat %val, ptr addrspace(5) %slot syncscope("cluster") unordered, align 2
+ ret void
+}
+
+define void @unordered_bfloat_local_gpu_volatile(bfloat %val) {
+; CHECK-LABEL: unordered_bfloat_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot343[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot343;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [unordered_bfloat_local_gpu_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile bfloat %val, ptr addrspace(5) %slot syncscope("device") unordered, align 2
+ ret void
+}
+
+define void @unordered_float_local_sys(float %val) {
+; CHECK-LABEL: unordered_float_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot344[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot344;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [unordered_float_local_sys_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic float %val, ptr addrspace(5) %slot syncscope("") unordered, align 4
+ ret void
+}
+
+define void @unordered_float_local_cta(float %val) {
+; CHECK-LABEL: unordered_float_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot345[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot345;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [unordered_float_local_cta_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic float %val, ptr addrspace(5) %slot syncscope("block") unordered, align 4
+ ret void
+}
+
+define void @unordered_float_local_cluster(float %val) {
+; CHECK-LABEL: unordered_float_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot346[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot346;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [unordered_float_local_cluster_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic float %val, ptr addrspace(5) %slot syncscope("cluster") unordered, align 4
+ ret void
+}
+
+define void @unordered_float_local_gpu(float %val) {
+; CHECK-LABEL: unordered_float_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot347[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot347;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [unordered_float_local_gpu_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic float %val, ptr addrspace(5) %slot syncscope("device") unordered, align 4
+ ret void
+}
+
+define void @unordered_float_local_sys_volatile(float %val) {
+; CHECK-LABEL: unordered_float_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot348[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot348;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [unordered_float_local_sys_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile float %val, ptr addrspace(5) %slot syncscope("") unordered, align 4
+ ret void
+}
+
+define void @unordered_float_local_cta_volatile(float %val) {
+; CHECK-LABEL: unordered_float_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot349[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot349;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [unordered_float_local_cta_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile float %val, ptr addrspace(5) %slot syncscope("block") unordered, align 4
+ ret void
+}
+
+define void @unordered_float_local_cluster_volatile(float %val) {
+; CHECK-LABEL: unordered_float_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot350[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot350;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [unordered_float_local_cluster_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile float %val, ptr addrspace(5) %slot syncscope("cluster") unordered, align 4
+ ret void
+}
+
+define void @unordered_float_local_gpu_volatile(float %val) {
+; CHECK-LABEL: unordered_float_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot351[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot351;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [unordered_float_local_gpu_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile float %val, ptr addrspace(5) %slot syncscope("device") unordered, align 4
+ ret void
+}
+
+define void @unordered_double_local_sys(double %val) {
+; CHECK-LABEL: unordered_double_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot352[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot352;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_local_sys_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic double %val, ptr addrspace(5) %slot syncscope("") unordered, align 8
+ ret void
+}
+
+define void @unordered_double_local_cta(double %val) {
+; CHECK-LABEL: unordered_double_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot353[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot353;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_local_cta_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic double %val, ptr addrspace(5) %slot syncscope("block") unordered, align 8
+ ret void
+}
+
+define void @unordered_double_local_cluster(double %val) {
+; CHECK-LABEL: unordered_double_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot354[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot354;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_local_cluster_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic double %val, ptr addrspace(5) %slot syncscope("cluster") unordered, align 8
+ ret void
+}
+
+define void @unordered_double_local_gpu(double %val) {
+; CHECK-LABEL: unordered_double_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot355[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot355;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_local_gpu_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic double %val, ptr addrspace(5) %slot syncscope("device") unordered, align 8
+ ret void
+}
+
+define void @unordered_double_local_sys_volatile(double %val) {
+; CHECK-LABEL: unordered_double_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot356[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot356;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_local_sys_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile double %val, ptr addrspace(5) %slot syncscope("") unordered, align 8
+ ret void
+}
+
+define void @unordered_double_local_cta_volatile(double %val) {
+; CHECK-LABEL: unordered_double_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot357[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot357;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_local_cta_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile double %val, ptr addrspace(5) %slot syncscope("block") unordered, align 8
+ ret void
+}
+
+define void @unordered_double_local_cluster_volatile(double %val) {
+; CHECK-LABEL: unordered_double_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot358[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot358;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_local_cluster_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile double %val, ptr addrspace(5) %slot syncscope("cluster") unordered, align 8
+ ret void
+}
+
+define void @unordered_double_local_gpu_volatile(double %val) {
+; CHECK-LABEL: unordered_double_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot359[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot359;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [unordered_double_local_gpu_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile double %val, ptr addrspace(5) %slot syncscope("device") unordered, align 8
+ ret void
+}
+
+define void @monotonic_i8_generic_sys(i8 %val, ptr %addr) {
+; CHECK-LABEL: monotonic_i8_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [monotonic_i8_generic_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_generic_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr %addr syncscope("") monotonic, align 1
+ ret void
+}
+
+define void @monotonic_i8_generic_cta(i8 %val, ptr %addr) {
+; CHECK-LABEL: monotonic_i8_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [monotonic_i8_generic_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_generic_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr %addr syncscope("block") monotonic, align 1
+ ret void
+}
+
+define void @monotonic_i8_generic_cluster(i8 %val, ptr %addr) {
+; CHECK-LABEL: monotonic_i8_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [monotonic_i8_generic_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_generic_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cluster.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr %addr syncscope("cluster") monotonic, align 1
+ ret void
+}
+
+define void @monotonic_i8_generic_gpu(i8 %val, ptr %addr) {
+; CHECK-LABEL: monotonic_i8_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [monotonic_i8_generic_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_generic_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr %addr syncscope("device") monotonic, align 1
+ ret void
+}
+
+define void @monotonic_i8_generic_sys_volatile(i8 %val, ptr %addr) {
+; CHECK-LABEL: monotonic_i8_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [monotonic_i8_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_generic_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr %addr syncscope("") monotonic, align 1
+ ret void
+}
+
+define void @monotonic_i8_generic_cta_volatile(i8 %val, ptr %addr) {
+; CHECK-LABEL: monotonic_i8_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [monotonic_i8_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_generic_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr %addr syncscope("block") monotonic, align 1
+ ret void
+}
+
+define void @monotonic_i8_generic_cluster_volatile(i8 %val, ptr %addr) {
+; CHECK-LABEL: monotonic_i8_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [monotonic_i8_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_generic_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr %addr syncscope("cluster") monotonic, align 1
+ ret void
+}
+
+define void @monotonic_i8_generic_gpu_volatile(i8 %val, ptr %addr) {
+; CHECK-LABEL: monotonic_i8_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [monotonic_i8_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_generic_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr %addr syncscope("device") monotonic, align 1
+ ret void
+}
+
+define void @monotonic_i16_generic_sys(i16 %val, ptr %addr) {
+; CHECK-LABEL: monotonic_i16_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_i16_generic_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_generic_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr %addr syncscope("") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_i16_generic_cta(i16 %val, ptr %addr) {
+; CHECK-LABEL: monotonic_i16_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_i16_generic_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_generic_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr %addr syncscope("block") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_i16_generic_cluster(i16 %val, ptr %addr) {
+; CHECK-LABEL: monotonic_i16_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_i16_generic_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_generic_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cluster.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr %addr syncscope("cluster") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_i16_generic_gpu(i16 %val, ptr %addr) {
+; CHECK-LABEL: monotonic_i16_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_i16_generic_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_generic_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr %addr syncscope("device") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_i16_generic_sys_volatile(i16 %val, ptr %addr) {
+; CHECK-LABEL: monotonic_i16_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_i16_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_generic_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr %addr syncscope("") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_i16_generic_cta_volatile(i16 %val, ptr %addr) {
+; CHECK-LABEL: monotonic_i16_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_i16_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_generic_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr %addr syncscope("block") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_i16_generic_cluster_volatile(i16 %val, ptr %addr) {
+; CHECK-LABEL: monotonic_i16_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_i16_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_generic_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr %addr syncscope("cluster") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_i16_generic_gpu_volatile(i16 %val, ptr %addr) {
+; CHECK-LABEL: monotonic_i16_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_i16_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_generic_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr %addr syncscope("device") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_i32_generic_sys(i32 %val, ptr %addr) {
+; CHECK-LABEL: monotonic_i32_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_i32_generic_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_generic_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr %addr syncscope("") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_i32_generic_cta(i32 %val, ptr %addr) {
+; CHECK-LABEL: monotonic_i32_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_i32_generic_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_generic_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr %addr syncscope("block") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_i32_generic_cluster(i32 %val, ptr %addr) {
+; CHECK-LABEL: monotonic_i32_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_i32_generic_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_generic_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cluster.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr %addr syncscope("cluster") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_i32_generic_gpu(i32 %val, ptr %addr) {
+; CHECK-LABEL: monotonic_i32_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_i32_generic_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_generic_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr %addr syncscope("device") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_i32_generic_sys_volatile(i32 %val, ptr %addr) {
+; CHECK-LABEL: monotonic_i32_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_i32_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_generic_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr %addr syncscope("") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_i32_generic_cta_volatile(i32 %val, ptr %addr) {
+; CHECK-LABEL: monotonic_i32_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_i32_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_generic_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr %addr syncscope("block") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_i32_generic_cluster_volatile(i32 %val, ptr %addr) {
+; CHECK-LABEL: monotonic_i32_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_i32_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_generic_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr %addr syncscope("cluster") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_i32_generic_gpu_volatile(i32 %val, ptr %addr) {
+; CHECK-LABEL: monotonic_i32_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_i32_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_generic_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr %addr syncscope("device") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_i64_generic_sys(i64 %val, ptr %addr) {
+; CHECK-LABEL: monotonic_i64_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_generic_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_i64_generic_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr %addr syncscope("") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_i64_generic_cta(i64 %val, ptr %addr) {
+; CHECK-LABEL: monotonic_i64_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_generic_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_i64_generic_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr %addr syncscope("block") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_i64_generic_cluster(i64 %val, ptr %addr) {
+; CHECK-LABEL: monotonic_i64_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_generic_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_i64_generic_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cluster.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr %addr syncscope("cluster") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_i64_generic_gpu(i64 %val, ptr %addr) {
+; CHECK-LABEL: monotonic_i64_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_generic_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_i64_generic_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr %addr syncscope("device") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_i64_generic_sys_volatile(i64 %val, ptr %addr) {
+; CHECK-LABEL: monotonic_i64_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_i64_generic_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr %addr syncscope("") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_i64_generic_cta_volatile(i64 %val, ptr %addr) {
+; CHECK-LABEL: monotonic_i64_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_i64_generic_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr %addr syncscope("block") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_i64_generic_cluster_volatile(i64 %val, ptr %addr) {
+; CHECK-LABEL: monotonic_i64_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_i64_generic_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr %addr syncscope("cluster") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_i64_generic_gpu_volatile(i64 %val, ptr %addr) {
+; CHECK-LABEL: monotonic_i64_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_i64_generic_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr %addr syncscope("device") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_i128_generic_sys(i128 %val, ptr %addr) {
+; CHECK-LABEL: monotonic_i128_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [monotonic_i128_generic_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd3, [monotonic_i128_generic_sys_param_1];
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 amt, dst;
+; CHECK-NEXT: mov.b128 amt, {%rd1, %rd2};
+; CHECK-NEXT: atom.relaxed.sys.exch.b128 dst, [%rd3], amt;
+; CHECK-NEXT: mov.b128 {%rd4, %rd5}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: ret;
+ store atomic i128 %val, ptr %addr syncscope("") monotonic, align 16
+ ret void
+}
+
+define void @monotonic_i128_generic_cta(i128 %val, ptr %addr) {
+; CHECK-LABEL: monotonic_i128_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [monotonic_i128_generic_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd3, [monotonic_i128_generic_cta_param_1];
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 amt, dst;
+; CHECK-NEXT: mov.b128 amt, {%rd1, %rd2};
+; CHECK-NEXT: atom.relaxed.cta.exch.b128 dst, [%rd3], amt;
+; CHECK-NEXT: mov.b128 {%rd4, %rd5}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: ret;
+ store atomic i128 %val, ptr %addr syncscope("block") monotonic, align 16
+ ret void
+}
+
+define void @monotonic_i128_generic_cluster(i128 %val, ptr %addr) {
+; CHECK-LABEL: monotonic_i128_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [monotonic_i128_generic_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd3, [monotonic_i128_generic_cluster_param_1];
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 amt, dst;
+; CHECK-NEXT: mov.b128 amt, {%rd1, %rd2};
+; CHECK-NEXT: atom.relaxed.cluster.exch.b128 dst, [%rd3], amt;
+; CHECK-NEXT: mov.b128 {%rd4, %rd5}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: ret;
+ store atomic i128 %val, ptr %addr syncscope("cluster") monotonic, align 16
+ ret void
+}
+
+define void @monotonic_i128_generic_gpu(i128 %val, ptr %addr) {
+; CHECK-LABEL: monotonic_i128_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [monotonic_i128_generic_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd3, [monotonic_i128_generic_gpu_param_1];
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 amt, dst;
+; CHECK-NEXT: mov.b128 amt, {%rd1, %rd2};
+; CHECK-NEXT: atom.relaxed.gpu.exch.b128 dst, [%rd3], amt;
+; CHECK-NEXT: mov.b128 {%rd4, %rd5}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: ret;
+ store atomic i128 %val, ptr %addr syncscope("device") monotonic, align 16
+ ret void
+}
+
+define void @monotonic_i128_generic_sys_volatile(i128 %val, ptr %addr) {
+; CHECK-LABEL: monotonic_i128_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [monotonic_i128_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd3, [monotonic_i128_generic_sys_volatile_param_1];
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 amt, dst;
+; CHECK-NEXT: mov.b128 amt, {%rd1, %rd2};
+; CHECK-NEXT: atom.relaxed.sys.exch.b128 dst, [%rd3], amt;
+; CHECK-NEXT: mov.b128 {%rd4, %rd5}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: ret;
+ store atomic volatile i128 %val, ptr %addr syncscope("") monotonic, align 16
+ ret void
+}
+
+define void @monotonic_i128_generic_cta_volatile(i128 %val, ptr %addr) {
+; CHECK-LABEL: monotonic_i128_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [monotonic_i128_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd3, [monotonic_i128_generic_cta_volatile_param_1];
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 amt, dst;
+; CHECK-NEXT: mov.b128 amt, {%rd1, %rd2};
+; CHECK-NEXT: atom.relaxed.cta.exch.b128 dst, [%rd3], amt;
+; CHECK-NEXT: mov.b128 {%rd4, %rd5}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: ret;
+ store atomic volatile i128 %val, ptr %addr syncscope("block") monotonic, align 16
+ ret void
+}
+
+define void @monotonic_i128_generic_cluster_volatile(i128 %val, ptr %addr) {
+; CHECK-LABEL: monotonic_i128_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [monotonic_i128_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd3, [monotonic_i128_generic_cluster_volatile_param_1];
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 amt, dst;
+; CHECK-NEXT: mov.b128 amt, {%rd1, %rd2};
+; CHECK-NEXT: atom.relaxed.cluster.exch.b128 dst, [%rd3], amt;
+; CHECK-NEXT: mov.b128 {%rd4, %rd5}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: ret;
+ store atomic volatile i128 %val, ptr %addr syncscope("cluster") monotonic, align 16
+ ret void
+}
+
+define void @monotonic_i128_generic_gpu_volatile(i128 %val, ptr %addr) {
+; CHECK-LABEL: monotonic_i128_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [monotonic_i128_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd3, [monotonic_i128_generic_gpu_volatile_param_1];
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 amt, dst;
+; CHECK-NEXT: mov.b128 amt, {%rd1, %rd2};
+; CHECK-NEXT: atom.relaxed.gpu.exch.b128 dst, [%rd3], amt;
+; CHECK-NEXT: mov.b128 {%rd4, %rd5}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: ret;
+ store atomic volatile i128 %val, ptr %addr syncscope("device") monotonic, align 16
+ ret void
+}
+
+define void @monotonic_half_generic_sys(half %val, ptr %addr) {
+; CHECK-LABEL: monotonic_half_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_half_generic_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_generic_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr %addr syncscope("") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_half_generic_cta(half %val, ptr %addr) {
+; CHECK-LABEL: monotonic_half_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_half_generic_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_generic_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr %addr syncscope("block") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_half_generic_cluster(half %val, ptr %addr) {
+; CHECK-LABEL: monotonic_half_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_half_generic_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_generic_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cluster.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr %addr syncscope("cluster") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_half_generic_gpu(half %val, ptr %addr) {
+; CHECK-LABEL: monotonic_half_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_half_generic_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_generic_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr %addr syncscope("device") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_half_generic_sys_volatile(half %val, ptr %addr) {
+; CHECK-LABEL: monotonic_half_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_half_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_generic_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr %addr syncscope("") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_half_generic_cta_volatile(half %val, ptr %addr) {
+; CHECK-LABEL: monotonic_half_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_half_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_generic_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr %addr syncscope("block") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_half_generic_cluster_volatile(half %val, ptr %addr) {
+; CHECK-LABEL: monotonic_half_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_half_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_generic_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr %addr syncscope("cluster") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_half_generic_gpu_volatile(half %val, ptr %addr) {
+; CHECK-LABEL: monotonic_half_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_half_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_generic_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr %addr syncscope("device") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_bfloat_generic_sys(bfloat %val, ptr %addr) {
+; CHECK-LABEL: monotonic_bfloat_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_bfloat_generic_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_generic_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr %addr syncscope("") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_bfloat_generic_cta(bfloat %val, ptr %addr) {
+; CHECK-LABEL: monotonic_bfloat_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_bfloat_generic_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_generic_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr %addr syncscope("block") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_bfloat_generic_cluster(bfloat %val, ptr %addr) {
+; CHECK-LABEL: monotonic_bfloat_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_bfloat_generic_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_generic_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cluster.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr %addr syncscope("cluster") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_bfloat_generic_gpu(bfloat %val, ptr %addr) {
+; CHECK-LABEL: monotonic_bfloat_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_bfloat_generic_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_generic_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr %addr syncscope("device") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_bfloat_generic_sys_volatile(bfloat %val, ptr %addr) {
+; CHECK-LABEL: monotonic_bfloat_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_bfloat_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_generic_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr %addr syncscope("") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_bfloat_generic_cta_volatile(bfloat %val, ptr %addr) {
+; CHECK-LABEL: monotonic_bfloat_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_bfloat_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_generic_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr %addr syncscope("block") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_bfloat_generic_cluster_volatile(bfloat %val, ptr %addr) {
+; CHECK-LABEL: monotonic_bfloat_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_bfloat_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_generic_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr %addr syncscope("cluster") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_bfloat_generic_gpu_volatile(bfloat %val, ptr %addr) {
+; CHECK-LABEL: monotonic_bfloat_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_bfloat_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_generic_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr %addr syncscope("device") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_float_generic_sys(float %val, ptr %addr) {
+; CHECK-LABEL: monotonic_float_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_float_generic_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_generic_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr %addr syncscope("") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_float_generic_cta(float %val, ptr %addr) {
+; CHECK-LABEL: monotonic_float_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_float_generic_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_generic_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr %addr syncscope("block") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_float_generic_cluster(float %val, ptr %addr) {
+; CHECK-LABEL: monotonic_float_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_float_generic_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_generic_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cluster.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr %addr syncscope("cluster") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_float_generic_gpu(float %val, ptr %addr) {
+; CHECK-LABEL: monotonic_float_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_float_generic_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_generic_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr %addr syncscope("device") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_float_generic_sys_volatile(float %val, ptr %addr) {
+; CHECK-LABEL: monotonic_float_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_float_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_generic_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr %addr syncscope("") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_float_generic_cta_volatile(float %val, ptr %addr) {
+; CHECK-LABEL: monotonic_float_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_float_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_generic_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr %addr syncscope("block") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_float_generic_cluster_volatile(float %val, ptr %addr) {
+; CHECK-LABEL: monotonic_float_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_float_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_generic_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr %addr syncscope("cluster") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_float_generic_gpu_volatile(float %val, ptr %addr) {
+; CHECK-LABEL: monotonic_float_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_float_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_generic_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr %addr syncscope("device") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_double_generic_sys(double %val, ptr %addr) {
+; CHECK-LABEL: monotonic_double_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_generic_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_double_generic_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr %addr syncscope("") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_double_generic_cta(double %val, ptr %addr) {
+; CHECK-LABEL: monotonic_double_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_generic_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_double_generic_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr %addr syncscope("block") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_double_generic_cluster(double %val, ptr %addr) {
+; CHECK-LABEL: monotonic_double_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_generic_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_double_generic_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cluster.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr %addr syncscope("cluster") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_double_generic_gpu(double %val, ptr %addr) {
+; CHECK-LABEL: monotonic_double_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_generic_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_double_generic_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr %addr syncscope("device") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_double_generic_sys_volatile(double %val, ptr %addr) {
+; CHECK-LABEL: monotonic_double_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_double_generic_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr %addr syncscope("") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_double_generic_cta_volatile(double %val, ptr %addr) {
+; CHECK-LABEL: monotonic_double_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_double_generic_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr %addr syncscope("block") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_double_generic_cluster_volatile(double %val, ptr %addr) {
+; CHECK-LABEL: monotonic_double_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_double_generic_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr %addr syncscope("cluster") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_double_generic_gpu_volatile(double %val, ptr %addr) {
+; CHECK-LABEL: monotonic_double_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_double_generic_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr %addr syncscope("device") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_i8_global_sys(i8 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i8_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [monotonic_i8_global_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_global_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.global.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr addrspace(1) %addr syncscope("") monotonic, align 1
+ ret void
+}
+
+define void @monotonic_i8_global_cta(i8 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i8_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [monotonic_i8_global_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_global_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.global.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr addrspace(1) %addr syncscope("block") monotonic, align 1
+ ret void
+}
+
+define void @monotonic_i8_global_cluster(i8 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i8_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [monotonic_i8_global_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_global_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cluster.global.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr addrspace(1) %addr syncscope("cluster") monotonic, align 1
+ ret void
+}
+
+define void @monotonic_i8_global_gpu(i8 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i8_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [monotonic_i8_global_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_global_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.global.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr addrspace(1) %addr syncscope("device") monotonic, align 1
+ ret void
+}
+
+define void @monotonic_i8_global_sys_volatile(i8 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i8_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [monotonic_i8_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_global_sys_volatile_param_1];
+; CHECK-NEXT: st.mmio.relaxed.sys.global.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr addrspace(1) %addr syncscope("") monotonic, align 1
+ ret void
+}
+
+define void @monotonic_i8_global_cta_volatile(i8 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i8_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [monotonic_i8_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_global_cta_volatile_param_1];
+; CHECK-NEXT: st.mmio.relaxed.sys.global.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr addrspace(1) %addr syncscope("block") monotonic, align 1
+ ret void
+}
+
+define void @monotonic_i8_global_cluster_volatile(i8 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i8_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [monotonic_i8_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_global_cluster_volatile_param_1];
+; CHECK-NEXT: st.mmio.relaxed.sys.global.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr addrspace(1) %addr syncscope("cluster") monotonic, align 1
+ ret void
+}
+
+define void @monotonic_i8_global_gpu_volatile(i8 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i8_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [monotonic_i8_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_global_gpu_volatile_param_1];
+; CHECK-NEXT: st.mmio.relaxed.sys.global.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr addrspace(1) %addr syncscope("device") monotonic, align 1
+ ret void
+}
+
+define void @monotonic_i16_global_sys(i16 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i16_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_i16_global_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_global_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr addrspace(1) %addr syncscope("") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_i16_global_cta(i16 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i16_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_i16_global_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_global_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr addrspace(1) %addr syncscope("block") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_i16_global_cluster(i16 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i16_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_i16_global_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_global_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cluster.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr addrspace(1) %addr syncscope("cluster") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_i16_global_gpu(i16 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i16_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_i16_global_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_global_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr addrspace(1) %addr syncscope("device") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_i16_global_sys_volatile(i16 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i16_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_i16_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_global_sys_volatile_param_1];
+; CHECK-NEXT: st.mmio.relaxed.sys.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr addrspace(1) %addr syncscope("") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_i16_global_cta_volatile(i16 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i16_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_i16_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_global_cta_volatile_param_1];
+; CHECK-NEXT: st.mmio.relaxed.sys.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr addrspace(1) %addr syncscope("block") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_i16_global_cluster_volatile(i16 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i16_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_i16_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_global_cluster_volatile_param_1];
+; CHECK-NEXT: st.mmio.relaxed.sys.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr addrspace(1) %addr syncscope("cluster") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_i16_global_gpu_volatile(i16 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i16_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_i16_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_global_gpu_volatile_param_1];
+; CHECK-NEXT: st.mmio.relaxed.sys.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr addrspace(1) %addr syncscope("device") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_i32_global_sys(i32 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i32_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_i32_global_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_global_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr addrspace(1) %addr syncscope("") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_i32_global_cta(i32 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i32_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_i32_global_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_global_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr addrspace(1) %addr syncscope("block") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_i32_global_cluster(i32 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i32_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_i32_global_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_global_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cluster.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr addrspace(1) %addr syncscope("cluster") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_i32_global_gpu(i32 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i32_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_i32_global_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_global_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr addrspace(1) %addr syncscope("device") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_i32_global_sys_volatile(i32 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i32_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_i32_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_global_sys_volatile_param_1];
+; CHECK-NEXT: st.mmio.relaxed.sys.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr addrspace(1) %addr syncscope("") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_i32_global_cta_volatile(i32 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i32_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_i32_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_global_cta_volatile_param_1];
+; CHECK-NEXT: st.mmio.relaxed.sys.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr addrspace(1) %addr syncscope("block") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_i32_global_cluster_volatile(i32 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i32_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_i32_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_global_cluster_volatile_param_1];
+; CHECK-NEXT: st.mmio.relaxed.sys.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr addrspace(1) %addr syncscope("cluster") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_i32_global_gpu_volatile(i32 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i32_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_i32_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_global_gpu_volatile_param_1];
+; CHECK-NEXT: st.mmio.relaxed.sys.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr addrspace(1) %addr syncscope("device") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_i64_global_sys(i64 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i64_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_global_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_i64_global_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr addrspace(1) %addr syncscope("") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_i64_global_cta(i64 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i64_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_global_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_i64_global_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr addrspace(1) %addr syncscope("block") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_i64_global_cluster(i64 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i64_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_global_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_i64_global_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cluster.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr addrspace(1) %addr syncscope("cluster") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_i64_global_gpu(i64 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i64_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_global_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_i64_global_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr addrspace(1) %addr syncscope("device") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_i64_global_sys_volatile(i64 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i64_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_i64_global_sys_volatile_param_1];
+; CHECK-NEXT: st.mmio.relaxed.sys.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr addrspace(1) %addr syncscope("") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_i64_global_cta_volatile(i64 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i64_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_i64_global_cta_volatile_param_1];
+; CHECK-NEXT: st.mmio.relaxed.sys.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr addrspace(1) %addr syncscope("block") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_i64_global_cluster_volatile(i64 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i64_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_i64_global_cluster_volatile_param_1];
+; CHECK-NEXT: st.mmio.relaxed.sys.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr addrspace(1) %addr syncscope("cluster") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_i64_global_gpu_volatile(i64 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i64_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_i64_global_gpu_volatile_param_1];
+; CHECK-NEXT: st.mmio.relaxed.sys.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr addrspace(1) %addr syncscope("device") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_i128_global_sys(i128 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i128_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [monotonic_i128_global_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd3, [monotonic_i128_global_sys_param_1];
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 amt, dst;
+; CHECK-NEXT: mov.b128 amt, {%rd1, %rd2};
+; CHECK-NEXT: atom.relaxed.sys.global.exch.b128 dst, [%rd3], amt;
+; CHECK-NEXT: mov.b128 {%rd4, %rd5}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: ret;
+ store atomic i128 %val, ptr addrspace(1) %addr syncscope("") monotonic, align 16
+ ret void
+}
+
+define void @monotonic_i128_global_cta(i128 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i128_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [monotonic_i128_global_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd3, [monotonic_i128_global_cta_param_1];
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 amt, dst;
+; CHECK-NEXT: mov.b128 amt, {%rd1, %rd2};
+; CHECK-NEXT: atom.relaxed.cta.global.exch.b128 dst, [%rd3], amt;
+; CHECK-NEXT: mov.b128 {%rd4, %rd5}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: ret;
+ store atomic i128 %val, ptr addrspace(1) %addr syncscope("block") monotonic, align 16
+ ret void
+}
+
+define void @monotonic_i128_global_cluster(i128 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i128_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [monotonic_i128_global_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd3, [monotonic_i128_global_cluster_param_1];
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 amt, dst;
+; CHECK-NEXT: mov.b128 amt, {%rd1, %rd2};
+; CHECK-NEXT: atom.relaxed.cluster.global.exch.b128 dst, [%rd3], amt;
+; CHECK-NEXT: mov.b128 {%rd4, %rd5}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: ret;
+ store atomic i128 %val, ptr addrspace(1) %addr syncscope("cluster") monotonic, align 16
+ ret void
+}
+
+define void @monotonic_i128_global_gpu(i128 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i128_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [monotonic_i128_global_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd3, [monotonic_i128_global_gpu_param_1];
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 amt, dst;
+; CHECK-NEXT: mov.b128 amt, {%rd1, %rd2};
+; CHECK-NEXT: atom.relaxed.gpu.global.exch.b128 dst, [%rd3], amt;
+; CHECK-NEXT: mov.b128 {%rd4, %rd5}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: ret;
+ store atomic i128 %val, ptr addrspace(1) %addr syncscope("device") monotonic, align 16
+ ret void
+}
+
+define void @monotonic_i128_global_sys_volatile(i128 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i128_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [monotonic_i128_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd3, [monotonic_i128_global_sys_volatile_param_1];
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 amt, dst;
+; CHECK-NEXT: mov.b128 amt, {%rd1, %rd2};
+; CHECK-NEXT: atom.relaxed.sys.global.exch.b128 dst, [%rd3], amt;
+; CHECK-NEXT: mov.b128 {%rd4, %rd5}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: ret;
+ store atomic volatile i128 %val, ptr addrspace(1) %addr syncscope("") monotonic, align 16
+ ret void
+}
+
+define void @monotonic_i128_global_cta_volatile(i128 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i128_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [monotonic_i128_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd3, [monotonic_i128_global_cta_volatile_param_1];
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 amt, dst;
+; CHECK-NEXT: mov.b128 amt, {%rd1, %rd2};
+; CHECK-NEXT: atom.relaxed.cta.global.exch.b128 dst, [%rd3], amt;
+; CHECK-NEXT: mov.b128 {%rd4, %rd5}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: ret;
+ store atomic volatile i128 %val, ptr addrspace(1) %addr syncscope("block") monotonic, align 16
+ ret void
+}
+
+define void @monotonic_i128_global_cluster_volatile(i128 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i128_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [monotonic_i128_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd3, [monotonic_i128_global_cluster_volatile_param_1];
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 amt, dst;
+; CHECK-NEXT: mov.b128 amt, {%rd1, %rd2};
+; CHECK-NEXT: atom.relaxed.cluster.global.exch.b128 dst, [%rd3], amt;
+; CHECK-NEXT: mov.b128 {%rd4, %rd5}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: ret;
+ store atomic volatile i128 %val, ptr addrspace(1) %addr syncscope("cluster") monotonic, align 16
+ ret void
+}
+
+define void @monotonic_i128_global_gpu_volatile(i128 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_i128_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [monotonic_i128_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd3, [monotonic_i128_global_gpu_volatile_param_1];
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 amt, dst;
+; CHECK-NEXT: mov.b128 amt, {%rd1, %rd2};
+; CHECK-NEXT: atom.relaxed.gpu.global.exch.b128 dst, [%rd3], amt;
+; CHECK-NEXT: mov.b128 {%rd4, %rd5}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: ret;
+ store atomic volatile i128 %val, ptr addrspace(1) %addr syncscope("device") monotonic, align 16
+ ret void
+}
+
+define void @monotonic_half_global_sys(half %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_half_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_half_global_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_global_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr addrspace(1) %addr syncscope("") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_half_global_cta(half %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_half_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_half_global_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_global_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr addrspace(1) %addr syncscope("block") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_half_global_cluster(half %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_half_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_half_global_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_global_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cluster.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr addrspace(1) %addr syncscope("cluster") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_half_global_gpu(half %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_half_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_half_global_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_global_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr addrspace(1) %addr syncscope("device") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_half_global_sys_volatile(half %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_half_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_half_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_global_sys_volatile_param_1];
+; CHECK-NEXT: st.mmio.relaxed.sys.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr addrspace(1) %addr syncscope("") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_half_global_cta_volatile(half %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_half_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_half_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_global_cta_volatile_param_1];
+; CHECK-NEXT: st.mmio.relaxed.sys.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr addrspace(1) %addr syncscope("block") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_half_global_cluster_volatile(half %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_half_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_half_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_global_cluster_volatile_param_1];
+; CHECK-NEXT: st.mmio.relaxed.sys.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr addrspace(1) %addr syncscope("cluster") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_half_global_gpu_volatile(half %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_half_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_half_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_global_gpu_volatile_param_1];
+; CHECK-NEXT: st.mmio.relaxed.sys.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr addrspace(1) %addr syncscope("device") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_bfloat_global_sys(bfloat %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_bfloat_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_bfloat_global_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_global_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr addrspace(1) %addr syncscope("") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_bfloat_global_cta(bfloat %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_bfloat_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_bfloat_global_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_global_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr addrspace(1) %addr syncscope("block") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_bfloat_global_cluster(bfloat %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_bfloat_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_bfloat_global_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_global_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cluster.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr addrspace(1) %addr syncscope("cluster") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_bfloat_global_gpu(bfloat %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_bfloat_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_bfloat_global_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_global_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr addrspace(1) %addr syncscope("device") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_bfloat_global_sys_volatile(bfloat %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_bfloat_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_bfloat_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_global_sys_volatile_param_1];
+; CHECK-NEXT: st.mmio.relaxed.sys.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr addrspace(1) %addr syncscope("") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_bfloat_global_cta_volatile(bfloat %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_bfloat_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_bfloat_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_global_cta_volatile_param_1];
+; CHECK-NEXT: st.mmio.relaxed.sys.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr addrspace(1) %addr syncscope("block") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_bfloat_global_cluster_volatile(bfloat %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_bfloat_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_bfloat_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_global_cluster_volatile_param_1];
+; CHECK-NEXT: st.mmio.relaxed.sys.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr addrspace(1) %addr syncscope("cluster") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_bfloat_global_gpu_volatile(bfloat %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_bfloat_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_bfloat_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_global_gpu_volatile_param_1];
+; CHECK-NEXT: st.mmio.relaxed.sys.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr addrspace(1) %addr syncscope("device") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_float_global_sys(float %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_float_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_float_global_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_global_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr addrspace(1) %addr syncscope("") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_float_global_cta(float %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_float_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_float_global_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_global_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr addrspace(1) %addr syncscope("block") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_float_global_cluster(float %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_float_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_float_global_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_global_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cluster.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr addrspace(1) %addr syncscope("cluster") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_float_global_gpu(float %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_float_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_float_global_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_global_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr addrspace(1) %addr syncscope("device") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_float_global_sys_volatile(float %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_float_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_float_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_global_sys_volatile_param_1];
+; CHECK-NEXT: st.mmio.relaxed.sys.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr addrspace(1) %addr syncscope("") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_float_global_cta_volatile(float %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_float_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_float_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_global_cta_volatile_param_1];
+; CHECK-NEXT: st.mmio.relaxed.sys.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr addrspace(1) %addr syncscope("block") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_float_global_cluster_volatile(float %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_float_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_float_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_global_cluster_volatile_param_1];
+; CHECK-NEXT: st.mmio.relaxed.sys.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr addrspace(1) %addr syncscope("cluster") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_float_global_gpu_volatile(float %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_float_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_float_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_global_gpu_volatile_param_1];
+; CHECK-NEXT: st.mmio.relaxed.sys.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr addrspace(1) %addr syncscope("device") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_double_global_sys(double %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_double_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_global_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_double_global_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr addrspace(1) %addr syncscope("") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_double_global_cta(double %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_double_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_global_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_double_global_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr addrspace(1) %addr syncscope("block") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_double_global_cluster(double %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_double_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_global_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_double_global_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cluster.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr addrspace(1) %addr syncscope("cluster") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_double_global_gpu(double %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_double_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_global_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_double_global_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr addrspace(1) %addr syncscope("device") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_double_global_sys_volatile(double %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_double_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_double_global_sys_volatile_param_1];
+; CHECK-NEXT: st.mmio.relaxed.sys.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr addrspace(1) %addr syncscope("") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_double_global_cta_volatile(double %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_double_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_double_global_cta_volatile_param_1];
+; CHECK-NEXT: st.mmio.relaxed.sys.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr addrspace(1) %addr syncscope("block") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_double_global_cluster_volatile(double %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_double_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_double_global_cluster_volatile_param_1];
+; CHECK-NEXT: st.mmio.relaxed.sys.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr addrspace(1) %addr syncscope("cluster") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_double_global_gpu_volatile(double %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: monotonic_double_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_double_global_gpu_volatile_param_1];
+; CHECK-NEXT: st.mmio.relaxed.sys.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr addrspace(1) %addr syncscope("device") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_i8_shared_sys(i8 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i8_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [monotonic_i8_shared_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_shared_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.shared.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr addrspace(3) %addr syncscope("") monotonic, align 1
+ ret void
+}
+
+define void @monotonic_i8_shared_cta(i8 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i8_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [monotonic_i8_shared_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_shared_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.shared.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr addrspace(3) %addr syncscope("block") monotonic, align 1
+ ret void
+}
+
+define void @monotonic_i8_shared_cluster(i8 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i8_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [monotonic_i8_shared_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_shared_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cluster.shared.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr addrspace(3) %addr syncscope("cluster") monotonic, align 1
+ ret void
+}
+
+define void @monotonic_i8_shared_gpu(i8 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i8_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [monotonic_i8_shared_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_shared_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.shared.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr addrspace(3) %addr syncscope("device") monotonic, align 1
+ ret void
+}
+
+define void @monotonic_i8_shared_sys_volatile(i8 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i8_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [monotonic_i8_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_shared_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr addrspace(3) %addr syncscope("") monotonic, align 1
+ ret void
+}
+
+define void @monotonic_i8_shared_cta_volatile(i8 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i8_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [monotonic_i8_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_shared_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr addrspace(3) %addr syncscope("block") monotonic, align 1
+ ret void
+}
+
+define void @monotonic_i8_shared_cluster_volatile(i8 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i8_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [monotonic_i8_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_shared_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr addrspace(3) %addr syncscope("cluster") monotonic, align 1
+ ret void
+}
+
+define void @monotonic_i8_shared_gpu_volatile(i8 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i8_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [monotonic_i8_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i8_shared_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr addrspace(3) %addr syncscope("device") monotonic, align 1
+ ret void
+}
+
+define void @monotonic_i16_shared_sys(i16 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i16_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_i16_shared_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_shared_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr addrspace(3) %addr syncscope("") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_i16_shared_cta(i16 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i16_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_i16_shared_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_shared_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr addrspace(3) %addr syncscope("block") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_i16_shared_cluster(i16 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i16_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_i16_shared_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_shared_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cluster.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr addrspace(3) %addr syncscope("cluster") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_i16_shared_gpu(i16 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i16_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_i16_shared_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_shared_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr addrspace(3) %addr syncscope("device") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_i16_shared_sys_volatile(i16 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i16_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_i16_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_shared_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr addrspace(3) %addr syncscope("") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_i16_shared_cta_volatile(i16 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i16_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_i16_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_shared_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr addrspace(3) %addr syncscope("block") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_i16_shared_cluster_volatile(i16 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i16_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_i16_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_shared_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr addrspace(3) %addr syncscope("cluster") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_i16_shared_gpu_volatile(i16 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i16_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_i16_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i16_shared_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr addrspace(3) %addr syncscope("device") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_i32_shared_sys(i32 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i32_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_i32_shared_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_shared_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr addrspace(3) %addr syncscope("") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_i32_shared_cta(i32 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i32_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_i32_shared_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_shared_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr addrspace(3) %addr syncscope("block") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_i32_shared_cluster(i32 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i32_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_i32_shared_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_shared_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cluster.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr addrspace(3) %addr syncscope("cluster") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_i32_shared_gpu(i32 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i32_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_i32_shared_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_shared_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr addrspace(3) %addr syncscope("device") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_i32_shared_sys_volatile(i32 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i32_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_i32_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_shared_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr addrspace(3) %addr syncscope("") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_i32_shared_cta_volatile(i32 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i32_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_i32_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_shared_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr addrspace(3) %addr syncscope("block") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_i32_shared_cluster_volatile(i32 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i32_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_i32_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_shared_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr addrspace(3) %addr syncscope("cluster") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_i32_shared_gpu_volatile(i32 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i32_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_i32_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i32_shared_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr addrspace(3) %addr syncscope("device") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_i64_shared_sys(i64 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i64_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_shared_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_i64_shared_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr addrspace(3) %addr syncscope("") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_i64_shared_cta(i64 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i64_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_shared_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_i64_shared_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr addrspace(3) %addr syncscope("block") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_i64_shared_cluster(i64 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i64_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_shared_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_i64_shared_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cluster.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr addrspace(3) %addr syncscope("cluster") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_i64_shared_gpu(i64 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i64_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_shared_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_i64_shared_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr addrspace(3) %addr syncscope("device") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_i64_shared_sys_volatile(i64 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i64_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_i64_shared_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr addrspace(3) %addr syncscope("") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_i64_shared_cta_volatile(i64 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i64_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_i64_shared_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr addrspace(3) %addr syncscope("block") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_i64_shared_cluster_volatile(i64 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i64_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_i64_shared_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr addrspace(3) %addr syncscope("cluster") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_i64_shared_gpu_volatile(i64 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i64_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_i64_shared_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr addrspace(3) %addr syncscope("device") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_i128_shared_sys(i128 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i128_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [monotonic_i128_shared_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd3, [monotonic_i128_shared_sys_param_1];
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 amt, dst;
+; CHECK-NEXT: mov.b128 amt, {%rd1, %rd2};
+; CHECK-NEXT: atom.relaxed.sys.shared.exch.b128 dst, [%rd3], amt;
+; CHECK-NEXT: mov.b128 {%rd4, %rd5}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: ret;
+ store atomic i128 %val, ptr addrspace(3) %addr syncscope("") monotonic, align 16
+ ret void
+}
+
+define void @monotonic_i128_shared_cta(i128 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i128_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [monotonic_i128_shared_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd3, [monotonic_i128_shared_cta_param_1];
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 amt, dst;
+; CHECK-NEXT: mov.b128 amt, {%rd1, %rd2};
+; CHECK-NEXT: atom.relaxed.cta.shared.exch.b128 dst, [%rd3], amt;
+; CHECK-NEXT: mov.b128 {%rd4, %rd5}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: ret;
+ store atomic i128 %val, ptr addrspace(3) %addr syncscope("block") monotonic, align 16
+ ret void
+}
+
+define void @monotonic_i128_shared_cluster(i128 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i128_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [monotonic_i128_shared_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd3, [monotonic_i128_shared_cluster_param_1];
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 amt, dst;
+; CHECK-NEXT: mov.b128 amt, {%rd1, %rd2};
+; CHECK-NEXT: atom.relaxed.cluster.shared.exch.b128 dst, [%rd3], amt;
+; CHECK-NEXT: mov.b128 {%rd4, %rd5}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: ret;
+ store atomic i128 %val, ptr addrspace(3) %addr syncscope("cluster") monotonic, align 16
+ ret void
+}
+
+define void @monotonic_i128_shared_gpu(i128 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i128_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [monotonic_i128_shared_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd3, [monotonic_i128_shared_gpu_param_1];
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 amt, dst;
+; CHECK-NEXT: mov.b128 amt, {%rd1, %rd2};
+; CHECK-NEXT: atom.relaxed.gpu.shared.exch.b128 dst, [%rd3], amt;
+; CHECK-NEXT: mov.b128 {%rd4, %rd5}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: ret;
+ store atomic i128 %val, ptr addrspace(3) %addr syncscope("device") monotonic, align 16
+ ret void
+}
+
+define void @monotonic_i128_shared_sys_volatile(i128 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i128_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [monotonic_i128_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd3, [monotonic_i128_shared_sys_volatile_param_1];
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 amt, dst;
+; CHECK-NEXT: mov.b128 amt, {%rd1, %rd2};
+; CHECK-NEXT: atom.relaxed.sys.shared.exch.b128 dst, [%rd3], amt;
+; CHECK-NEXT: mov.b128 {%rd4, %rd5}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: ret;
+ store atomic volatile i128 %val, ptr addrspace(3) %addr syncscope("") monotonic, align 16
+ ret void
+}
+
+define void @monotonic_i128_shared_cta_volatile(i128 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i128_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [monotonic_i128_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd3, [monotonic_i128_shared_cta_volatile_param_1];
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 amt, dst;
+; CHECK-NEXT: mov.b128 amt, {%rd1, %rd2};
+; CHECK-NEXT: atom.relaxed.cta.shared.exch.b128 dst, [%rd3], amt;
+; CHECK-NEXT: mov.b128 {%rd4, %rd5}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: ret;
+ store atomic volatile i128 %val, ptr addrspace(3) %addr syncscope("block") monotonic, align 16
+ ret void
+}
+
+define void @monotonic_i128_shared_cluster_volatile(i128 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i128_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [monotonic_i128_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd3, [monotonic_i128_shared_cluster_volatile_param_1];
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 amt, dst;
+; CHECK-NEXT: mov.b128 amt, {%rd1, %rd2};
+; CHECK-NEXT: atom.relaxed.cluster.shared.exch.b128 dst, [%rd3], amt;
+; CHECK-NEXT: mov.b128 {%rd4, %rd5}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: ret;
+ store atomic volatile i128 %val, ptr addrspace(3) %addr syncscope("cluster") monotonic, align 16
+ ret void
+}
+
+define void @monotonic_i128_shared_gpu_volatile(i128 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_i128_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [monotonic_i128_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd3, [monotonic_i128_shared_gpu_volatile_param_1];
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 amt, dst;
+; CHECK-NEXT: mov.b128 amt, {%rd1, %rd2};
+; CHECK-NEXT: atom.relaxed.gpu.shared.exch.b128 dst, [%rd3], amt;
+; CHECK-NEXT: mov.b128 {%rd4, %rd5}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: ret;
+ store atomic volatile i128 %val, ptr addrspace(3) %addr syncscope("device") monotonic, align 16
+ ret void
+}
+
+define void @monotonic_half_shared_sys(half %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_half_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_half_shared_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_shared_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr addrspace(3) %addr syncscope("") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_half_shared_cta(half %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_half_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_half_shared_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_shared_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr addrspace(3) %addr syncscope("block") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_half_shared_cluster(half %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_half_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_half_shared_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_shared_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cluster.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr addrspace(3) %addr syncscope("cluster") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_half_shared_gpu(half %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_half_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_half_shared_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_shared_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr addrspace(3) %addr syncscope("device") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_half_shared_sys_volatile(half %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_half_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_half_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_shared_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr addrspace(3) %addr syncscope("") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_half_shared_cta_volatile(half %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_half_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_half_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_shared_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr addrspace(3) %addr syncscope("block") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_half_shared_cluster_volatile(half %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_half_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_half_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_shared_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr addrspace(3) %addr syncscope("cluster") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_half_shared_gpu_volatile(half %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_half_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_half_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_half_shared_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr addrspace(3) %addr syncscope("device") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_bfloat_shared_sys(bfloat %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_bfloat_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_bfloat_shared_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_shared_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr addrspace(3) %addr syncscope("") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_bfloat_shared_cta(bfloat %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_bfloat_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_bfloat_shared_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_shared_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr addrspace(3) %addr syncscope("block") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_bfloat_shared_cluster(bfloat %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_bfloat_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_bfloat_shared_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_shared_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cluster.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr addrspace(3) %addr syncscope("cluster") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_bfloat_shared_gpu(bfloat %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_bfloat_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_bfloat_shared_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_shared_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr addrspace(3) %addr syncscope("device") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_bfloat_shared_sys_volatile(bfloat %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_bfloat_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_bfloat_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_shared_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr addrspace(3) %addr syncscope("") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_bfloat_shared_cta_volatile(bfloat %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_bfloat_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_bfloat_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_shared_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr addrspace(3) %addr syncscope("block") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_bfloat_shared_cluster_volatile(bfloat %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_bfloat_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_bfloat_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_shared_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr addrspace(3) %addr syncscope("cluster") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_bfloat_shared_gpu_volatile(bfloat %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_bfloat_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_bfloat_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_bfloat_shared_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr addrspace(3) %addr syncscope("device") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_float_shared_sys(float %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_float_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_float_shared_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_shared_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr addrspace(3) %addr syncscope("") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_float_shared_cta(float %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_float_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_float_shared_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_shared_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr addrspace(3) %addr syncscope("block") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_float_shared_cluster(float %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_float_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_float_shared_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_shared_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cluster.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr addrspace(3) %addr syncscope("cluster") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_float_shared_gpu(float %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_float_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_float_shared_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_shared_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr addrspace(3) %addr syncscope("device") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_float_shared_sys_volatile(float %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_float_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_float_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_shared_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr addrspace(3) %addr syncscope("") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_float_shared_cta_volatile(float %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_float_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_float_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_shared_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr addrspace(3) %addr syncscope("block") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_float_shared_cluster_volatile(float %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_float_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_float_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_shared_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr addrspace(3) %addr syncscope("cluster") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_float_shared_gpu_volatile(float %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_float_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [monotonic_float_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_float_shared_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr addrspace(3) %addr syncscope("device") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_double_shared_sys(double %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_double_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_shared_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_double_shared_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr addrspace(3) %addr syncscope("") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_double_shared_cta(double %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_double_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_shared_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_double_shared_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr addrspace(3) %addr syncscope("block") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_double_shared_cluster(double %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_double_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_shared_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_double_shared_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cluster.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr addrspace(3) %addr syncscope("cluster") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_double_shared_gpu(double %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_double_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_shared_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_double_shared_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr addrspace(3) %addr syncscope("device") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_double_shared_sys_volatile(double %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_double_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_double_shared_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr addrspace(3) %addr syncscope("") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_double_shared_cta_volatile(double %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_double_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_double_shared_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr addrspace(3) %addr syncscope("block") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_double_shared_cluster_volatile(double %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_double_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_double_shared_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr addrspace(3) %addr syncscope("cluster") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_double_shared_gpu_volatile(double %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: monotonic_double_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [monotonic_double_shared_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr addrspace(3) %addr syncscope("device") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_i8_local_sys(i8 %val) {
+; CHECK-LABEL: monotonic_i8_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot576[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot576;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b8 %rs1, [monotonic_i8_local_sys_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b8 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i8 %val, ptr addrspace(5) %slot syncscope("") monotonic, align 1
+ ret void
+}
+
+define void @monotonic_i8_local_cta(i8 %val) {
+; CHECK-LABEL: monotonic_i8_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot577[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot577;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b8 %rs1, [monotonic_i8_local_cta_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b8 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i8 %val, ptr addrspace(5) %slot syncscope("block") monotonic, align 1
+ ret void
+}
+
+define void @monotonic_i8_local_cluster(i8 %val) {
+; CHECK-LABEL: monotonic_i8_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot578[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot578;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b8 %rs1, [monotonic_i8_local_cluster_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b8 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i8 %val, ptr addrspace(5) %slot syncscope("cluster") monotonic, align 1
+ ret void
+}
+
+define void @monotonic_i8_local_gpu(i8 %val) {
+; CHECK-LABEL: monotonic_i8_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot579[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot579;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b8 %rs1, [monotonic_i8_local_gpu_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b8 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i8 %val, ptr addrspace(5) %slot syncscope("device") monotonic, align 1
+ ret void
+}
+
+define void @monotonic_i8_local_sys_volatile(i8 %val) {
+; CHECK-LABEL: monotonic_i8_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot580[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot580;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b8 %rs1, [monotonic_i8_local_sys_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b8 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i8 %val, ptr addrspace(5) %slot syncscope("") monotonic, align 1
+ ret void
+}
+
+define void @monotonic_i8_local_cta_volatile(i8 %val) {
+; CHECK-LABEL: monotonic_i8_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot581[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot581;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b8 %rs1, [monotonic_i8_local_cta_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b8 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i8 %val, ptr addrspace(5) %slot syncscope("block") monotonic, align 1
+ ret void
+}
+
+define void @monotonic_i8_local_cluster_volatile(i8 %val) {
+; CHECK-LABEL: monotonic_i8_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot582[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot582;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b8 %rs1, [monotonic_i8_local_cluster_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b8 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i8 %val, ptr addrspace(5) %slot syncscope("cluster") monotonic, align 1
+ ret void
+}
+
+define void @monotonic_i8_local_gpu_volatile(i8 %val) {
+; CHECK-LABEL: monotonic_i8_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot583[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot583;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b8 %rs1, [monotonic_i8_local_gpu_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b8 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i8 %val, ptr addrspace(5) %slot syncscope("device") monotonic, align 1
+ ret void
+}
+
+define void @monotonic_i16_local_sys(i16 %val) {
+; CHECK-LABEL: monotonic_i16_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot584[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot584;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_i16_local_sys_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i16 %val, ptr addrspace(5) %slot syncscope("") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_i16_local_cta(i16 %val) {
+; CHECK-LABEL: monotonic_i16_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot585[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot585;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_i16_local_cta_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i16 %val, ptr addrspace(5) %slot syncscope("block") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_i16_local_cluster(i16 %val) {
+; CHECK-LABEL: monotonic_i16_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot586[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot586;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_i16_local_cluster_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i16 %val, ptr addrspace(5) %slot syncscope("cluster") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_i16_local_gpu(i16 %val) {
+; CHECK-LABEL: monotonic_i16_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot587[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot587;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_i16_local_gpu_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i16 %val, ptr addrspace(5) %slot syncscope("device") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_i16_local_sys_volatile(i16 %val) {
+; CHECK-LABEL: monotonic_i16_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot588[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot588;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_i16_local_sys_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i16 %val, ptr addrspace(5) %slot syncscope("") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_i16_local_cta_volatile(i16 %val) {
+; CHECK-LABEL: monotonic_i16_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot589[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot589;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_i16_local_cta_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i16 %val, ptr addrspace(5) %slot syncscope("block") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_i16_local_cluster_volatile(i16 %val) {
+; CHECK-LABEL: monotonic_i16_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot590[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot590;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_i16_local_cluster_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i16 %val, ptr addrspace(5) %slot syncscope("cluster") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_i16_local_gpu_volatile(i16 %val) {
+; CHECK-LABEL: monotonic_i16_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot591[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot591;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_i16_local_gpu_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i16 %val, ptr addrspace(5) %slot syncscope("device") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_i32_local_sys(i32 %val) {
+; CHECK-LABEL: monotonic_i32_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot592[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot592;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [monotonic_i32_local_sys_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i32 %val, ptr addrspace(5) %slot syncscope("") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_i32_local_cta(i32 %val) {
+; CHECK-LABEL: monotonic_i32_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot593[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot593;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [monotonic_i32_local_cta_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i32 %val, ptr addrspace(5) %slot syncscope("block") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_i32_local_cluster(i32 %val) {
+; CHECK-LABEL: monotonic_i32_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot594[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot594;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [monotonic_i32_local_cluster_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i32 %val, ptr addrspace(5) %slot syncscope("cluster") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_i32_local_gpu(i32 %val) {
+; CHECK-LABEL: monotonic_i32_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot595[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot595;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [monotonic_i32_local_gpu_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i32 %val, ptr addrspace(5) %slot syncscope("device") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_i32_local_sys_volatile(i32 %val) {
+; CHECK-LABEL: monotonic_i32_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot596[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot596;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [monotonic_i32_local_sys_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i32 %val, ptr addrspace(5) %slot syncscope("") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_i32_local_cta_volatile(i32 %val) {
+; CHECK-LABEL: monotonic_i32_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot597[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot597;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [monotonic_i32_local_cta_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i32 %val, ptr addrspace(5) %slot syncscope("block") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_i32_local_cluster_volatile(i32 %val) {
+; CHECK-LABEL: monotonic_i32_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot598[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot598;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [monotonic_i32_local_cluster_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i32 %val, ptr addrspace(5) %slot syncscope("cluster") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_i32_local_gpu_volatile(i32 %val) {
+; CHECK-LABEL: monotonic_i32_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot599[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot599;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [monotonic_i32_local_gpu_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i32 %val, ptr addrspace(5) %slot syncscope("device") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_i64_local_sys(i64 %val) {
+; CHECK-LABEL: monotonic_i64_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot600[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot600;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_local_sys_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i64 %val, ptr addrspace(5) %slot syncscope("") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_i64_local_cta(i64 %val) {
+; CHECK-LABEL: monotonic_i64_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot601[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot601;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_local_cta_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i64 %val, ptr addrspace(5) %slot syncscope("block") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_i64_local_cluster(i64 %val) {
+; CHECK-LABEL: monotonic_i64_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot602[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot602;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_local_cluster_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i64 %val, ptr addrspace(5) %slot syncscope("cluster") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_i64_local_gpu(i64 %val) {
+; CHECK-LABEL: monotonic_i64_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot603[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot603;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_local_gpu_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i64 %val, ptr addrspace(5) %slot syncscope("device") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_i64_local_sys_volatile(i64 %val) {
+; CHECK-LABEL: monotonic_i64_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot604[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot604;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_local_sys_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i64 %val, ptr addrspace(5) %slot syncscope("") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_i64_local_cta_volatile(i64 %val) {
+; CHECK-LABEL: monotonic_i64_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot605[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot605;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_local_cta_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i64 %val, ptr addrspace(5) %slot syncscope("block") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_i64_local_cluster_volatile(i64 %val) {
+; CHECK-LABEL: monotonic_i64_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot606[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot606;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_local_cluster_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i64 %val, ptr addrspace(5) %slot syncscope("cluster") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_i64_local_gpu_volatile(i64 %val) {
+; CHECK-LABEL: monotonic_i64_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot607[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot607;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_i64_local_gpu_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i64 %val, ptr addrspace(5) %slot syncscope("device") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_i128_local_sys(i128 %val) {
+; CHECK-LABEL: monotonic_i128_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 16 .b8 __local_depot608[16];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot608;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [monotonic_i128_local_sys_param_0];
+; CHECK-NEXT: add.u64 %rd3, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd3;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.v2.b64 [%SP], {%rd1, %rd2};
+; CHECK-NEXT: ret;
+ %slot = alloca i128, align 16, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i128 %val, ptr addrspace(5) %slot syncscope("") monotonic, align 16
+ ret void
+}
+
+define void @monotonic_i128_local_cta(i128 %val) {
+; CHECK-LABEL: monotonic_i128_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 16 .b8 __local_depot609[16];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot609;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [monotonic_i128_local_cta_param_0];
+; CHECK-NEXT: add.u64 %rd3, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd3;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.v2.b64 [%SP], {%rd1, %rd2};
+; CHECK-NEXT: ret;
+ %slot = alloca i128, align 16, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i128 %val, ptr addrspace(5) %slot syncscope("block") monotonic, align 16
+ ret void
+}
+
+define void @monotonic_i128_local_cluster(i128 %val) {
+; CHECK-LABEL: monotonic_i128_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 16 .b8 __local_depot610[16];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot610;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [monotonic_i128_local_cluster_param_0];
+; CHECK-NEXT: add.u64 %rd3, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd3;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.v2.b64 [%SP], {%rd1, %rd2};
+; CHECK-NEXT: ret;
+ %slot = alloca i128, align 16, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i128 %val, ptr addrspace(5) %slot syncscope("cluster") monotonic, align 16
+ ret void
+}
+
+define void @monotonic_i128_local_gpu(i128 %val) {
+; CHECK-LABEL: monotonic_i128_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 16 .b8 __local_depot611[16];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot611;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [monotonic_i128_local_gpu_param_0];
+; CHECK-NEXT: add.u64 %rd3, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd3;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.v2.b64 [%SP], {%rd1, %rd2};
+; CHECK-NEXT: ret;
+ %slot = alloca i128, align 16, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i128 %val, ptr addrspace(5) %slot syncscope("device") monotonic, align 16
+ ret void
+}
+
+define void @monotonic_i128_local_sys_volatile(i128 %val) {
+; CHECK-LABEL: monotonic_i128_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 16 .b8 __local_depot612[16];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot612;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [monotonic_i128_local_sys_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd3, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd3;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.v2.b64 [%SP], {%rd1, %rd2};
+; CHECK-NEXT: ret;
+ %slot = alloca i128, align 16, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i128 %val, ptr addrspace(5) %slot syncscope("") monotonic, align 16
+ ret void
+}
+
+define void @monotonic_i128_local_cta_volatile(i128 %val) {
+; CHECK-LABEL: monotonic_i128_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 16 .b8 __local_depot613[16];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot613;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [monotonic_i128_local_cta_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd3, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd3;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.v2.b64 [%SP], {%rd1, %rd2};
+; CHECK-NEXT: ret;
+ %slot = alloca i128, align 16, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i128 %val, ptr addrspace(5) %slot syncscope("block") monotonic, align 16
+ ret void
+}
+
+define void @monotonic_i128_local_cluster_volatile(i128 %val) {
+; CHECK-LABEL: monotonic_i128_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 16 .b8 __local_depot614[16];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot614;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [monotonic_i128_local_cluster_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd3, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd3;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.v2.b64 [%SP], {%rd1, %rd2};
+; CHECK-NEXT: ret;
+ %slot = alloca i128, align 16, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i128 %val, ptr addrspace(5) %slot syncscope("cluster") monotonic, align 16
+ ret void
+}
+
+define void @monotonic_i128_local_gpu_volatile(i128 %val) {
+; CHECK-LABEL: monotonic_i128_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 16 .b8 __local_depot615[16];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot615;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [monotonic_i128_local_gpu_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd3, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd3;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.v2.b64 [%SP], {%rd1, %rd2};
+; CHECK-NEXT: ret;
+ %slot = alloca i128, align 16, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i128 %val, ptr addrspace(5) %slot syncscope("device") monotonic, align 16
+ ret void
+}
+
+define void @monotonic_half_local_sys(half %val) {
+; CHECK-LABEL: monotonic_half_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot616[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot616;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_half_local_sys_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic half %val, ptr addrspace(5) %slot syncscope("") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_half_local_cta(half %val) {
+; CHECK-LABEL: monotonic_half_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot617[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot617;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_half_local_cta_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic half %val, ptr addrspace(5) %slot syncscope("block") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_half_local_cluster(half %val) {
+; CHECK-LABEL: monotonic_half_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot618[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot618;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_half_local_cluster_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic half %val, ptr addrspace(5) %slot syncscope("cluster") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_half_local_gpu(half %val) {
+; CHECK-LABEL: monotonic_half_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot619[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot619;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_half_local_gpu_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic half %val, ptr addrspace(5) %slot syncscope("device") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_half_local_sys_volatile(half %val) {
+; CHECK-LABEL: monotonic_half_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot620[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot620;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_half_local_sys_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile half %val, ptr addrspace(5) %slot syncscope("") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_half_local_cta_volatile(half %val) {
+; CHECK-LABEL: monotonic_half_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot621[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot621;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_half_local_cta_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile half %val, ptr addrspace(5) %slot syncscope("block") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_half_local_cluster_volatile(half %val) {
+; CHECK-LABEL: monotonic_half_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot622[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot622;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_half_local_cluster_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile half %val, ptr addrspace(5) %slot syncscope("cluster") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_half_local_gpu_volatile(half %val) {
+; CHECK-LABEL: monotonic_half_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot623[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot623;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_half_local_gpu_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile half %val, ptr addrspace(5) %slot syncscope("device") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_bfloat_local_sys(bfloat %val) {
+; CHECK-LABEL: monotonic_bfloat_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot624[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot624;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_bfloat_local_sys_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic bfloat %val, ptr addrspace(5) %slot syncscope("") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_bfloat_local_cta(bfloat %val) {
+; CHECK-LABEL: monotonic_bfloat_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot625[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot625;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_bfloat_local_cta_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic bfloat %val, ptr addrspace(5) %slot syncscope("block") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_bfloat_local_cluster(bfloat %val) {
+; CHECK-LABEL: monotonic_bfloat_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot626[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot626;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_bfloat_local_cluster_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic bfloat %val, ptr addrspace(5) %slot syncscope("cluster") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_bfloat_local_gpu(bfloat %val) {
+; CHECK-LABEL: monotonic_bfloat_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot627[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot627;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_bfloat_local_gpu_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic bfloat %val, ptr addrspace(5) %slot syncscope("device") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_bfloat_local_sys_volatile(bfloat %val) {
+; CHECK-LABEL: monotonic_bfloat_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot628[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot628;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_bfloat_local_sys_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile bfloat %val, ptr addrspace(5) %slot syncscope("") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_bfloat_local_cta_volatile(bfloat %val) {
+; CHECK-LABEL: monotonic_bfloat_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot629[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot629;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_bfloat_local_cta_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile bfloat %val, ptr addrspace(5) %slot syncscope("block") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_bfloat_local_cluster_volatile(bfloat %val) {
+; CHECK-LABEL: monotonic_bfloat_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot630[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot630;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_bfloat_local_cluster_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile bfloat %val, ptr addrspace(5) %slot syncscope("cluster") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_bfloat_local_gpu_volatile(bfloat %val) {
+; CHECK-LABEL: monotonic_bfloat_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot631[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot631;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [monotonic_bfloat_local_gpu_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile bfloat %val, ptr addrspace(5) %slot syncscope("device") monotonic, align 2
+ ret void
+}
+
+define void @monotonic_float_local_sys(float %val) {
+; CHECK-LABEL: monotonic_float_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot632[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot632;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [monotonic_float_local_sys_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic float %val, ptr addrspace(5) %slot syncscope("") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_float_local_cta(float %val) {
+; CHECK-LABEL: monotonic_float_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot633[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot633;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [monotonic_float_local_cta_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic float %val, ptr addrspace(5) %slot syncscope("block") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_float_local_cluster(float %val) {
+; CHECK-LABEL: monotonic_float_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot634[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot634;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [monotonic_float_local_cluster_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic float %val, ptr addrspace(5) %slot syncscope("cluster") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_float_local_gpu(float %val) {
+; CHECK-LABEL: monotonic_float_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot635[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot635;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [monotonic_float_local_gpu_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic float %val, ptr addrspace(5) %slot syncscope("device") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_float_local_sys_volatile(float %val) {
+; CHECK-LABEL: monotonic_float_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot636[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot636;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [monotonic_float_local_sys_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile float %val, ptr addrspace(5) %slot syncscope("") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_float_local_cta_volatile(float %val) {
+; CHECK-LABEL: monotonic_float_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot637[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot637;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [monotonic_float_local_cta_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile float %val, ptr addrspace(5) %slot syncscope("block") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_float_local_cluster_volatile(float %val) {
+; CHECK-LABEL: monotonic_float_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot638[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot638;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [monotonic_float_local_cluster_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile float %val, ptr addrspace(5) %slot syncscope("cluster") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_float_local_gpu_volatile(float %val) {
+; CHECK-LABEL: monotonic_float_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot639[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot639;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [monotonic_float_local_gpu_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile float %val, ptr addrspace(5) %slot syncscope("device") monotonic, align 4
+ ret void
+}
+
+define void @monotonic_double_local_sys(double %val) {
+; CHECK-LABEL: monotonic_double_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot640[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot640;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_local_sys_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic double %val, ptr addrspace(5) %slot syncscope("") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_double_local_cta(double %val) {
+; CHECK-LABEL: monotonic_double_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot641[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot641;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_local_cta_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic double %val, ptr addrspace(5) %slot syncscope("block") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_double_local_cluster(double %val) {
+; CHECK-LABEL: monotonic_double_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot642[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot642;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_local_cluster_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic double %val, ptr addrspace(5) %slot syncscope("cluster") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_double_local_gpu(double %val) {
+; CHECK-LABEL: monotonic_double_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot643[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot643;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_local_gpu_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic double %val, ptr addrspace(5) %slot syncscope("device") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_double_local_sys_volatile(double %val) {
+; CHECK-LABEL: monotonic_double_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot644[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot644;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_local_sys_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile double %val, ptr addrspace(5) %slot syncscope("") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_double_local_cta_volatile(double %val) {
+; CHECK-LABEL: monotonic_double_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot645[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot645;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_local_cta_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile double %val, ptr addrspace(5) %slot syncscope("block") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_double_local_cluster_volatile(double %val) {
+; CHECK-LABEL: monotonic_double_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot646[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot646;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_local_cluster_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile double %val, ptr addrspace(5) %slot syncscope("cluster") monotonic, align 8
+ ret void
+}
+
+define void @monotonic_double_local_gpu_volatile(double %val) {
+; CHECK-LABEL: monotonic_double_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot647[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot647;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [monotonic_double_local_gpu_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile double %val, ptr addrspace(5) %slot syncscope("device") monotonic, align 8
+ ret void
+}
+
+define void @release_i8_generic_sys(i8 %val, ptr %addr) {
+; CHECK-LABEL: release_i8_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [release_i8_generic_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i8_generic_sys_param_1];
+; CHECK-NEXT: st.release.sys.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr %addr syncscope("") release, align 1
+ ret void
+}
+
+define void @release_i8_generic_cta(i8 %val, ptr %addr) {
+; CHECK-LABEL: release_i8_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [release_i8_generic_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i8_generic_cta_param_1];
+; CHECK-NEXT: st.release.cta.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr %addr syncscope("block") release, align 1
+ ret void
+}
+
+define void @release_i8_generic_cluster(i8 %val, ptr %addr) {
+; CHECK-LABEL: release_i8_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [release_i8_generic_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i8_generic_cluster_param_1];
+; CHECK-NEXT: st.release.cluster.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr %addr syncscope("cluster") release, align 1
+ ret void
+}
+
+define void @release_i8_generic_gpu(i8 %val, ptr %addr) {
+; CHECK-LABEL: release_i8_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [release_i8_generic_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i8_generic_gpu_param_1];
+; CHECK-NEXT: st.release.gpu.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr %addr syncscope("device") release, align 1
+ ret void
+}
+
+define void @release_i8_generic_sys_volatile(i8 %val, ptr %addr) {
+; CHECK-LABEL: release_i8_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [release_i8_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i8_generic_sys_volatile_param_1];
+; CHECK-NEXT: st.release.sys.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr %addr syncscope("") release, align 1
+ ret void
+}
+
+define void @release_i8_generic_cta_volatile(i8 %val, ptr %addr) {
+; CHECK-LABEL: release_i8_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [release_i8_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i8_generic_cta_volatile_param_1];
+; CHECK-NEXT: st.release.sys.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr %addr syncscope("block") release, align 1
+ ret void
+}
+
+define void @release_i8_generic_cluster_volatile(i8 %val, ptr %addr) {
+; CHECK-LABEL: release_i8_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [release_i8_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i8_generic_cluster_volatile_param_1];
+; CHECK-NEXT: st.release.sys.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr %addr syncscope("cluster") release, align 1
+ ret void
+}
+
+define void @release_i8_generic_gpu_volatile(i8 %val, ptr %addr) {
+; CHECK-LABEL: release_i8_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [release_i8_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i8_generic_gpu_volatile_param_1];
+; CHECK-NEXT: st.release.sys.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr %addr syncscope("device") release, align 1
+ ret void
+}
+
+define void @release_i16_generic_sys(i16 %val, ptr %addr) {
+; CHECK-LABEL: release_i16_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_i16_generic_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i16_generic_sys_param_1];
+; CHECK-NEXT: st.release.sys.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr %addr syncscope("") release, align 2
+ ret void
+}
+
+define void @release_i16_generic_cta(i16 %val, ptr %addr) {
+; CHECK-LABEL: release_i16_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_i16_generic_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i16_generic_cta_param_1];
+; CHECK-NEXT: st.release.cta.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr %addr syncscope("block") release, align 2
+ ret void
+}
+
+define void @release_i16_generic_cluster(i16 %val, ptr %addr) {
+; CHECK-LABEL: release_i16_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_i16_generic_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i16_generic_cluster_param_1];
+; CHECK-NEXT: st.release.cluster.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr %addr syncscope("cluster") release, align 2
+ ret void
+}
+
+define void @release_i16_generic_gpu(i16 %val, ptr %addr) {
+; CHECK-LABEL: release_i16_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_i16_generic_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i16_generic_gpu_param_1];
+; CHECK-NEXT: st.release.gpu.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr %addr syncscope("device") release, align 2
+ ret void
+}
+
+define void @release_i16_generic_sys_volatile(i16 %val, ptr %addr) {
+; CHECK-LABEL: release_i16_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_i16_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i16_generic_sys_volatile_param_1];
+; CHECK-NEXT: st.release.sys.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr %addr syncscope("") release, align 2
+ ret void
+}
+
+define void @release_i16_generic_cta_volatile(i16 %val, ptr %addr) {
+; CHECK-LABEL: release_i16_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_i16_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i16_generic_cta_volatile_param_1];
+; CHECK-NEXT: st.release.sys.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr %addr syncscope("block") release, align 2
+ ret void
+}
+
+define void @release_i16_generic_cluster_volatile(i16 %val, ptr %addr) {
+; CHECK-LABEL: release_i16_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_i16_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i16_generic_cluster_volatile_param_1];
+; CHECK-NEXT: st.release.sys.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr %addr syncscope("cluster") release, align 2
+ ret void
+}
+
+define void @release_i16_generic_gpu_volatile(i16 %val, ptr %addr) {
+; CHECK-LABEL: release_i16_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_i16_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i16_generic_gpu_volatile_param_1];
+; CHECK-NEXT: st.release.sys.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr %addr syncscope("device") release, align 2
+ ret void
+}
+
+define void @release_i32_generic_sys(i32 %val, ptr %addr) {
+; CHECK-LABEL: release_i32_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_i32_generic_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i32_generic_sys_param_1];
+; CHECK-NEXT: st.release.sys.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr %addr syncscope("") release, align 4
+ ret void
+}
+
+define void @release_i32_generic_cta(i32 %val, ptr %addr) {
+; CHECK-LABEL: release_i32_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_i32_generic_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i32_generic_cta_param_1];
+; CHECK-NEXT: st.release.cta.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr %addr syncscope("block") release, align 4
+ ret void
+}
+
+define void @release_i32_generic_cluster(i32 %val, ptr %addr) {
+; CHECK-LABEL: release_i32_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_i32_generic_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i32_generic_cluster_param_1];
+; CHECK-NEXT: st.release.cluster.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr %addr syncscope("cluster") release, align 4
+ ret void
+}
+
+define void @release_i32_generic_gpu(i32 %val, ptr %addr) {
+; CHECK-LABEL: release_i32_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_i32_generic_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i32_generic_gpu_param_1];
+; CHECK-NEXT: st.release.gpu.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr %addr syncscope("device") release, align 4
+ ret void
+}
+
+define void @release_i32_generic_sys_volatile(i32 %val, ptr %addr) {
+; CHECK-LABEL: release_i32_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_i32_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i32_generic_sys_volatile_param_1];
+; CHECK-NEXT: st.release.sys.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr %addr syncscope("") release, align 4
+ ret void
+}
+
+define void @release_i32_generic_cta_volatile(i32 %val, ptr %addr) {
+; CHECK-LABEL: release_i32_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_i32_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i32_generic_cta_volatile_param_1];
+; CHECK-NEXT: st.release.sys.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr %addr syncscope("block") release, align 4
+ ret void
+}
+
+define void @release_i32_generic_cluster_volatile(i32 %val, ptr %addr) {
+; CHECK-LABEL: release_i32_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_i32_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i32_generic_cluster_volatile_param_1];
+; CHECK-NEXT: st.release.sys.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr %addr syncscope("cluster") release, align 4
+ ret void
+}
+
+define void @release_i32_generic_gpu_volatile(i32 %val, ptr %addr) {
+; CHECK-LABEL: release_i32_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_i32_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i32_generic_gpu_volatile_param_1];
+; CHECK-NEXT: st.release.sys.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr %addr syncscope("device") release, align 4
+ ret void
+}
+
+define void @release_i64_generic_sys(i64 %val, ptr %addr) {
+; CHECK-LABEL: release_i64_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i64_generic_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [release_i64_generic_sys_param_1];
+; CHECK-NEXT: st.release.sys.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr %addr syncscope("") release, align 8
+ ret void
+}
+
+define void @release_i64_generic_cta(i64 %val, ptr %addr) {
+; CHECK-LABEL: release_i64_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i64_generic_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [release_i64_generic_cta_param_1];
+; CHECK-NEXT: st.release.cta.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr %addr syncscope("block") release, align 8
+ ret void
+}
+
+define void @release_i64_generic_cluster(i64 %val, ptr %addr) {
+; CHECK-LABEL: release_i64_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i64_generic_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [release_i64_generic_cluster_param_1];
+; CHECK-NEXT: st.release.cluster.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr %addr syncscope("cluster") release, align 8
+ ret void
+}
+
+define void @release_i64_generic_gpu(i64 %val, ptr %addr) {
+; CHECK-LABEL: release_i64_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i64_generic_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [release_i64_generic_gpu_param_1];
+; CHECK-NEXT: st.release.gpu.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr %addr syncscope("device") release, align 8
+ ret void
+}
+
+define void @release_i64_generic_sys_volatile(i64 %val, ptr %addr) {
+; CHECK-LABEL: release_i64_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i64_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [release_i64_generic_sys_volatile_param_1];
+; CHECK-NEXT: st.release.sys.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr %addr syncscope("") release, align 8
+ ret void
+}
+
+define void @release_i64_generic_cta_volatile(i64 %val, ptr %addr) {
+; CHECK-LABEL: release_i64_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i64_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [release_i64_generic_cta_volatile_param_1];
+; CHECK-NEXT: st.release.sys.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr %addr syncscope("block") release, align 8
+ ret void
+}
+
+define void @release_i64_generic_cluster_volatile(i64 %val, ptr %addr) {
+; CHECK-LABEL: release_i64_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i64_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [release_i64_generic_cluster_volatile_param_1];
+; CHECK-NEXT: st.release.sys.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr %addr syncscope("cluster") release, align 8
+ ret void
+}
+
+define void @release_i64_generic_gpu_volatile(i64 %val, ptr %addr) {
+; CHECK-LABEL: release_i64_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i64_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [release_i64_generic_gpu_volatile_param_1];
+; CHECK-NEXT: st.release.sys.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr %addr syncscope("device") release, align 8
+ ret void
+}
+
+define void @release_i128_generic_sys(i128 %val, ptr %addr) {
+; CHECK-LABEL: release_i128_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [release_i128_generic_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd3, [release_i128_generic_sys_param_1];
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 amt, dst;
+; CHECK-NEXT: mov.b128 amt, {%rd1, %rd2};
+; CHECK-NEXT: atom.release.sys.exch.b128 dst, [%rd3], amt;
+; CHECK-NEXT: mov.b128 {%rd4, %rd5}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: ret;
+ store atomic i128 %val, ptr %addr syncscope("") release, align 16
+ ret void
+}
+
+define void @release_i128_generic_cta(i128 %val, ptr %addr) {
+; CHECK-LABEL: release_i128_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [release_i128_generic_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd3, [release_i128_generic_cta_param_1];
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 amt, dst;
+; CHECK-NEXT: mov.b128 amt, {%rd1, %rd2};
+; CHECK-NEXT: atom.release.cta.exch.b128 dst, [%rd3], amt;
+; CHECK-NEXT: mov.b128 {%rd4, %rd5}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: ret;
+ store atomic i128 %val, ptr %addr syncscope("block") release, align 16
+ ret void
+}
+
+define void @release_i128_generic_cluster(i128 %val, ptr %addr) {
+; CHECK-LABEL: release_i128_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [release_i128_generic_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd3, [release_i128_generic_cluster_param_1];
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 amt, dst;
+; CHECK-NEXT: mov.b128 amt, {%rd1, %rd2};
+; CHECK-NEXT: atom.release.cluster.exch.b128 dst, [%rd3], amt;
+; CHECK-NEXT: mov.b128 {%rd4, %rd5}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: ret;
+ store atomic i128 %val, ptr %addr syncscope("cluster") release, align 16
+ ret void
+}
+
+define void @release_i128_generic_gpu(i128 %val, ptr %addr) {
+; CHECK-LABEL: release_i128_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [release_i128_generic_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd3, [release_i128_generic_gpu_param_1];
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 amt, dst;
+; CHECK-NEXT: mov.b128 amt, {%rd1, %rd2};
+; CHECK-NEXT: atom.release.gpu.exch.b128 dst, [%rd3], amt;
+; CHECK-NEXT: mov.b128 {%rd4, %rd5}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: ret;
+ store atomic i128 %val, ptr %addr syncscope("device") release, align 16
+ ret void
+}
+
+define void @release_i128_generic_sys_volatile(i128 %val, ptr %addr) {
+; CHECK-LABEL: release_i128_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [release_i128_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd3, [release_i128_generic_sys_volatile_param_1];
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 amt, dst;
+; CHECK-NEXT: mov.b128 amt, {%rd1, %rd2};
+; CHECK-NEXT: atom.release.sys.exch.b128 dst, [%rd3], amt;
+; CHECK-NEXT: mov.b128 {%rd4, %rd5}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: ret;
+ store atomic volatile i128 %val, ptr %addr syncscope("") release, align 16
+ ret void
+}
+
+define void @release_i128_generic_cta_volatile(i128 %val, ptr %addr) {
+; CHECK-LABEL: release_i128_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [release_i128_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd3, [release_i128_generic_cta_volatile_param_1];
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 amt, dst;
+; CHECK-NEXT: mov.b128 amt, {%rd1, %rd2};
+; CHECK-NEXT: atom.release.cta.exch.b128 dst, [%rd3], amt;
+; CHECK-NEXT: mov.b128 {%rd4, %rd5}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: ret;
+ store atomic volatile i128 %val, ptr %addr syncscope("block") release, align 16
+ ret void
+}
+
+define void @release_i128_generic_cluster_volatile(i128 %val, ptr %addr) {
+; CHECK-LABEL: release_i128_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [release_i128_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd3, [release_i128_generic_cluster_volatile_param_1];
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 amt, dst;
+; CHECK-NEXT: mov.b128 amt, {%rd1, %rd2};
+; CHECK-NEXT: atom.release.cluster.exch.b128 dst, [%rd3], amt;
+; CHECK-NEXT: mov.b128 {%rd4, %rd5}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: ret;
+ store atomic volatile i128 %val, ptr %addr syncscope("cluster") release, align 16
+ ret void
+}
+
+define void @release_i128_generic_gpu_volatile(i128 %val, ptr %addr) {
+; CHECK-LABEL: release_i128_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [release_i128_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd3, [release_i128_generic_gpu_volatile_param_1];
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 amt, dst;
+; CHECK-NEXT: mov.b128 amt, {%rd1, %rd2};
+; CHECK-NEXT: atom.release.gpu.exch.b128 dst, [%rd3], amt;
+; CHECK-NEXT: mov.b128 {%rd4, %rd5}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: ret;
+ store atomic volatile i128 %val, ptr %addr syncscope("device") release, align 16
+ ret void
+}
+
+define void @release_half_generic_sys(half %val, ptr %addr) {
+; CHECK-LABEL: release_half_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_half_generic_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_half_generic_sys_param_1];
+; CHECK-NEXT: st.release.sys.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr %addr syncscope("") release, align 2
+ ret void
+}
+
+define void @release_half_generic_cta(half %val, ptr %addr) {
+; CHECK-LABEL: release_half_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_half_generic_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_half_generic_cta_param_1];
+; CHECK-NEXT: st.release.cta.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr %addr syncscope("block") release, align 2
+ ret void
+}
+
+define void @release_half_generic_cluster(half %val, ptr %addr) {
+; CHECK-LABEL: release_half_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_half_generic_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_half_generic_cluster_param_1];
+; CHECK-NEXT: st.release.cluster.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr %addr syncscope("cluster") release, align 2
+ ret void
+}
+
+define void @release_half_generic_gpu(half %val, ptr %addr) {
+; CHECK-LABEL: release_half_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_half_generic_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_half_generic_gpu_param_1];
+; CHECK-NEXT: st.release.gpu.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr %addr syncscope("device") release, align 2
+ ret void
+}
+
+define void @release_half_generic_sys_volatile(half %val, ptr %addr) {
+; CHECK-LABEL: release_half_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_half_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_half_generic_sys_volatile_param_1];
+; CHECK-NEXT: st.release.sys.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr %addr syncscope("") release, align 2
+ ret void
+}
+
+define void @release_half_generic_cta_volatile(half %val, ptr %addr) {
+; CHECK-LABEL: release_half_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_half_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_half_generic_cta_volatile_param_1];
+; CHECK-NEXT: st.release.sys.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr %addr syncscope("block") release, align 2
+ ret void
+}
+
+define void @release_half_generic_cluster_volatile(half %val, ptr %addr) {
+; CHECK-LABEL: release_half_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_half_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_half_generic_cluster_volatile_param_1];
+; CHECK-NEXT: st.release.sys.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr %addr syncscope("cluster") release, align 2
+ ret void
+}
+
+define void @release_half_generic_gpu_volatile(half %val, ptr %addr) {
+; CHECK-LABEL: release_half_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_half_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_half_generic_gpu_volatile_param_1];
+; CHECK-NEXT: st.release.sys.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr %addr syncscope("device") release, align 2
+ ret void
+}
+
+define void @release_bfloat_generic_sys(bfloat %val, ptr %addr) {
+; CHECK-LABEL: release_bfloat_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_bfloat_generic_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_bfloat_generic_sys_param_1];
+; CHECK-NEXT: st.release.sys.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr %addr syncscope("") release, align 2
+ ret void
+}
+
+define void @release_bfloat_generic_cta(bfloat %val, ptr %addr) {
+; CHECK-LABEL: release_bfloat_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_bfloat_generic_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_bfloat_generic_cta_param_1];
+; CHECK-NEXT: st.release.cta.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr %addr syncscope("block") release, align 2
+ ret void
+}
+
+define void @release_bfloat_generic_cluster(bfloat %val, ptr %addr) {
+; CHECK-LABEL: release_bfloat_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_bfloat_generic_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_bfloat_generic_cluster_param_1];
+; CHECK-NEXT: st.release.cluster.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr %addr syncscope("cluster") release, align 2
+ ret void
+}
+
+define void @release_bfloat_generic_gpu(bfloat %val, ptr %addr) {
+; CHECK-LABEL: release_bfloat_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_bfloat_generic_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_bfloat_generic_gpu_param_1];
+; CHECK-NEXT: st.release.gpu.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr %addr syncscope("device") release, align 2
+ ret void
+}
+
+define void @release_bfloat_generic_sys_volatile(bfloat %val, ptr %addr) {
+; CHECK-LABEL: release_bfloat_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_bfloat_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_bfloat_generic_sys_volatile_param_1];
+; CHECK-NEXT: st.release.sys.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr %addr syncscope("") release, align 2
+ ret void
+}
+
+define void @release_bfloat_generic_cta_volatile(bfloat %val, ptr %addr) {
+; CHECK-LABEL: release_bfloat_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_bfloat_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_bfloat_generic_cta_volatile_param_1];
+; CHECK-NEXT: st.release.sys.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr %addr syncscope("block") release, align 2
+ ret void
+}
+
+define void @release_bfloat_generic_cluster_volatile(bfloat %val, ptr %addr) {
+; CHECK-LABEL: release_bfloat_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_bfloat_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_bfloat_generic_cluster_volatile_param_1];
+; CHECK-NEXT: st.release.sys.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr %addr syncscope("cluster") release, align 2
+ ret void
+}
+
+define void @release_bfloat_generic_gpu_volatile(bfloat %val, ptr %addr) {
+; CHECK-LABEL: release_bfloat_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_bfloat_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_bfloat_generic_gpu_volatile_param_1];
+; CHECK-NEXT: st.release.sys.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr %addr syncscope("device") release, align 2
+ ret void
+}
+
+define void @release_float_generic_sys(float %val, ptr %addr) {
+; CHECK-LABEL: release_float_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_float_generic_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_float_generic_sys_param_1];
+; CHECK-NEXT: st.release.sys.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr %addr syncscope("") release, align 4
+ ret void
+}
+
+define void @release_float_generic_cta(float %val, ptr %addr) {
+; CHECK-LABEL: release_float_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_float_generic_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_float_generic_cta_param_1];
+; CHECK-NEXT: st.release.cta.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr %addr syncscope("block") release, align 4
+ ret void
+}
+
+define void @release_float_generic_cluster(float %val, ptr %addr) {
+; CHECK-LABEL: release_float_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_float_generic_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_float_generic_cluster_param_1];
+; CHECK-NEXT: st.release.cluster.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr %addr syncscope("cluster") release, align 4
+ ret void
+}
+
+define void @release_float_generic_gpu(float %val, ptr %addr) {
+; CHECK-LABEL: release_float_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_float_generic_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_float_generic_gpu_param_1];
+; CHECK-NEXT: st.release.gpu.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr %addr syncscope("device") release, align 4
+ ret void
+}
+
+define void @release_float_generic_sys_volatile(float %val, ptr %addr) {
+; CHECK-LABEL: release_float_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_float_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_float_generic_sys_volatile_param_1];
+; CHECK-NEXT: st.release.sys.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr %addr syncscope("") release, align 4
+ ret void
+}
+
+define void @release_float_generic_cta_volatile(float %val, ptr %addr) {
+; CHECK-LABEL: release_float_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_float_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_float_generic_cta_volatile_param_1];
+; CHECK-NEXT: st.release.sys.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr %addr syncscope("block") release, align 4
+ ret void
+}
+
+define void @release_float_generic_cluster_volatile(float %val, ptr %addr) {
+; CHECK-LABEL: release_float_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_float_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_float_generic_cluster_volatile_param_1];
+; CHECK-NEXT: st.release.sys.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr %addr syncscope("cluster") release, align 4
+ ret void
+}
+
+define void @release_float_generic_gpu_volatile(float %val, ptr %addr) {
+; CHECK-LABEL: release_float_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_float_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_float_generic_gpu_volatile_param_1];
+; CHECK-NEXT: st.release.sys.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr %addr syncscope("device") release, align 4
+ ret void
+}
+
+define void @release_double_generic_sys(double %val, ptr %addr) {
+; CHECK-LABEL: release_double_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_double_generic_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [release_double_generic_sys_param_1];
+; CHECK-NEXT: st.release.sys.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr %addr syncscope("") release, align 8
+ ret void
+}
+
+define void @release_double_generic_cta(double %val, ptr %addr) {
+; CHECK-LABEL: release_double_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_double_generic_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [release_double_generic_cta_param_1];
+; CHECK-NEXT: st.release.cta.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr %addr syncscope("block") release, align 8
+ ret void
+}
+
+define void @release_double_generic_cluster(double %val, ptr %addr) {
+; CHECK-LABEL: release_double_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_double_generic_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [release_double_generic_cluster_param_1];
+; CHECK-NEXT: st.release.cluster.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr %addr syncscope("cluster") release, align 8
+ ret void
+}
+
+define void @release_double_generic_gpu(double %val, ptr %addr) {
+; CHECK-LABEL: release_double_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_double_generic_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [release_double_generic_gpu_param_1];
+; CHECK-NEXT: st.release.gpu.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr %addr syncscope("device") release, align 8
+ ret void
+}
+
+define void @release_double_generic_sys_volatile(double %val, ptr %addr) {
+; CHECK-LABEL: release_double_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_double_generic_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [release_double_generic_sys_volatile_param_1];
+; CHECK-NEXT: st.release.sys.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr %addr syncscope("") release, align 8
+ ret void
+}
+
+define void @release_double_generic_cta_volatile(double %val, ptr %addr) {
+; CHECK-LABEL: release_double_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_double_generic_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [release_double_generic_cta_volatile_param_1];
+; CHECK-NEXT: st.release.sys.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr %addr syncscope("block") release, align 8
+ ret void
+}
+
+define void @release_double_generic_cluster_volatile(double %val, ptr %addr) {
+; CHECK-LABEL: release_double_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_double_generic_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [release_double_generic_cluster_volatile_param_1];
+; CHECK-NEXT: st.release.sys.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr %addr syncscope("cluster") release, align 8
+ ret void
+}
+
+define void @release_double_generic_gpu_volatile(double %val, ptr %addr) {
+; CHECK-LABEL: release_double_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_double_generic_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [release_double_generic_gpu_volatile_param_1];
+; CHECK-NEXT: st.release.sys.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr %addr syncscope("device") release, align 8
+ ret void
+}
+
+define void @release_i8_global_sys(i8 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_i8_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [release_i8_global_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i8_global_sys_param_1];
+; CHECK-NEXT: st.release.sys.global.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr addrspace(1) %addr syncscope("") release, align 1
+ ret void
+}
+
+define void @release_i8_global_cta(i8 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_i8_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [release_i8_global_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i8_global_cta_param_1];
+; CHECK-NEXT: st.release.cta.global.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr addrspace(1) %addr syncscope("block") release, align 1
+ ret void
+}
+
+define void @release_i8_global_cluster(i8 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_i8_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [release_i8_global_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i8_global_cluster_param_1];
+; CHECK-NEXT: st.release.cluster.global.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr addrspace(1) %addr syncscope("cluster") release, align 1
+ ret void
+}
+
+define void @release_i8_global_gpu(i8 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_i8_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [release_i8_global_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i8_global_gpu_param_1];
+; CHECK-NEXT: st.release.gpu.global.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr addrspace(1) %addr syncscope("device") release, align 1
+ ret void
+}
+
+define void @release_i8_global_sys_volatile(i8 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_i8_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [release_i8_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i8_global_sys_volatile_param_1];
+; CHECK-NEXT: st.release.sys.global.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr addrspace(1) %addr syncscope("") release, align 1
+ ret void
+}
+
+define void @release_i8_global_cta_volatile(i8 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_i8_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [release_i8_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i8_global_cta_volatile_param_1];
+; CHECK-NEXT: st.release.sys.global.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr addrspace(1) %addr syncscope("block") release, align 1
+ ret void
+}
+
+define void @release_i8_global_cluster_volatile(i8 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_i8_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [release_i8_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i8_global_cluster_volatile_param_1];
+; CHECK-NEXT: st.release.sys.global.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr addrspace(1) %addr syncscope("cluster") release, align 1
+ ret void
+}
+
+define void @release_i8_global_gpu_volatile(i8 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_i8_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [release_i8_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i8_global_gpu_volatile_param_1];
+; CHECK-NEXT: st.release.sys.global.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr addrspace(1) %addr syncscope("device") release, align 1
+ ret void
+}
+
+define void @release_i16_global_sys(i16 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_i16_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_i16_global_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i16_global_sys_param_1];
+; CHECK-NEXT: st.release.sys.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr addrspace(1) %addr syncscope("") release, align 2
+ ret void
+}
+
+define void @release_i16_global_cta(i16 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_i16_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_i16_global_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i16_global_cta_param_1];
+; CHECK-NEXT: st.release.cta.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr addrspace(1) %addr syncscope("block") release, align 2
+ ret void
+}
+
+define void @release_i16_global_cluster(i16 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_i16_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_i16_global_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i16_global_cluster_param_1];
+; CHECK-NEXT: st.release.cluster.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr addrspace(1) %addr syncscope("cluster") release, align 2
+ ret void
+}
+
+define void @release_i16_global_gpu(i16 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_i16_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_i16_global_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i16_global_gpu_param_1];
+; CHECK-NEXT: st.release.gpu.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr addrspace(1) %addr syncscope("device") release, align 2
+ ret void
+}
+
+define void @release_i16_global_sys_volatile(i16 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_i16_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_i16_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i16_global_sys_volatile_param_1];
+; CHECK-NEXT: st.release.sys.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr addrspace(1) %addr syncscope("") release, align 2
+ ret void
+}
+
+define void @release_i16_global_cta_volatile(i16 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_i16_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_i16_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i16_global_cta_volatile_param_1];
+; CHECK-NEXT: st.release.sys.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr addrspace(1) %addr syncscope("block") release, align 2
+ ret void
+}
+
+define void @release_i16_global_cluster_volatile(i16 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_i16_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_i16_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i16_global_cluster_volatile_param_1];
+; CHECK-NEXT: st.release.sys.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr addrspace(1) %addr syncscope("cluster") release, align 2
+ ret void
+}
+
+define void @release_i16_global_gpu_volatile(i16 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_i16_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_i16_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i16_global_gpu_volatile_param_1];
+; CHECK-NEXT: st.release.sys.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr addrspace(1) %addr syncscope("device") release, align 2
+ ret void
+}
+
+define void @release_i32_global_sys(i32 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_i32_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_i32_global_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i32_global_sys_param_1];
+; CHECK-NEXT: st.release.sys.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr addrspace(1) %addr syncscope("") release, align 4
+ ret void
+}
+
+define void @release_i32_global_cta(i32 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_i32_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_i32_global_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i32_global_cta_param_1];
+; CHECK-NEXT: st.release.cta.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr addrspace(1) %addr syncscope("block") release, align 4
+ ret void
+}
+
+define void @release_i32_global_cluster(i32 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_i32_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_i32_global_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i32_global_cluster_param_1];
+; CHECK-NEXT: st.release.cluster.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr addrspace(1) %addr syncscope("cluster") release, align 4
+ ret void
+}
+
+define void @release_i32_global_gpu(i32 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_i32_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_i32_global_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i32_global_gpu_param_1];
+; CHECK-NEXT: st.release.gpu.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr addrspace(1) %addr syncscope("device") release, align 4
+ ret void
+}
+
+define void @release_i32_global_sys_volatile(i32 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_i32_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_i32_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i32_global_sys_volatile_param_1];
+; CHECK-NEXT: st.release.sys.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr addrspace(1) %addr syncscope("") release, align 4
+ ret void
+}
+
+define void @release_i32_global_cta_volatile(i32 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_i32_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_i32_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i32_global_cta_volatile_param_1];
+; CHECK-NEXT: st.release.sys.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr addrspace(1) %addr syncscope("block") release, align 4
+ ret void
+}
+
+define void @release_i32_global_cluster_volatile(i32 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_i32_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_i32_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i32_global_cluster_volatile_param_1];
+; CHECK-NEXT: st.release.sys.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr addrspace(1) %addr syncscope("cluster") release, align 4
+ ret void
+}
+
+define void @release_i32_global_gpu_volatile(i32 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_i32_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_i32_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i32_global_gpu_volatile_param_1];
+; CHECK-NEXT: st.release.sys.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr addrspace(1) %addr syncscope("device") release, align 4
+ ret void
+}
+
+define void @release_i64_global_sys(i64 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_i64_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i64_global_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [release_i64_global_sys_param_1];
+; CHECK-NEXT: st.release.sys.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr addrspace(1) %addr syncscope("") release, align 8
+ ret void
+}
+
+define void @release_i64_global_cta(i64 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_i64_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i64_global_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [release_i64_global_cta_param_1];
+; CHECK-NEXT: st.release.cta.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr addrspace(1) %addr syncscope("block") release, align 8
+ ret void
+}
+
+define void @release_i64_global_cluster(i64 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_i64_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i64_global_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [release_i64_global_cluster_param_1];
+; CHECK-NEXT: st.release.cluster.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr addrspace(1) %addr syncscope("cluster") release, align 8
+ ret void
+}
+
+define void @release_i64_global_gpu(i64 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_i64_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i64_global_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [release_i64_global_gpu_param_1];
+; CHECK-NEXT: st.release.gpu.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr addrspace(1) %addr syncscope("device") release, align 8
+ ret void
+}
+
+define void @release_i64_global_sys_volatile(i64 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_i64_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i64_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [release_i64_global_sys_volatile_param_1];
+; CHECK-NEXT: st.release.sys.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr addrspace(1) %addr syncscope("") release, align 8
+ ret void
+}
+
+define void @release_i64_global_cta_volatile(i64 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_i64_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i64_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [release_i64_global_cta_volatile_param_1];
+; CHECK-NEXT: st.release.sys.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr addrspace(1) %addr syncscope("block") release, align 8
+ ret void
+}
+
+define void @release_i64_global_cluster_volatile(i64 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_i64_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i64_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [release_i64_global_cluster_volatile_param_1];
+; CHECK-NEXT: st.release.sys.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr addrspace(1) %addr syncscope("cluster") release, align 8
+ ret void
+}
+
+define void @release_i64_global_gpu_volatile(i64 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_i64_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i64_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [release_i64_global_gpu_volatile_param_1];
+; CHECK-NEXT: st.release.sys.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr addrspace(1) %addr syncscope("device") release, align 8
+ ret void
+}
+
+define void @release_i128_global_sys(i128 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_i128_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [release_i128_global_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd3, [release_i128_global_sys_param_1];
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 amt, dst;
+; CHECK-NEXT: mov.b128 amt, {%rd1, %rd2};
+; CHECK-NEXT: atom.release.sys.global.exch.b128 dst, [%rd3], amt;
+; CHECK-NEXT: mov.b128 {%rd4, %rd5}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: ret;
+ store atomic i128 %val, ptr addrspace(1) %addr syncscope("") release, align 16
+ ret void
+}
+
+define void @release_i128_global_cta(i128 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_i128_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [release_i128_global_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd3, [release_i128_global_cta_param_1];
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 amt, dst;
+; CHECK-NEXT: mov.b128 amt, {%rd1, %rd2};
+; CHECK-NEXT: atom.release.cta.global.exch.b128 dst, [%rd3], amt;
+; CHECK-NEXT: mov.b128 {%rd4, %rd5}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: ret;
+ store atomic i128 %val, ptr addrspace(1) %addr syncscope("block") release, align 16
+ ret void
+}
+
+define void @release_i128_global_cluster(i128 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_i128_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [release_i128_global_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd3, [release_i128_global_cluster_param_1];
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 amt, dst;
+; CHECK-NEXT: mov.b128 amt, {%rd1, %rd2};
+; CHECK-NEXT: atom.release.cluster.global.exch.b128 dst, [%rd3], amt;
+; CHECK-NEXT: mov.b128 {%rd4, %rd5}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: ret;
+ store atomic i128 %val, ptr addrspace(1) %addr syncscope("cluster") release, align 16
+ ret void
+}
+
+define void @release_i128_global_gpu(i128 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_i128_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [release_i128_global_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd3, [release_i128_global_gpu_param_1];
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 amt, dst;
+; CHECK-NEXT: mov.b128 amt, {%rd1, %rd2};
+; CHECK-NEXT: atom.release.gpu.global.exch.b128 dst, [%rd3], amt;
+; CHECK-NEXT: mov.b128 {%rd4, %rd5}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: ret;
+ store atomic i128 %val, ptr addrspace(1) %addr syncscope("device") release, align 16
+ ret void
+}
+
+define void @release_i128_global_sys_volatile(i128 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_i128_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [release_i128_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd3, [release_i128_global_sys_volatile_param_1];
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 amt, dst;
+; CHECK-NEXT: mov.b128 amt, {%rd1, %rd2};
+; CHECK-NEXT: atom.release.sys.global.exch.b128 dst, [%rd3], amt;
+; CHECK-NEXT: mov.b128 {%rd4, %rd5}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: ret;
+ store atomic volatile i128 %val, ptr addrspace(1) %addr syncscope("") release, align 16
+ ret void
+}
+
+define void @release_i128_global_cta_volatile(i128 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_i128_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [release_i128_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd3, [release_i128_global_cta_volatile_param_1];
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 amt, dst;
+; CHECK-NEXT: mov.b128 amt, {%rd1, %rd2};
+; CHECK-NEXT: atom.release.cta.global.exch.b128 dst, [%rd3], amt;
+; CHECK-NEXT: mov.b128 {%rd4, %rd5}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: ret;
+ store atomic volatile i128 %val, ptr addrspace(1) %addr syncscope("block") release, align 16
+ ret void
+}
+
+define void @release_i128_global_cluster_volatile(i128 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_i128_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [release_i128_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd3, [release_i128_global_cluster_volatile_param_1];
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 amt, dst;
+; CHECK-NEXT: mov.b128 amt, {%rd1, %rd2};
+; CHECK-NEXT: atom.release.cluster.global.exch.b128 dst, [%rd3], amt;
+; CHECK-NEXT: mov.b128 {%rd4, %rd5}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: ret;
+ store atomic volatile i128 %val, ptr addrspace(1) %addr syncscope("cluster") release, align 16
+ ret void
+}
+
+define void @release_i128_global_gpu_volatile(i128 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_i128_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [release_i128_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd3, [release_i128_global_gpu_volatile_param_1];
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 amt, dst;
+; CHECK-NEXT: mov.b128 amt, {%rd1, %rd2};
+; CHECK-NEXT: atom.release.gpu.global.exch.b128 dst, [%rd3], amt;
+; CHECK-NEXT: mov.b128 {%rd4, %rd5}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: ret;
+ store atomic volatile i128 %val, ptr addrspace(1) %addr syncscope("device") release, align 16
+ ret void
+}
+
+define void @release_half_global_sys(half %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_half_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_half_global_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_half_global_sys_param_1];
+; CHECK-NEXT: st.release.sys.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr addrspace(1) %addr syncscope("") release, align 2
+ ret void
+}
+
+define void @release_half_global_cta(half %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_half_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_half_global_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_half_global_cta_param_1];
+; CHECK-NEXT: st.release.cta.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr addrspace(1) %addr syncscope("block") release, align 2
+ ret void
+}
+
+define void @release_half_global_cluster(half %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_half_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_half_global_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_half_global_cluster_param_1];
+; CHECK-NEXT: st.release.cluster.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr addrspace(1) %addr syncscope("cluster") release, align 2
+ ret void
+}
+
+define void @release_half_global_gpu(half %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_half_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_half_global_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_half_global_gpu_param_1];
+; CHECK-NEXT: st.release.gpu.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr addrspace(1) %addr syncscope("device") release, align 2
+ ret void
+}
+
+define void @release_half_global_sys_volatile(half %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_half_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_half_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_half_global_sys_volatile_param_1];
+; CHECK-NEXT: st.release.sys.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr addrspace(1) %addr syncscope("") release, align 2
+ ret void
+}
+
+define void @release_half_global_cta_volatile(half %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_half_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_half_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_half_global_cta_volatile_param_1];
+; CHECK-NEXT: st.release.sys.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr addrspace(1) %addr syncscope("block") release, align 2
+ ret void
+}
+
+define void @release_half_global_cluster_volatile(half %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_half_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_half_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_half_global_cluster_volatile_param_1];
+; CHECK-NEXT: st.release.sys.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr addrspace(1) %addr syncscope("cluster") release, align 2
+ ret void
+}
+
+define void @release_half_global_gpu_volatile(half %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_half_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_half_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_half_global_gpu_volatile_param_1];
+; CHECK-NEXT: st.release.sys.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr addrspace(1) %addr syncscope("device") release, align 2
+ ret void
+}
+
+define void @release_bfloat_global_sys(bfloat %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_bfloat_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_bfloat_global_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_bfloat_global_sys_param_1];
+; CHECK-NEXT: st.release.sys.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr addrspace(1) %addr syncscope("") release, align 2
+ ret void
+}
+
+define void @release_bfloat_global_cta(bfloat %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_bfloat_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_bfloat_global_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_bfloat_global_cta_param_1];
+; CHECK-NEXT: st.release.cta.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr addrspace(1) %addr syncscope("block") release, align 2
+ ret void
+}
+
+define void @release_bfloat_global_cluster(bfloat %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_bfloat_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_bfloat_global_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_bfloat_global_cluster_param_1];
+; CHECK-NEXT: st.release.cluster.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr addrspace(1) %addr syncscope("cluster") release, align 2
+ ret void
+}
+
+define void @release_bfloat_global_gpu(bfloat %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_bfloat_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_bfloat_global_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_bfloat_global_gpu_param_1];
+; CHECK-NEXT: st.release.gpu.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr addrspace(1) %addr syncscope("device") release, align 2
+ ret void
+}
+
+define void @release_bfloat_global_sys_volatile(bfloat %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_bfloat_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_bfloat_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_bfloat_global_sys_volatile_param_1];
+; CHECK-NEXT: st.release.sys.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr addrspace(1) %addr syncscope("") release, align 2
+ ret void
+}
+
+define void @release_bfloat_global_cta_volatile(bfloat %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_bfloat_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_bfloat_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_bfloat_global_cta_volatile_param_1];
+; CHECK-NEXT: st.release.sys.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr addrspace(1) %addr syncscope("block") release, align 2
+ ret void
+}
+
+define void @release_bfloat_global_cluster_volatile(bfloat %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_bfloat_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_bfloat_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_bfloat_global_cluster_volatile_param_1];
+; CHECK-NEXT: st.release.sys.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr addrspace(1) %addr syncscope("cluster") release, align 2
+ ret void
+}
+
+define void @release_bfloat_global_gpu_volatile(bfloat %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_bfloat_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_bfloat_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_bfloat_global_gpu_volatile_param_1];
+; CHECK-NEXT: st.release.sys.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr addrspace(1) %addr syncscope("device") release, align 2
+ ret void
+}
+
+define void @release_float_global_sys(float %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_float_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_float_global_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_float_global_sys_param_1];
+; CHECK-NEXT: st.release.sys.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr addrspace(1) %addr syncscope("") release, align 4
+ ret void
+}
+
+define void @release_float_global_cta(float %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_float_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_float_global_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_float_global_cta_param_1];
+; CHECK-NEXT: st.release.cta.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr addrspace(1) %addr syncscope("block") release, align 4
+ ret void
+}
+
+define void @release_float_global_cluster(float %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_float_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_float_global_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_float_global_cluster_param_1];
+; CHECK-NEXT: st.release.cluster.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr addrspace(1) %addr syncscope("cluster") release, align 4
+ ret void
+}
+
+define void @release_float_global_gpu(float %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_float_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_float_global_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_float_global_gpu_param_1];
+; CHECK-NEXT: st.release.gpu.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr addrspace(1) %addr syncscope("device") release, align 4
+ ret void
+}
+
+define void @release_float_global_sys_volatile(float %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_float_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_float_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_float_global_sys_volatile_param_1];
+; CHECK-NEXT: st.release.sys.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr addrspace(1) %addr syncscope("") release, align 4
+ ret void
+}
+
+define void @release_float_global_cta_volatile(float %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_float_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_float_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_float_global_cta_volatile_param_1];
+; CHECK-NEXT: st.release.sys.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr addrspace(1) %addr syncscope("block") release, align 4
+ ret void
+}
+
+define void @release_float_global_cluster_volatile(float %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_float_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_float_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_float_global_cluster_volatile_param_1];
+; CHECK-NEXT: st.release.sys.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr addrspace(1) %addr syncscope("cluster") release, align 4
+ ret void
+}
+
+define void @release_float_global_gpu_volatile(float %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_float_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_float_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_float_global_gpu_volatile_param_1];
+; CHECK-NEXT: st.release.sys.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr addrspace(1) %addr syncscope("device") release, align 4
+ ret void
+}
+
+define void @release_double_global_sys(double %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_double_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_double_global_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [release_double_global_sys_param_1];
+; CHECK-NEXT: st.release.sys.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr addrspace(1) %addr syncscope("") release, align 8
+ ret void
+}
+
+define void @release_double_global_cta(double %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_double_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_double_global_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [release_double_global_cta_param_1];
+; CHECK-NEXT: st.release.cta.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr addrspace(1) %addr syncscope("block") release, align 8
+ ret void
+}
+
+define void @release_double_global_cluster(double %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_double_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_double_global_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [release_double_global_cluster_param_1];
+; CHECK-NEXT: st.release.cluster.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr addrspace(1) %addr syncscope("cluster") release, align 8
+ ret void
+}
+
+define void @release_double_global_gpu(double %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_double_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_double_global_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [release_double_global_gpu_param_1];
+; CHECK-NEXT: st.release.gpu.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr addrspace(1) %addr syncscope("device") release, align 8
+ ret void
+}
+
+define void @release_double_global_sys_volatile(double %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_double_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_double_global_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [release_double_global_sys_volatile_param_1];
+; CHECK-NEXT: st.release.sys.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr addrspace(1) %addr syncscope("") release, align 8
+ ret void
+}
+
+define void @release_double_global_cta_volatile(double %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_double_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_double_global_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [release_double_global_cta_volatile_param_1];
+; CHECK-NEXT: st.release.sys.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr addrspace(1) %addr syncscope("block") release, align 8
+ ret void
+}
+
+define void @release_double_global_cluster_volatile(double %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_double_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_double_global_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [release_double_global_cluster_volatile_param_1];
+; CHECK-NEXT: st.release.sys.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr addrspace(1) %addr syncscope("cluster") release, align 8
+ ret void
+}
+
+define void @release_double_global_gpu_volatile(double %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: release_double_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_double_global_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [release_double_global_gpu_volatile_param_1];
+; CHECK-NEXT: st.release.sys.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr addrspace(1) %addr syncscope("device") release, align 8
+ ret void
+}
+
+define void @release_i8_shared_sys(i8 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_i8_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [release_i8_shared_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i8_shared_sys_param_1];
+; CHECK-NEXT: st.release.sys.shared.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr addrspace(3) %addr syncscope("") release, align 1
+ ret void
+}
+
+define void @release_i8_shared_cta(i8 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_i8_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [release_i8_shared_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i8_shared_cta_param_1];
+; CHECK-NEXT: st.release.cta.shared.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr addrspace(3) %addr syncscope("block") release, align 1
+ ret void
+}
+
+define void @release_i8_shared_cluster(i8 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_i8_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [release_i8_shared_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i8_shared_cluster_param_1];
+; CHECK-NEXT: st.release.cluster.shared.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr addrspace(3) %addr syncscope("cluster") release, align 1
+ ret void
+}
+
+define void @release_i8_shared_gpu(i8 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_i8_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [release_i8_shared_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i8_shared_gpu_param_1];
+; CHECK-NEXT: st.release.gpu.shared.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr addrspace(3) %addr syncscope("device") release, align 1
+ ret void
+}
+
+define void @release_i8_shared_sys_volatile(i8 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_i8_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [release_i8_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i8_shared_sys_volatile_param_1];
+; CHECK-NEXT: st.release.sys.shared.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr addrspace(3) %addr syncscope("") release, align 1
+ ret void
+}
+
+define void @release_i8_shared_cta_volatile(i8 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_i8_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [release_i8_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i8_shared_cta_volatile_param_1];
+; CHECK-NEXT: st.release.sys.shared.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr addrspace(3) %addr syncscope("block") release, align 1
+ ret void
+}
+
+define void @release_i8_shared_cluster_volatile(i8 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_i8_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [release_i8_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i8_shared_cluster_volatile_param_1];
+; CHECK-NEXT: st.release.sys.shared.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr addrspace(3) %addr syncscope("cluster") release, align 1
+ ret void
+}
+
+define void @release_i8_shared_gpu_volatile(i8 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_i8_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [release_i8_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i8_shared_gpu_volatile_param_1];
+; CHECK-NEXT: st.release.sys.shared.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr addrspace(3) %addr syncscope("device") release, align 1
+ ret void
+}
+
+define void @release_i16_shared_sys(i16 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_i16_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_i16_shared_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i16_shared_sys_param_1];
+; CHECK-NEXT: st.release.sys.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr addrspace(3) %addr syncscope("") release, align 2
+ ret void
+}
+
+define void @release_i16_shared_cta(i16 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_i16_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_i16_shared_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i16_shared_cta_param_1];
+; CHECK-NEXT: st.release.cta.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr addrspace(3) %addr syncscope("block") release, align 2
+ ret void
+}
+
+define void @release_i16_shared_cluster(i16 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_i16_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_i16_shared_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i16_shared_cluster_param_1];
+; CHECK-NEXT: st.release.cluster.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr addrspace(3) %addr syncscope("cluster") release, align 2
+ ret void
+}
+
+define void @release_i16_shared_gpu(i16 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_i16_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_i16_shared_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i16_shared_gpu_param_1];
+; CHECK-NEXT: st.release.gpu.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr addrspace(3) %addr syncscope("device") release, align 2
+ ret void
+}
+
+define void @release_i16_shared_sys_volatile(i16 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_i16_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_i16_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i16_shared_sys_volatile_param_1];
+; CHECK-NEXT: st.release.sys.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr addrspace(3) %addr syncscope("") release, align 2
+ ret void
+}
+
+define void @release_i16_shared_cta_volatile(i16 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_i16_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_i16_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i16_shared_cta_volatile_param_1];
+; CHECK-NEXT: st.release.sys.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr addrspace(3) %addr syncscope("block") release, align 2
+ ret void
+}
+
+define void @release_i16_shared_cluster_volatile(i16 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_i16_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_i16_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i16_shared_cluster_volatile_param_1];
+; CHECK-NEXT: st.release.sys.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr addrspace(3) %addr syncscope("cluster") release, align 2
+ ret void
+}
+
+define void @release_i16_shared_gpu_volatile(i16 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_i16_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_i16_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i16_shared_gpu_volatile_param_1];
+; CHECK-NEXT: st.release.sys.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr addrspace(3) %addr syncscope("device") release, align 2
+ ret void
+}
+
+define void @release_i32_shared_sys(i32 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_i32_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_i32_shared_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i32_shared_sys_param_1];
+; CHECK-NEXT: st.release.sys.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr addrspace(3) %addr syncscope("") release, align 4
+ ret void
+}
+
+define void @release_i32_shared_cta(i32 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_i32_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_i32_shared_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i32_shared_cta_param_1];
+; CHECK-NEXT: st.release.cta.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr addrspace(3) %addr syncscope("block") release, align 4
+ ret void
+}
+
+define void @release_i32_shared_cluster(i32 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_i32_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_i32_shared_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i32_shared_cluster_param_1];
+; CHECK-NEXT: st.release.cluster.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr addrspace(3) %addr syncscope("cluster") release, align 4
+ ret void
+}
+
+define void @release_i32_shared_gpu(i32 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_i32_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_i32_shared_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i32_shared_gpu_param_1];
+; CHECK-NEXT: st.release.gpu.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr addrspace(3) %addr syncscope("device") release, align 4
+ ret void
+}
+
+define void @release_i32_shared_sys_volatile(i32 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_i32_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_i32_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i32_shared_sys_volatile_param_1];
+; CHECK-NEXT: st.release.sys.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr addrspace(3) %addr syncscope("") release, align 4
+ ret void
+}
+
+define void @release_i32_shared_cta_volatile(i32 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_i32_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_i32_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i32_shared_cta_volatile_param_1];
+; CHECK-NEXT: st.release.sys.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr addrspace(3) %addr syncscope("block") release, align 4
+ ret void
+}
+
+define void @release_i32_shared_cluster_volatile(i32 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_i32_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_i32_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i32_shared_cluster_volatile_param_1];
+; CHECK-NEXT: st.release.sys.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr addrspace(3) %addr syncscope("cluster") release, align 4
+ ret void
+}
+
+define void @release_i32_shared_gpu_volatile(i32 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_i32_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_i32_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i32_shared_gpu_volatile_param_1];
+; CHECK-NEXT: st.release.sys.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr addrspace(3) %addr syncscope("device") release, align 4
+ ret void
+}
+
+define void @release_i64_shared_sys(i64 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_i64_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i64_shared_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [release_i64_shared_sys_param_1];
+; CHECK-NEXT: st.release.sys.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr addrspace(3) %addr syncscope("") release, align 8
+ ret void
+}
+
+define void @release_i64_shared_cta(i64 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_i64_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i64_shared_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [release_i64_shared_cta_param_1];
+; CHECK-NEXT: st.release.cta.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr addrspace(3) %addr syncscope("block") release, align 8
+ ret void
+}
+
+define void @release_i64_shared_cluster(i64 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_i64_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i64_shared_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [release_i64_shared_cluster_param_1];
+; CHECK-NEXT: st.release.cluster.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr addrspace(3) %addr syncscope("cluster") release, align 8
+ ret void
+}
+
+define void @release_i64_shared_gpu(i64 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_i64_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i64_shared_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [release_i64_shared_gpu_param_1];
+; CHECK-NEXT: st.release.gpu.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr addrspace(3) %addr syncscope("device") release, align 8
+ ret void
+}
+
+define void @release_i64_shared_sys_volatile(i64 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_i64_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i64_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [release_i64_shared_sys_volatile_param_1];
+; CHECK-NEXT: st.release.sys.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr addrspace(3) %addr syncscope("") release, align 8
+ ret void
+}
+
+define void @release_i64_shared_cta_volatile(i64 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_i64_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i64_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [release_i64_shared_cta_volatile_param_1];
+; CHECK-NEXT: st.release.sys.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr addrspace(3) %addr syncscope("block") release, align 8
+ ret void
+}
+
+define void @release_i64_shared_cluster_volatile(i64 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_i64_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i64_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [release_i64_shared_cluster_volatile_param_1];
+; CHECK-NEXT: st.release.sys.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr addrspace(3) %addr syncscope("cluster") release, align 8
+ ret void
+}
+
+define void @release_i64_shared_gpu_volatile(i64 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_i64_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i64_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [release_i64_shared_gpu_volatile_param_1];
+; CHECK-NEXT: st.release.sys.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr addrspace(3) %addr syncscope("device") release, align 8
+ ret void
+}
+
+define void @release_i128_shared_sys(i128 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_i128_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [release_i128_shared_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd3, [release_i128_shared_sys_param_1];
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 amt, dst;
+; CHECK-NEXT: mov.b128 amt, {%rd1, %rd2};
+; CHECK-NEXT: atom.release.sys.shared.exch.b128 dst, [%rd3], amt;
+; CHECK-NEXT: mov.b128 {%rd4, %rd5}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: ret;
+ store atomic i128 %val, ptr addrspace(3) %addr syncscope("") release, align 16
+ ret void
+}
+
+define void @release_i128_shared_cta(i128 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_i128_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [release_i128_shared_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd3, [release_i128_shared_cta_param_1];
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 amt, dst;
+; CHECK-NEXT: mov.b128 amt, {%rd1, %rd2};
+; CHECK-NEXT: atom.release.cta.shared.exch.b128 dst, [%rd3], amt;
+; CHECK-NEXT: mov.b128 {%rd4, %rd5}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: ret;
+ store atomic i128 %val, ptr addrspace(3) %addr syncscope("block") release, align 16
+ ret void
+}
+
+define void @release_i128_shared_cluster(i128 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_i128_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [release_i128_shared_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd3, [release_i128_shared_cluster_param_1];
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 amt, dst;
+; CHECK-NEXT: mov.b128 amt, {%rd1, %rd2};
+; CHECK-NEXT: atom.release.cluster.shared.exch.b128 dst, [%rd3], amt;
+; CHECK-NEXT: mov.b128 {%rd4, %rd5}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: ret;
+ store atomic i128 %val, ptr addrspace(3) %addr syncscope("cluster") release, align 16
+ ret void
+}
+
+define void @release_i128_shared_gpu(i128 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_i128_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [release_i128_shared_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd3, [release_i128_shared_gpu_param_1];
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 amt, dst;
+; CHECK-NEXT: mov.b128 amt, {%rd1, %rd2};
+; CHECK-NEXT: atom.release.gpu.shared.exch.b128 dst, [%rd3], amt;
+; CHECK-NEXT: mov.b128 {%rd4, %rd5}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: ret;
+ store atomic i128 %val, ptr addrspace(3) %addr syncscope("device") release, align 16
+ ret void
+}
+
+define void @release_i128_shared_sys_volatile(i128 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_i128_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [release_i128_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd3, [release_i128_shared_sys_volatile_param_1];
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 amt, dst;
+; CHECK-NEXT: mov.b128 amt, {%rd1, %rd2};
+; CHECK-NEXT: atom.release.sys.shared.exch.b128 dst, [%rd3], amt;
+; CHECK-NEXT: mov.b128 {%rd4, %rd5}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: ret;
+ store atomic volatile i128 %val, ptr addrspace(3) %addr syncscope("") release, align 16
+ ret void
+}
+
+define void @release_i128_shared_cta_volatile(i128 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_i128_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [release_i128_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd3, [release_i128_shared_cta_volatile_param_1];
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 amt, dst;
+; CHECK-NEXT: mov.b128 amt, {%rd1, %rd2};
+; CHECK-NEXT: atom.release.cta.shared.exch.b128 dst, [%rd3], amt;
+; CHECK-NEXT: mov.b128 {%rd4, %rd5}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: ret;
+ store atomic volatile i128 %val, ptr addrspace(3) %addr syncscope("block") release, align 16
+ ret void
+}
+
+define void @release_i128_shared_cluster_volatile(i128 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_i128_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [release_i128_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd3, [release_i128_shared_cluster_volatile_param_1];
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 amt, dst;
+; CHECK-NEXT: mov.b128 amt, {%rd1, %rd2};
+; CHECK-NEXT: atom.release.cluster.shared.exch.b128 dst, [%rd3], amt;
+; CHECK-NEXT: mov.b128 {%rd4, %rd5}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: ret;
+ store atomic volatile i128 %val, ptr addrspace(3) %addr syncscope("cluster") release, align 16
+ ret void
+}
+
+define void @release_i128_shared_gpu_volatile(i128 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_i128_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [release_i128_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd3, [release_i128_shared_gpu_volatile_param_1];
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 amt, dst;
+; CHECK-NEXT: mov.b128 amt, {%rd1, %rd2};
+; CHECK-NEXT: atom.release.gpu.shared.exch.b128 dst, [%rd3], amt;
+; CHECK-NEXT: mov.b128 {%rd4, %rd5}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: ret;
+ store atomic volatile i128 %val, ptr addrspace(3) %addr syncscope("device") release, align 16
+ ret void
+}
+
+define void @release_half_shared_sys(half %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_half_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_half_shared_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_half_shared_sys_param_1];
+; CHECK-NEXT: st.release.sys.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr addrspace(3) %addr syncscope("") release, align 2
+ ret void
+}
+
+define void @release_half_shared_cta(half %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_half_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_half_shared_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_half_shared_cta_param_1];
+; CHECK-NEXT: st.release.cta.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr addrspace(3) %addr syncscope("block") release, align 2
+ ret void
+}
+
+define void @release_half_shared_cluster(half %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_half_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_half_shared_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_half_shared_cluster_param_1];
+; CHECK-NEXT: st.release.cluster.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr addrspace(3) %addr syncscope("cluster") release, align 2
+ ret void
+}
+
+define void @release_half_shared_gpu(half %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_half_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_half_shared_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_half_shared_gpu_param_1];
+; CHECK-NEXT: st.release.gpu.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr addrspace(3) %addr syncscope("device") release, align 2
+ ret void
+}
+
+define void @release_half_shared_sys_volatile(half %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_half_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_half_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_half_shared_sys_volatile_param_1];
+; CHECK-NEXT: st.release.sys.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr addrspace(3) %addr syncscope("") release, align 2
+ ret void
+}
+
+define void @release_half_shared_cta_volatile(half %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_half_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_half_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_half_shared_cta_volatile_param_1];
+; CHECK-NEXT: st.release.sys.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr addrspace(3) %addr syncscope("block") release, align 2
+ ret void
+}
+
+define void @release_half_shared_cluster_volatile(half %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_half_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_half_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_half_shared_cluster_volatile_param_1];
+; CHECK-NEXT: st.release.sys.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr addrspace(3) %addr syncscope("cluster") release, align 2
+ ret void
+}
+
+define void @release_half_shared_gpu_volatile(half %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_half_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_half_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_half_shared_gpu_volatile_param_1];
+; CHECK-NEXT: st.release.sys.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr addrspace(3) %addr syncscope("device") release, align 2
+ ret void
+}
+
+define void @release_bfloat_shared_sys(bfloat %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_bfloat_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_bfloat_shared_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_bfloat_shared_sys_param_1];
+; CHECK-NEXT: st.release.sys.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr addrspace(3) %addr syncscope("") release, align 2
+ ret void
+}
+
+define void @release_bfloat_shared_cta(bfloat %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_bfloat_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_bfloat_shared_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_bfloat_shared_cta_param_1];
+; CHECK-NEXT: st.release.cta.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr addrspace(3) %addr syncscope("block") release, align 2
+ ret void
+}
+
+define void @release_bfloat_shared_cluster(bfloat %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_bfloat_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_bfloat_shared_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_bfloat_shared_cluster_param_1];
+; CHECK-NEXT: st.release.cluster.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr addrspace(3) %addr syncscope("cluster") release, align 2
+ ret void
+}
+
+define void @release_bfloat_shared_gpu(bfloat %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_bfloat_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_bfloat_shared_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_bfloat_shared_gpu_param_1];
+; CHECK-NEXT: st.release.gpu.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr addrspace(3) %addr syncscope("device") release, align 2
+ ret void
+}
+
+define void @release_bfloat_shared_sys_volatile(bfloat %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_bfloat_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_bfloat_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_bfloat_shared_sys_volatile_param_1];
+; CHECK-NEXT: st.release.sys.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr addrspace(3) %addr syncscope("") release, align 2
+ ret void
+}
+
+define void @release_bfloat_shared_cta_volatile(bfloat %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_bfloat_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_bfloat_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_bfloat_shared_cta_volatile_param_1];
+; CHECK-NEXT: st.release.sys.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr addrspace(3) %addr syncscope("block") release, align 2
+ ret void
+}
+
+define void @release_bfloat_shared_cluster_volatile(bfloat %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_bfloat_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_bfloat_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_bfloat_shared_cluster_volatile_param_1];
+; CHECK-NEXT: st.release.sys.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr addrspace(3) %addr syncscope("cluster") release, align 2
+ ret void
+}
+
+define void @release_bfloat_shared_gpu_volatile(bfloat %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_bfloat_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [release_bfloat_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_bfloat_shared_gpu_volatile_param_1];
+; CHECK-NEXT: st.release.sys.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr addrspace(3) %addr syncscope("device") release, align 2
+ ret void
+}
+
+define void @release_float_shared_sys(float %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_float_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_float_shared_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_float_shared_sys_param_1];
+; CHECK-NEXT: st.release.sys.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr addrspace(3) %addr syncscope("") release, align 4
+ ret void
+}
+
+define void @release_float_shared_cta(float %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_float_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_float_shared_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_float_shared_cta_param_1];
+; CHECK-NEXT: st.release.cta.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr addrspace(3) %addr syncscope("block") release, align 4
+ ret void
+}
+
+define void @release_float_shared_cluster(float %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_float_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_float_shared_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_float_shared_cluster_param_1];
+; CHECK-NEXT: st.release.cluster.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr addrspace(3) %addr syncscope("cluster") release, align 4
+ ret void
+}
+
+define void @release_float_shared_gpu(float %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_float_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_float_shared_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_float_shared_gpu_param_1];
+; CHECK-NEXT: st.release.gpu.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr addrspace(3) %addr syncscope("device") release, align 4
+ ret void
+}
+
+define void @release_float_shared_sys_volatile(float %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_float_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_float_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_float_shared_sys_volatile_param_1];
+; CHECK-NEXT: st.release.sys.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr addrspace(3) %addr syncscope("") release, align 4
+ ret void
+}
+
+define void @release_float_shared_cta_volatile(float %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_float_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_float_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_float_shared_cta_volatile_param_1];
+; CHECK-NEXT: st.release.sys.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr addrspace(3) %addr syncscope("block") release, align 4
+ ret void
+}
+
+define void @release_float_shared_cluster_volatile(float %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_float_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_float_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_float_shared_cluster_volatile_param_1];
+; CHECK-NEXT: st.release.sys.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr addrspace(3) %addr syncscope("cluster") release, align 4
+ ret void
+}
+
+define void @release_float_shared_gpu_volatile(float %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_float_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [release_float_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd1, [release_float_shared_gpu_volatile_param_1];
+; CHECK-NEXT: st.release.sys.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr addrspace(3) %addr syncscope("device") release, align 4
+ ret void
+}
+
+define void @release_double_shared_sys(double %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_double_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_double_shared_sys_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [release_double_shared_sys_param_1];
+; CHECK-NEXT: st.release.sys.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr addrspace(3) %addr syncscope("") release, align 8
+ ret void
+}
+
+define void @release_double_shared_cta(double %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_double_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_double_shared_cta_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [release_double_shared_cta_param_1];
+; CHECK-NEXT: st.release.cta.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr addrspace(3) %addr syncscope("block") release, align 8
+ ret void
+}
+
+define void @release_double_shared_cluster(double %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_double_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_double_shared_cluster_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [release_double_shared_cluster_param_1];
+; CHECK-NEXT: st.release.cluster.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr addrspace(3) %addr syncscope("cluster") release, align 8
+ ret void
+}
+
+define void @release_double_shared_gpu(double %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_double_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_double_shared_gpu_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [release_double_shared_gpu_param_1];
+; CHECK-NEXT: st.release.gpu.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr addrspace(3) %addr syncscope("device") release, align 8
+ ret void
+}
+
+define void @release_double_shared_sys_volatile(double %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_double_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_double_shared_sys_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [release_double_shared_sys_volatile_param_1];
+; CHECK-NEXT: st.release.sys.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr addrspace(3) %addr syncscope("") release, align 8
+ ret void
+}
+
+define void @release_double_shared_cta_volatile(double %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_double_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_double_shared_cta_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [release_double_shared_cta_volatile_param_1];
+; CHECK-NEXT: st.release.sys.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr addrspace(3) %addr syncscope("block") release, align 8
+ ret void
+}
+
+define void @release_double_shared_cluster_volatile(double %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_double_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_double_shared_cluster_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [release_double_shared_cluster_volatile_param_1];
+; CHECK-NEXT: st.release.sys.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr addrspace(3) %addr syncscope("cluster") release, align 8
+ ret void
+}
+
+define void @release_double_shared_gpu_volatile(double %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: release_double_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [release_double_shared_gpu_volatile_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [release_double_shared_gpu_volatile_param_1];
+; CHECK-NEXT: st.release.sys.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr addrspace(3) %addr syncscope("device") release, align 8
+ ret void
+}
+
+define void @release_i8_local_sys(i8 %val) {
+; CHECK-LABEL: release_i8_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot864[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot864;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b8 %rs1, [release_i8_local_sys_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b8 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i8 %val, ptr addrspace(5) %slot syncscope("") release, align 1
+ ret void
+}
+
+define void @release_i8_local_cta(i8 %val) {
+; CHECK-LABEL: release_i8_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot865[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot865;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b8 %rs1, [release_i8_local_cta_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b8 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i8 %val, ptr addrspace(5) %slot syncscope("block") release, align 1
+ ret void
+}
+
+define void @release_i8_local_cluster(i8 %val) {
+; CHECK-LABEL: release_i8_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot866[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot866;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b8 %rs1, [release_i8_local_cluster_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b8 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i8 %val, ptr addrspace(5) %slot syncscope("cluster") release, align 1
+ ret void
+}
+
+define void @release_i8_local_gpu(i8 %val) {
+; CHECK-LABEL: release_i8_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot867[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot867;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b8 %rs1, [release_i8_local_gpu_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b8 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i8 %val, ptr addrspace(5) %slot syncscope("device") release, align 1
+ ret void
+}
+
+define void @release_i8_local_sys_volatile(i8 %val) {
+; CHECK-LABEL: release_i8_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot868[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot868;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b8 %rs1, [release_i8_local_sys_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b8 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i8 %val, ptr addrspace(5) %slot syncscope("") release, align 1
+ ret void
+}
+
+define void @release_i8_local_cta_volatile(i8 %val) {
+; CHECK-LABEL: release_i8_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot869[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot869;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b8 %rs1, [release_i8_local_cta_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b8 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i8 %val, ptr addrspace(5) %slot syncscope("block") release, align 1
+ ret void
+}
+
+define void @release_i8_local_cluster_volatile(i8 %val) {
+; CHECK-LABEL: release_i8_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot870[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot870;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b8 %rs1, [release_i8_local_cluster_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b8 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i8 %val, ptr addrspace(5) %slot syncscope("cluster") release, align 1
+ ret void
+}
+
+define void @release_i8_local_gpu_volatile(i8 %val) {
+; CHECK-LABEL: release_i8_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot871[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot871;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b8 %rs1, [release_i8_local_gpu_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b8 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i8 %val, ptr addrspace(5) %slot syncscope("device") release, align 1
+ ret void
+}
+
+define void @release_i16_local_sys(i16 %val) {
+; CHECK-LABEL: release_i16_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot872[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot872;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [release_i16_local_sys_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i16 %val, ptr addrspace(5) %slot syncscope("") release, align 2
+ ret void
+}
+
+define void @release_i16_local_cta(i16 %val) {
+; CHECK-LABEL: release_i16_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot873[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot873;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [release_i16_local_cta_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i16 %val, ptr addrspace(5) %slot syncscope("block") release, align 2
+ ret void
+}
+
+define void @release_i16_local_cluster(i16 %val) {
+; CHECK-LABEL: release_i16_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot874[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot874;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [release_i16_local_cluster_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i16 %val, ptr addrspace(5) %slot syncscope("cluster") release, align 2
+ ret void
+}
+
+define void @release_i16_local_gpu(i16 %val) {
+; CHECK-LABEL: release_i16_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot875[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot875;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [release_i16_local_gpu_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i16 %val, ptr addrspace(5) %slot syncscope("device") release, align 2
+ ret void
+}
+
+define void @release_i16_local_sys_volatile(i16 %val) {
+; CHECK-LABEL: release_i16_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot876[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot876;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [release_i16_local_sys_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i16 %val, ptr addrspace(5) %slot syncscope("") release, align 2
+ ret void
+}
+
+define void @release_i16_local_cta_volatile(i16 %val) {
+; CHECK-LABEL: release_i16_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot877[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot877;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [release_i16_local_cta_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i16 %val, ptr addrspace(5) %slot syncscope("block") release, align 2
+ ret void
+}
+
+define void @release_i16_local_cluster_volatile(i16 %val) {
+; CHECK-LABEL: release_i16_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot878[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot878;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [release_i16_local_cluster_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i16 %val, ptr addrspace(5) %slot syncscope("cluster") release, align 2
+ ret void
+}
+
+define void @release_i16_local_gpu_volatile(i16 %val) {
+; CHECK-LABEL: release_i16_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot879[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot879;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [release_i16_local_gpu_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i16 %val, ptr addrspace(5) %slot syncscope("device") release, align 2
+ ret void
+}
+
+define void @release_i32_local_sys(i32 %val) {
+; CHECK-LABEL: release_i32_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot880[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot880;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [release_i32_local_sys_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i32 %val, ptr addrspace(5) %slot syncscope("") release, align 4
+ ret void
+}
+
+define void @release_i32_local_cta(i32 %val) {
+; CHECK-LABEL: release_i32_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot881[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot881;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [release_i32_local_cta_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i32 %val, ptr addrspace(5) %slot syncscope("block") release, align 4
+ ret void
+}
+
+define void @release_i32_local_cluster(i32 %val) {
+; CHECK-LABEL: release_i32_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot882[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot882;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [release_i32_local_cluster_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i32 %val, ptr addrspace(5) %slot syncscope("cluster") release, align 4
+ ret void
+}
+
+define void @release_i32_local_gpu(i32 %val) {
+; CHECK-LABEL: release_i32_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot883[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot883;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [release_i32_local_gpu_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i32 %val, ptr addrspace(5) %slot syncscope("device") release, align 4
+ ret void
+}
+
+define void @release_i32_local_sys_volatile(i32 %val) {
+; CHECK-LABEL: release_i32_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot884[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot884;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [release_i32_local_sys_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i32 %val, ptr addrspace(5) %slot syncscope("") release, align 4
+ ret void
+}
+
+define void @release_i32_local_cta_volatile(i32 %val) {
+; CHECK-LABEL: release_i32_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot885[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot885;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [release_i32_local_cta_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i32 %val, ptr addrspace(5) %slot syncscope("block") release, align 4
+ ret void
+}
+
+define void @release_i32_local_cluster_volatile(i32 %val) {
+; CHECK-LABEL: release_i32_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot886[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot886;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [release_i32_local_cluster_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i32 %val, ptr addrspace(5) %slot syncscope("cluster") release, align 4
+ ret void
+}
+
+define void @release_i32_local_gpu_volatile(i32 %val) {
+; CHECK-LABEL: release_i32_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot887[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot887;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [release_i32_local_gpu_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i32 %val, ptr addrspace(5) %slot syncscope("device") release, align 4
+ ret void
+}
+
+define void @release_i64_local_sys(i64 %val) {
+; CHECK-LABEL: release_i64_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot888[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot888;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i64_local_sys_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i64 %val, ptr addrspace(5) %slot syncscope("") release, align 8
+ ret void
+}
+
+define void @release_i64_local_cta(i64 %val) {
+; CHECK-LABEL: release_i64_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot889[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot889;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i64_local_cta_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i64 %val, ptr addrspace(5) %slot syncscope("block") release, align 8
+ ret void
+}
+
+define void @release_i64_local_cluster(i64 %val) {
+; CHECK-LABEL: release_i64_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot890[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot890;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i64_local_cluster_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i64 %val, ptr addrspace(5) %slot syncscope("cluster") release, align 8
+ ret void
+}
+
+define void @release_i64_local_gpu(i64 %val) {
+; CHECK-LABEL: release_i64_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot891[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot891;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i64_local_gpu_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i64 %val, ptr addrspace(5) %slot syncscope("device") release, align 8
+ ret void
+}
+
+define void @release_i64_local_sys_volatile(i64 %val) {
+; CHECK-LABEL: release_i64_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot892[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot892;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i64_local_sys_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i64 %val, ptr addrspace(5) %slot syncscope("") release, align 8
+ ret void
+}
+
+define void @release_i64_local_cta_volatile(i64 %val) {
+; CHECK-LABEL: release_i64_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot893[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot893;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i64_local_cta_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i64 %val, ptr addrspace(5) %slot syncscope("block") release, align 8
+ ret void
+}
+
+define void @release_i64_local_cluster_volatile(i64 %val) {
+; CHECK-LABEL: release_i64_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot894[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot894;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i64_local_cluster_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i64 %val, ptr addrspace(5) %slot syncscope("cluster") release, align 8
+ ret void
+}
+
+define void @release_i64_local_gpu_volatile(i64 %val) {
+; CHECK-LABEL: release_i64_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot895[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot895;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_i64_local_gpu_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i64 %val, ptr addrspace(5) %slot syncscope("device") release, align 8
+ ret void
+}
+
+define void @release_i128_local_sys(i128 %val) {
+; CHECK-LABEL: release_i128_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 16 .b8 __local_depot896[16];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot896;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [release_i128_local_sys_param_0];
+; CHECK-NEXT: add.u64 %rd3, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd3;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.v2.b64 [%SP], {%rd1, %rd2};
+; CHECK-NEXT: ret;
+ %slot = alloca i128, align 16, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i128 %val, ptr addrspace(5) %slot syncscope("") release, align 16
+ ret void
+}
+
+define void @release_i128_local_cta(i128 %val) {
+; CHECK-LABEL: release_i128_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 16 .b8 __local_depot897[16];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot897;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [release_i128_local_cta_param_0];
+; CHECK-NEXT: add.u64 %rd3, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd3;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.v2.b64 [%SP], {%rd1, %rd2};
+; CHECK-NEXT: ret;
+ %slot = alloca i128, align 16, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i128 %val, ptr addrspace(5) %slot syncscope("block") release, align 16
+ ret void
+}
+
+define void @release_i128_local_cluster(i128 %val) {
+; CHECK-LABEL: release_i128_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 16 .b8 __local_depot898[16];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot898;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [release_i128_local_cluster_param_0];
+; CHECK-NEXT: add.u64 %rd3, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd3;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.v2.b64 [%SP], {%rd1, %rd2};
+; CHECK-NEXT: ret;
+ %slot = alloca i128, align 16, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i128 %val, ptr addrspace(5) %slot syncscope("cluster") release, align 16
+ ret void
+}
+
+define void @release_i128_local_gpu(i128 %val) {
+; CHECK-LABEL: release_i128_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 16 .b8 __local_depot899[16];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot899;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [release_i128_local_gpu_param_0];
+; CHECK-NEXT: add.u64 %rd3, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd3;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.v2.b64 [%SP], {%rd1, %rd2};
+; CHECK-NEXT: ret;
+ %slot = alloca i128, align 16, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i128 %val, ptr addrspace(5) %slot syncscope("device") release, align 16
+ ret void
+}
+
+define void @release_i128_local_sys_volatile(i128 %val) {
+; CHECK-LABEL: release_i128_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 16 .b8 __local_depot900[16];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot900;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [release_i128_local_sys_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd3, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd3;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.v2.b64 [%SP], {%rd1, %rd2};
+; CHECK-NEXT: ret;
+ %slot = alloca i128, align 16, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i128 %val, ptr addrspace(5) %slot syncscope("") release, align 16
+ ret void
+}
+
+define void @release_i128_local_cta_volatile(i128 %val) {
+; CHECK-LABEL: release_i128_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 16 .b8 __local_depot901[16];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot901;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [release_i128_local_cta_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd3, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd3;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.v2.b64 [%SP], {%rd1, %rd2};
+; CHECK-NEXT: ret;
+ %slot = alloca i128, align 16, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i128 %val, ptr addrspace(5) %slot syncscope("block") release, align 16
+ ret void
+}
+
+define void @release_i128_local_cluster_volatile(i128 %val) {
+; CHECK-LABEL: release_i128_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 16 .b8 __local_depot902[16];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot902;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [release_i128_local_cluster_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd3, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd3;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.v2.b64 [%SP], {%rd1, %rd2};
+; CHECK-NEXT: ret;
+ %slot = alloca i128, align 16, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i128 %val, ptr addrspace(5) %slot syncscope("cluster") release, align 16
+ ret void
+}
+
+define void @release_i128_local_gpu_volatile(i128 %val) {
+; CHECK-LABEL: release_i128_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 16 .b8 __local_depot903[16];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot903;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [release_i128_local_gpu_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd3, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd3;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.v2.b64 [%SP], {%rd1, %rd2};
+; CHECK-NEXT: ret;
+ %slot = alloca i128, align 16, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i128 %val, ptr addrspace(5) %slot syncscope("device") release, align 16
+ ret void
+}
+
+define void @release_half_local_sys(half %val) {
+; CHECK-LABEL: release_half_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot904[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot904;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [release_half_local_sys_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic half %val, ptr addrspace(5) %slot syncscope("") release, align 2
+ ret void
+}
+
+define void @release_half_local_cta(half %val) {
+; CHECK-LABEL: release_half_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot905[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot905;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [release_half_local_cta_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic half %val, ptr addrspace(5) %slot syncscope("block") release, align 2
+ ret void
+}
+
+define void @release_half_local_cluster(half %val) {
+; CHECK-LABEL: release_half_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot906[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot906;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [release_half_local_cluster_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic half %val, ptr addrspace(5) %slot syncscope("cluster") release, align 2
+ ret void
+}
+
+define void @release_half_local_gpu(half %val) {
+; CHECK-LABEL: release_half_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot907[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot907;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [release_half_local_gpu_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic half %val, ptr addrspace(5) %slot syncscope("device") release, align 2
+ ret void
+}
+
+define void @release_half_local_sys_volatile(half %val) {
+; CHECK-LABEL: release_half_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot908[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot908;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [release_half_local_sys_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile half %val, ptr addrspace(5) %slot syncscope("") release, align 2
+ ret void
+}
+
+define void @release_half_local_cta_volatile(half %val) {
+; CHECK-LABEL: release_half_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot909[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot909;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [release_half_local_cta_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile half %val, ptr addrspace(5) %slot syncscope("block") release, align 2
+ ret void
+}
+
+define void @release_half_local_cluster_volatile(half %val) {
+; CHECK-LABEL: release_half_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot910[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot910;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [release_half_local_cluster_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile half %val, ptr addrspace(5) %slot syncscope("cluster") release, align 2
+ ret void
+}
+
+define void @release_half_local_gpu_volatile(half %val) {
+; CHECK-LABEL: release_half_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot911[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot911;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [release_half_local_gpu_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile half %val, ptr addrspace(5) %slot syncscope("device") release, align 2
+ ret void
+}
+
+define void @release_bfloat_local_sys(bfloat %val) {
+; CHECK-LABEL: release_bfloat_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot912[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot912;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [release_bfloat_local_sys_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic bfloat %val, ptr addrspace(5) %slot syncscope("") release, align 2
+ ret void
+}
+
+define void @release_bfloat_local_cta(bfloat %val) {
+; CHECK-LABEL: release_bfloat_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot913[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot913;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [release_bfloat_local_cta_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic bfloat %val, ptr addrspace(5) %slot syncscope("block") release, align 2
+ ret void
+}
+
+define void @release_bfloat_local_cluster(bfloat %val) {
+; CHECK-LABEL: release_bfloat_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot914[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot914;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [release_bfloat_local_cluster_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic bfloat %val, ptr addrspace(5) %slot syncscope("cluster") release, align 2
+ ret void
+}
+
+define void @release_bfloat_local_gpu(bfloat %val) {
+; CHECK-LABEL: release_bfloat_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot915[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot915;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [release_bfloat_local_gpu_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic bfloat %val, ptr addrspace(5) %slot syncscope("device") release, align 2
+ ret void
+}
+
+define void @release_bfloat_local_sys_volatile(bfloat %val) {
+; CHECK-LABEL: release_bfloat_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot916[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot916;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [release_bfloat_local_sys_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile bfloat %val, ptr addrspace(5) %slot syncscope("") release, align 2
+ ret void
+}
+
+define void @release_bfloat_local_cta_volatile(bfloat %val) {
+; CHECK-LABEL: release_bfloat_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot917[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot917;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [release_bfloat_local_cta_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile bfloat %val, ptr addrspace(5) %slot syncscope("block") release, align 2
+ ret void
+}
+
+define void @release_bfloat_local_cluster_volatile(bfloat %val) {
+; CHECK-LABEL: release_bfloat_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot918[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot918;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [release_bfloat_local_cluster_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile bfloat %val, ptr addrspace(5) %slot syncscope("cluster") release, align 2
+ ret void
+}
+
+define void @release_bfloat_local_gpu_volatile(bfloat %val) {
+; CHECK-LABEL: release_bfloat_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot919[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot919;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [release_bfloat_local_gpu_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile bfloat %val, ptr addrspace(5) %slot syncscope("device") release, align 2
+ ret void
+}
+
+define void @release_float_local_sys(float %val) {
+; CHECK-LABEL: release_float_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot920[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot920;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [release_float_local_sys_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic float %val, ptr addrspace(5) %slot syncscope("") release, align 4
+ ret void
+}
+
+define void @release_float_local_cta(float %val) {
+; CHECK-LABEL: release_float_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot921[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot921;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [release_float_local_cta_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic float %val, ptr addrspace(5) %slot syncscope("block") release, align 4
+ ret void
+}
+
+define void @release_float_local_cluster(float %val) {
+; CHECK-LABEL: release_float_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot922[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot922;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [release_float_local_cluster_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic float %val, ptr addrspace(5) %slot syncscope("cluster") release, align 4
+ ret void
+}
+
+define void @release_float_local_gpu(float %val) {
+; CHECK-LABEL: release_float_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot923[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot923;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [release_float_local_gpu_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic float %val, ptr addrspace(5) %slot syncscope("device") release, align 4
+ ret void
+}
+
+define void @release_float_local_sys_volatile(float %val) {
+; CHECK-LABEL: release_float_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot924[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot924;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [release_float_local_sys_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile float %val, ptr addrspace(5) %slot syncscope("") release, align 4
+ ret void
+}
+
+define void @release_float_local_cta_volatile(float %val) {
+; CHECK-LABEL: release_float_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot925[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot925;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [release_float_local_cta_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile float %val, ptr addrspace(5) %slot syncscope("block") release, align 4
+ ret void
+}
+
+define void @release_float_local_cluster_volatile(float %val) {
+; CHECK-LABEL: release_float_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot926[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot926;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [release_float_local_cluster_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile float %val, ptr addrspace(5) %slot syncscope("cluster") release, align 4
+ ret void
+}
+
+define void @release_float_local_gpu_volatile(float %val) {
+; CHECK-LABEL: release_float_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot927[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot927;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [release_float_local_gpu_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile float %val, ptr addrspace(5) %slot syncscope("device") release, align 4
+ ret void
+}
+
+define void @release_double_local_sys(double %val) {
+; CHECK-LABEL: release_double_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot928[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot928;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_double_local_sys_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic double %val, ptr addrspace(5) %slot syncscope("") release, align 8
+ ret void
+}
+
+define void @release_double_local_cta(double %val) {
+; CHECK-LABEL: release_double_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot929[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot929;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_double_local_cta_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic double %val, ptr addrspace(5) %slot syncscope("block") release, align 8
+ ret void
+}
+
+define void @release_double_local_cluster(double %val) {
+; CHECK-LABEL: release_double_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot930[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot930;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_double_local_cluster_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic double %val, ptr addrspace(5) %slot syncscope("cluster") release, align 8
+ ret void
+}
+
+define void @release_double_local_gpu(double %val) {
+; CHECK-LABEL: release_double_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot931[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot931;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_double_local_gpu_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic double %val, ptr addrspace(5) %slot syncscope("device") release, align 8
+ ret void
+}
+
+define void @release_double_local_sys_volatile(double %val) {
+; CHECK-LABEL: release_double_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot932[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot932;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_double_local_sys_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile double %val, ptr addrspace(5) %slot syncscope("") release, align 8
+ ret void
+}
+
+define void @release_double_local_cta_volatile(double %val) {
+; CHECK-LABEL: release_double_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot933[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot933;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_double_local_cta_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile double %val, ptr addrspace(5) %slot syncscope("block") release, align 8
+ ret void
+}
+
+define void @release_double_local_cluster_volatile(double %val) {
+; CHECK-LABEL: release_double_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot934[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot934;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_double_local_cluster_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile double %val, ptr addrspace(5) %slot syncscope("cluster") release, align 8
+ ret void
+}
+
+define void @release_double_local_gpu_volatile(double %val) {
+; CHECK-LABEL: release_double_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot935[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot935;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [release_double_local_gpu_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile double %val, ptr addrspace(5) %slot syncscope("device") release, align 8
+ ret void
+}
+
+define void @seq_cst_i8_generic_sys(i8 %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_i8_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [seq_cst_i8_generic_sys_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_generic_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr %addr syncscope("") seq_cst, align 1
+ ret void
+}
+
+define void @seq_cst_i8_generic_cta(i8 %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_i8_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [seq_cst_i8_generic_cta_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_generic_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr %addr syncscope("block") seq_cst, align 1
+ ret void
+}
+
+define void @seq_cst_i8_generic_cluster(i8 %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_i8_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [seq_cst_i8_generic_cluster_param_0];
+; CHECK-NEXT: fence.sc.cluster;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_generic_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cluster.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr %addr syncscope("cluster") seq_cst, align 1
+ ret void
+}
+
+define void @seq_cst_i8_generic_gpu(i8 %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_i8_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [seq_cst_i8_generic_gpu_param_0];
+; CHECK-NEXT: fence.sc.gpu;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_generic_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr %addr syncscope("device") seq_cst, align 1
+ ret void
+}
+
+define void @seq_cst_i8_generic_sys_volatile(i8 %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_i8_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [seq_cst_i8_generic_sys_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_generic_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr %addr syncscope("") seq_cst, align 1
+ ret void
+}
+
+define void @seq_cst_i8_generic_cta_volatile(i8 %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_i8_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [seq_cst_i8_generic_cta_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_generic_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr %addr syncscope("block") seq_cst, align 1
+ ret void
+}
+
+define void @seq_cst_i8_generic_cluster_volatile(i8 %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_i8_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [seq_cst_i8_generic_cluster_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_generic_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr %addr syncscope("cluster") seq_cst, align 1
+ ret void
+}
+
+define void @seq_cst_i8_generic_gpu_volatile(i8 %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_i8_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [seq_cst_i8_generic_gpu_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_generic_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr %addr syncscope("device") seq_cst, align 1
+ ret void
+}
+
+define void @seq_cst_i16_generic_sys(i16 %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_i16_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_i16_generic_sys_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_generic_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr %addr syncscope("") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_i16_generic_cta(i16 %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_i16_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_i16_generic_cta_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_generic_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr %addr syncscope("block") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_i16_generic_cluster(i16 %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_i16_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_i16_generic_cluster_param_0];
+; CHECK-NEXT: fence.sc.cluster;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_generic_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cluster.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr %addr syncscope("cluster") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_i16_generic_gpu(i16 %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_i16_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_i16_generic_gpu_param_0];
+; CHECK-NEXT: fence.sc.gpu;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_generic_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr %addr syncscope("device") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_i16_generic_sys_volatile(i16 %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_i16_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_i16_generic_sys_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_generic_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr %addr syncscope("") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_i16_generic_cta_volatile(i16 %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_i16_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_i16_generic_cta_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_generic_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr %addr syncscope("block") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_i16_generic_cluster_volatile(i16 %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_i16_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_i16_generic_cluster_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_generic_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr %addr syncscope("cluster") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_i16_generic_gpu_volatile(i16 %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_i16_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_i16_generic_gpu_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_generic_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr %addr syncscope("device") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_i32_generic_sys(i32 %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_i32_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_i32_generic_sys_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_generic_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr %addr syncscope("") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_i32_generic_cta(i32 %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_i32_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_i32_generic_cta_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_generic_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr %addr syncscope("block") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_i32_generic_cluster(i32 %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_i32_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_i32_generic_cluster_param_0];
+; CHECK-NEXT: fence.sc.cluster;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_generic_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cluster.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr %addr syncscope("cluster") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_i32_generic_gpu(i32 %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_i32_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_i32_generic_gpu_param_0];
+; CHECK-NEXT: fence.sc.gpu;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_generic_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr %addr syncscope("device") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_i32_generic_sys_volatile(i32 %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_i32_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_i32_generic_sys_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_generic_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr %addr syncscope("") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_i32_generic_cta_volatile(i32 %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_i32_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_i32_generic_cta_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_generic_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr %addr syncscope("block") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_i32_generic_cluster_volatile(i32 %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_i32_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_i32_generic_cluster_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_generic_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr %addr syncscope("cluster") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_i32_generic_gpu_volatile(i32 %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_i32_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_i32_generic_gpu_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_generic_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr %addr syncscope("device") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_i64_generic_sys(i64 %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_i64_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_generic_sys_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_i64_generic_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr %addr syncscope("") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_i64_generic_cta(i64 %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_i64_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_generic_cta_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_i64_generic_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr %addr syncscope("block") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_i64_generic_cluster(i64 %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_i64_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_generic_cluster_param_0];
+; CHECK-NEXT: fence.sc.cluster;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_i64_generic_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cluster.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr %addr syncscope("cluster") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_i64_generic_gpu(i64 %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_i64_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_generic_gpu_param_0];
+; CHECK-NEXT: fence.sc.gpu;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_i64_generic_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr %addr syncscope("device") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_i64_generic_sys_volatile(i64 %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_i64_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_generic_sys_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_i64_generic_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr %addr syncscope("") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_i64_generic_cta_volatile(i64 %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_i64_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_generic_cta_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_i64_generic_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr %addr syncscope("block") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_i64_generic_cluster_volatile(i64 %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_i64_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_generic_cluster_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_i64_generic_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr %addr syncscope("cluster") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_i64_generic_gpu_volatile(i64 %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_i64_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_generic_gpu_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_i64_generic_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr %addr syncscope("device") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_i128_generic_sys(i128 %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_i128_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [seq_cst_i128_generic_sys_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd3, [seq_cst_i128_generic_sys_param_1];
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 amt, dst;
+; CHECK-NEXT: mov.b128 amt, {%rd1, %rd2};
+; CHECK-NEXT: atom.relaxed.sys.exch.b128 dst, [%rd3], amt;
+; CHECK-NEXT: mov.b128 {%rd4, %rd5}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: ret;
+ store atomic i128 %val, ptr %addr syncscope("") seq_cst, align 16
+ ret void
+}
+
+define void @seq_cst_i128_generic_cta(i128 %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_i128_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [seq_cst_i128_generic_cta_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.param.b64 %rd3, [seq_cst_i128_generic_cta_param_1];
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 amt, dst;
+; CHECK-NEXT: mov.b128 amt, {%rd1, %rd2};
+; CHECK-NEXT: atom.relaxed.cta.exch.b128 dst, [%rd3], amt;
+; CHECK-NEXT: mov.b128 {%rd4, %rd5}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: ret;
+ store atomic i128 %val, ptr %addr syncscope("block") seq_cst, align 16
+ ret void
+}
+
+define void @seq_cst_i128_generic_cluster(i128 %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_i128_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [seq_cst_i128_generic_cluster_param_0];
+; CHECK-NEXT: fence.sc.cluster;
+; CHECK-NEXT: ld.param.b64 %rd3, [seq_cst_i128_generic_cluster_param_1];
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 amt, dst;
+; CHECK-NEXT: mov.b128 amt, {%rd1, %rd2};
+; CHECK-NEXT: atom.relaxed.cluster.exch.b128 dst, [%rd3], amt;
+; CHECK-NEXT: mov.b128 {%rd4, %rd5}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: ret;
+ store atomic i128 %val, ptr %addr syncscope("cluster") seq_cst, align 16
+ ret void
+}
+
+define void @seq_cst_i128_generic_gpu(i128 %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_i128_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [seq_cst_i128_generic_gpu_param_0];
+; CHECK-NEXT: fence.sc.gpu;
+; CHECK-NEXT: ld.param.b64 %rd3, [seq_cst_i128_generic_gpu_param_1];
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 amt, dst;
+; CHECK-NEXT: mov.b128 amt, {%rd1, %rd2};
+; CHECK-NEXT: atom.relaxed.gpu.exch.b128 dst, [%rd3], amt;
+; CHECK-NEXT: mov.b128 {%rd4, %rd5}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: ret;
+ store atomic i128 %val, ptr %addr syncscope("device") seq_cst, align 16
+ ret void
+}
+
+define void @seq_cst_i128_generic_sys_volatile(i128 %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_i128_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [seq_cst_i128_generic_sys_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd3, [seq_cst_i128_generic_sys_volatile_param_1];
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 amt, dst;
+; CHECK-NEXT: mov.b128 amt, {%rd1, %rd2};
+; CHECK-NEXT: atom.relaxed.sys.exch.b128 dst, [%rd3], amt;
+; CHECK-NEXT: mov.b128 {%rd4, %rd5}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: ret;
+ store atomic volatile i128 %val, ptr %addr syncscope("") seq_cst, align 16
+ ret void
+}
+
+define void @seq_cst_i128_generic_cta_volatile(i128 %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_i128_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [seq_cst_i128_generic_cta_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd3, [seq_cst_i128_generic_cta_volatile_param_1];
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 amt, dst;
+; CHECK-NEXT: mov.b128 amt, {%rd1, %rd2};
+; CHECK-NEXT: atom.relaxed.cta.exch.b128 dst, [%rd3], amt;
+; CHECK-NEXT: mov.b128 {%rd4, %rd5}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: ret;
+ store atomic volatile i128 %val, ptr %addr syncscope("block") seq_cst, align 16
+ ret void
+}
+
+define void @seq_cst_i128_generic_cluster_volatile(i128 %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_i128_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [seq_cst_i128_generic_cluster_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd3, [seq_cst_i128_generic_cluster_volatile_param_1];
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 amt, dst;
+; CHECK-NEXT: mov.b128 amt, {%rd1, %rd2};
+; CHECK-NEXT: atom.relaxed.cluster.exch.b128 dst, [%rd3], amt;
+; CHECK-NEXT: mov.b128 {%rd4, %rd5}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: ret;
+ store atomic volatile i128 %val, ptr %addr syncscope("cluster") seq_cst, align 16
+ ret void
+}
+
+define void @seq_cst_i128_generic_gpu_volatile(i128 %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_i128_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [seq_cst_i128_generic_gpu_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd3, [seq_cst_i128_generic_gpu_volatile_param_1];
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 amt, dst;
+; CHECK-NEXT: mov.b128 amt, {%rd1, %rd2};
+; CHECK-NEXT: atom.relaxed.gpu.exch.b128 dst, [%rd3], amt;
+; CHECK-NEXT: mov.b128 {%rd4, %rd5}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: ret;
+ store atomic volatile i128 %val, ptr %addr syncscope("device") seq_cst, align 16
+ ret void
+}
+
+define void @seq_cst_half_generic_sys(half %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_half_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_half_generic_sys_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_generic_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr %addr syncscope("") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_half_generic_cta(half %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_half_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_half_generic_cta_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_generic_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr %addr syncscope("block") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_half_generic_cluster(half %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_half_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_half_generic_cluster_param_0];
+; CHECK-NEXT: fence.sc.cluster;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_generic_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cluster.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr %addr syncscope("cluster") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_half_generic_gpu(half %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_half_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_half_generic_gpu_param_0];
+; CHECK-NEXT: fence.sc.gpu;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_generic_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr %addr syncscope("device") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_half_generic_sys_volatile(half %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_half_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_half_generic_sys_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_generic_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr %addr syncscope("") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_half_generic_cta_volatile(half %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_half_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_half_generic_cta_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_generic_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr %addr syncscope("block") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_half_generic_cluster_volatile(half %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_half_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_half_generic_cluster_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_generic_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr %addr syncscope("cluster") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_half_generic_gpu_volatile(half %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_half_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_half_generic_gpu_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_generic_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr %addr syncscope("device") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_bfloat_generic_sys(bfloat %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_bfloat_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_bfloat_generic_sys_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_generic_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr %addr syncscope("") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_bfloat_generic_cta(bfloat %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_bfloat_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_bfloat_generic_cta_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_generic_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr %addr syncscope("block") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_bfloat_generic_cluster(bfloat %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_bfloat_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_bfloat_generic_cluster_param_0];
+; CHECK-NEXT: fence.sc.cluster;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_generic_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cluster.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr %addr syncscope("cluster") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_bfloat_generic_gpu(bfloat %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_bfloat_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_bfloat_generic_gpu_param_0];
+; CHECK-NEXT: fence.sc.gpu;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_generic_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr %addr syncscope("device") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_bfloat_generic_sys_volatile(bfloat %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_bfloat_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_bfloat_generic_sys_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_generic_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr %addr syncscope("") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_bfloat_generic_cta_volatile(bfloat %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_bfloat_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_bfloat_generic_cta_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_generic_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr %addr syncscope("block") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_bfloat_generic_cluster_volatile(bfloat %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_bfloat_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_bfloat_generic_cluster_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_generic_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr %addr syncscope("cluster") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_bfloat_generic_gpu_volatile(bfloat %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_bfloat_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_bfloat_generic_gpu_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_generic_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr %addr syncscope("device") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_float_generic_sys(float %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_float_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_float_generic_sys_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_generic_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr %addr syncscope("") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_float_generic_cta(float %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_float_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_float_generic_cta_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_generic_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr %addr syncscope("block") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_float_generic_cluster(float %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_float_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_float_generic_cluster_param_0];
+; CHECK-NEXT: fence.sc.cluster;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_generic_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cluster.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr %addr syncscope("cluster") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_float_generic_gpu(float %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_float_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_float_generic_gpu_param_0];
+; CHECK-NEXT: fence.sc.gpu;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_generic_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr %addr syncscope("device") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_float_generic_sys_volatile(float %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_float_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_float_generic_sys_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_generic_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr %addr syncscope("") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_float_generic_cta_volatile(float %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_float_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_float_generic_cta_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_generic_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr %addr syncscope("block") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_float_generic_cluster_volatile(float %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_float_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_float_generic_cluster_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_generic_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr %addr syncscope("cluster") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_float_generic_gpu_volatile(float %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_float_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_float_generic_gpu_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_generic_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr %addr syncscope("device") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_double_generic_sys(double %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_double_generic_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_generic_sys_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_double_generic_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr %addr syncscope("") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_double_generic_cta(double %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_double_generic_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_generic_cta_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_double_generic_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr %addr syncscope("block") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_double_generic_cluster(double %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_double_generic_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_generic_cluster_param_0];
+; CHECK-NEXT: fence.sc.cluster;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_double_generic_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cluster.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr %addr syncscope("cluster") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_double_generic_gpu(double %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_double_generic_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_generic_gpu_param_0];
+; CHECK-NEXT: fence.sc.gpu;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_double_generic_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr %addr syncscope("device") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_double_generic_sys_volatile(double %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_double_generic_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_generic_sys_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_double_generic_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr %addr syncscope("") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_double_generic_cta_volatile(double %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_double_generic_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_generic_cta_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_double_generic_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr %addr syncscope("block") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_double_generic_cluster_volatile(double %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_double_generic_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_generic_cluster_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_double_generic_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr %addr syncscope("cluster") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_double_generic_gpu_volatile(double %val, ptr %addr) {
+; CHECK-LABEL: seq_cst_double_generic_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_generic_gpu_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_double_generic_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr %addr syncscope("device") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_i8_global_sys(i8 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i8_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [seq_cst_i8_global_sys_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_global_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.global.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr addrspace(1) %addr syncscope("") seq_cst, align 1
+ ret void
+}
+
+define void @seq_cst_i8_global_cta(i8 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i8_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [seq_cst_i8_global_cta_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_global_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.global.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr addrspace(1) %addr syncscope("block") seq_cst, align 1
+ ret void
+}
+
+define void @seq_cst_i8_global_cluster(i8 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i8_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [seq_cst_i8_global_cluster_param_0];
+; CHECK-NEXT: fence.sc.cluster;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_global_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cluster.global.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr addrspace(1) %addr syncscope("cluster") seq_cst, align 1
+ ret void
+}
+
+define void @seq_cst_i8_global_gpu(i8 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i8_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [seq_cst_i8_global_gpu_param_0];
+; CHECK-NEXT: fence.sc.gpu;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_global_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.global.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr addrspace(1) %addr syncscope("device") seq_cst, align 1
+ ret void
+}
+
+define void @seq_cst_i8_global_sys_volatile(i8 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i8_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [seq_cst_i8_global_sys_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_global_sys_volatile_param_1];
+; CHECK-NEXT: st.mmio.relaxed.sys.global.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr addrspace(1) %addr syncscope("") seq_cst, align 1
+ ret void
+}
+
+define void @seq_cst_i8_global_cta_volatile(i8 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i8_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [seq_cst_i8_global_cta_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_global_cta_volatile_param_1];
+; CHECK-NEXT: st.mmio.relaxed.sys.global.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr addrspace(1) %addr syncscope("block") seq_cst, align 1
+ ret void
+}
+
+define void @seq_cst_i8_global_cluster_volatile(i8 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i8_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [seq_cst_i8_global_cluster_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_global_cluster_volatile_param_1];
+; CHECK-NEXT: st.mmio.relaxed.sys.global.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr addrspace(1) %addr syncscope("cluster") seq_cst, align 1
+ ret void
+}
+
+define void @seq_cst_i8_global_gpu_volatile(i8 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i8_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [seq_cst_i8_global_gpu_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_global_gpu_volatile_param_1];
+; CHECK-NEXT: st.mmio.relaxed.sys.global.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr addrspace(1) %addr syncscope("device") seq_cst, align 1
+ ret void
+}
+
+define void @seq_cst_i16_global_sys(i16 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i16_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_i16_global_sys_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_global_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr addrspace(1) %addr syncscope("") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_i16_global_cta(i16 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i16_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_i16_global_cta_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_global_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr addrspace(1) %addr syncscope("block") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_i16_global_cluster(i16 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i16_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_i16_global_cluster_param_0];
+; CHECK-NEXT: fence.sc.cluster;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_global_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cluster.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr addrspace(1) %addr syncscope("cluster") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_i16_global_gpu(i16 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i16_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_i16_global_gpu_param_0];
+; CHECK-NEXT: fence.sc.gpu;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_global_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr addrspace(1) %addr syncscope("device") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_i16_global_sys_volatile(i16 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i16_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_i16_global_sys_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_global_sys_volatile_param_1];
+; CHECK-NEXT: st.mmio.relaxed.sys.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr addrspace(1) %addr syncscope("") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_i16_global_cta_volatile(i16 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i16_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_i16_global_cta_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_global_cta_volatile_param_1];
+; CHECK-NEXT: st.mmio.relaxed.sys.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr addrspace(1) %addr syncscope("block") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_i16_global_cluster_volatile(i16 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i16_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_i16_global_cluster_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_global_cluster_volatile_param_1];
+; CHECK-NEXT: st.mmio.relaxed.sys.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr addrspace(1) %addr syncscope("cluster") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_i16_global_gpu_volatile(i16 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i16_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_i16_global_gpu_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_global_gpu_volatile_param_1];
+; CHECK-NEXT: st.mmio.relaxed.sys.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr addrspace(1) %addr syncscope("device") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_i32_global_sys(i32 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i32_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_i32_global_sys_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_global_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr addrspace(1) %addr syncscope("") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_i32_global_cta(i32 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i32_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_i32_global_cta_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_global_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr addrspace(1) %addr syncscope("block") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_i32_global_cluster(i32 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i32_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_i32_global_cluster_param_0];
+; CHECK-NEXT: fence.sc.cluster;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_global_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cluster.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr addrspace(1) %addr syncscope("cluster") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_i32_global_gpu(i32 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i32_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_i32_global_gpu_param_0];
+; CHECK-NEXT: fence.sc.gpu;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_global_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr addrspace(1) %addr syncscope("device") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_i32_global_sys_volatile(i32 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i32_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_i32_global_sys_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_global_sys_volatile_param_1];
+; CHECK-NEXT: st.mmio.relaxed.sys.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr addrspace(1) %addr syncscope("") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_i32_global_cta_volatile(i32 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i32_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_i32_global_cta_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_global_cta_volatile_param_1];
+; CHECK-NEXT: st.mmio.relaxed.sys.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr addrspace(1) %addr syncscope("block") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_i32_global_cluster_volatile(i32 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i32_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_i32_global_cluster_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_global_cluster_volatile_param_1];
+; CHECK-NEXT: st.mmio.relaxed.sys.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr addrspace(1) %addr syncscope("cluster") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_i32_global_gpu_volatile(i32 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i32_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_i32_global_gpu_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_global_gpu_volatile_param_1];
+; CHECK-NEXT: st.mmio.relaxed.sys.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr addrspace(1) %addr syncscope("device") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_i64_global_sys(i64 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i64_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_global_sys_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_i64_global_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr addrspace(1) %addr syncscope("") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_i64_global_cta(i64 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i64_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_global_cta_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_i64_global_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr addrspace(1) %addr syncscope("block") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_i64_global_cluster(i64 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i64_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_global_cluster_param_0];
+; CHECK-NEXT: fence.sc.cluster;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_i64_global_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cluster.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr addrspace(1) %addr syncscope("cluster") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_i64_global_gpu(i64 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i64_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_global_gpu_param_0];
+; CHECK-NEXT: fence.sc.gpu;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_i64_global_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr addrspace(1) %addr syncscope("device") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_i64_global_sys_volatile(i64 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i64_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_global_sys_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_i64_global_sys_volatile_param_1];
+; CHECK-NEXT: st.mmio.relaxed.sys.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr addrspace(1) %addr syncscope("") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_i64_global_cta_volatile(i64 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i64_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_global_cta_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_i64_global_cta_volatile_param_1];
+; CHECK-NEXT: st.mmio.relaxed.sys.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr addrspace(1) %addr syncscope("block") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_i64_global_cluster_volatile(i64 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i64_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_global_cluster_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_i64_global_cluster_volatile_param_1];
+; CHECK-NEXT: st.mmio.relaxed.sys.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr addrspace(1) %addr syncscope("cluster") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_i64_global_gpu_volatile(i64 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i64_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_global_gpu_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_i64_global_gpu_volatile_param_1];
+; CHECK-NEXT: st.mmio.relaxed.sys.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr addrspace(1) %addr syncscope("device") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_i128_global_sys(i128 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i128_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [seq_cst_i128_global_sys_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd3, [seq_cst_i128_global_sys_param_1];
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 amt, dst;
+; CHECK-NEXT: mov.b128 amt, {%rd1, %rd2};
+; CHECK-NEXT: atom.relaxed.sys.global.exch.b128 dst, [%rd3], amt;
+; CHECK-NEXT: mov.b128 {%rd4, %rd5}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: ret;
+ store atomic i128 %val, ptr addrspace(1) %addr syncscope("") seq_cst, align 16
+ ret void
+}
+
+define void @seq_cst_i128_global_cta(i128 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i128_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [seq_cst_i128_global_cta_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.param.b64 %rd3, [seq_cst_i128_global_cta_param_1];
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 amt, dst;
+; CHECK-NEXT: mov.b128 amt, {%rd1, %rd2};
+; CHECK-NEXT: atom.relaxed.cta.global.exch.b128 dst, [%rd3], amt;
+; CHECK-NEXT: mov.b128 {%rd4, %rd5}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: ret;
+ store atomic i128 %val, ptr addrspace(1) %addr syncscope("block") seq_cst, align 16
+ ret void
+}
+
+define void @seq_cst_i128_global_cluster(i128 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i128_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [seq_cst_i128_global_cluster_param_0];
+; CHECK-NEXT: fence.sc.cluster;
+; CHECK-NEXT: ld.param.b64 %rd3, [seq_cst_i128_global_cluster_param_1];
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 amt, dst;
+; CHECK-NEXT: mov.b128 amt, {%rd1, %rd2};
+; CHECK-NEXT: atom.relaxed.cluster.global.exch.b128 dst, [%rd3], amt;
+; CHECK-NEXT: mov.b128 {%rd4, %rd5}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: ret;
+ store atomic i128 %val, ptr addrspace(1) %addr syncscope("cluster") seq_cst, align 16
+ ret void
+}
+
+define void @seq_cst_i128_global_gpu(i128 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i128_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [seq_cst_i128_global_gpu_param_0];
+; CHECK-NEXT: fence.sc.gpu;
+; CHECK-NEXT: ld.param.b64 %rd3, [seq_cst_i128_global_gpu_param_1];
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 amt, dst;
+; CHECK-NEXT: mov.b128 amt, {%rd1, %rd2};
+; CHECK-NEXT: atom.relaxed.gpu.global.exch.b128 dst, [%rd3], amt;
+; CHECK-NEXT: mov.b128 {%rd4, %rd5}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: ret;
+ store atomic i128 %val, ptr addrspace(1) %addr syncscope("device") seq_cst, align 16
+ ret void
+}
+
+define void @seq_cst_i128_global_sys_volatile(i128 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i128_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [seq_cst_i128_global_sys_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd3, [seq_cst_i128_global_sys_volatile_param_1];
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 amt, dst;
+; CHECK-NEXT: mov.b128 amt, {%rd1, %rd2};
+; CHECK-NEXT: atom.relaxed.sys.global.exch.b128 dst, [%rd3], amt;
+; CHECK-NEXT: mov.b128 {%rd4, %rd5}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: ret;
+ store atomic volatile i128 %val, ptr addrspace(1) %addr syncscope("") seq_cst, align 16
+ ret void
+}
+
+define void @seq_cst_i128_global_cta_volatile(i128 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i128_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [seq_cst_i128_global_cta_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd3, [seq_cst_i128_global_cta_volatile_param_1];
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 amt, dst;
+; CHECK-NEXT: mov.b128 amt, {%rd1, %rd2};
+; CHECK-NEXT: atom.relaxed.cta.global.exch.b128 dst, [%rd3], amt;
+; CHECK-NEXT: mov.b128 {%rd4, %rd5}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: ret;
+ store atomic volatile i128 %val, ptr addrspace(1) %addr syncscope("block") seq_cst, align 16
+ ret void
+}
+
+define void @seq_cst_i128_global_cluster_volatile(i128 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i128_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [seq_cst_i128_global_cluster_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd3, [seq_cst_i128_global_cluster_volatile_param_1];
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 amt, dst;
+; CHECK-NEXT: mov.b128 amt, {%rd1, %rd2};
+; CHECK-NEXT: atom.relaxed.cluster.global.exch.b128 dst, [%rd3], amt;
+; CHECK-NEXT: mov.b128 {%rd4, %rd5}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: ret;
+ store atomic volatile i128 %val, ptr addrspace(1) %addr syncscope("cluster") seq_cst, align 16
+ ret void
+}
+
+define void @seq_cst_i128_global_gpu_volatile(i128 %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_i128_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [seq_cst_i128_global_gpu_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd3, [seq_cst_i128_global_gpu_volatile_param_1];
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 amt, dst;
+; CHECK-NEXT: mov.b128 amt, {%rd1, %rd2};
+; CHECK-NEXT: atom.relaxed.gpu.global.exch.b128 dst, [%rd3], amt;
+; CHECK-NEXT: mov.b128 {%rd4, %rd5}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: ret;
+ store atomic volatile i128 %val, ptr addrspace(1) %addr syncscope("device") seq_cst, align 16
+ ret void
+}
+
+define void @seq_cst_half_global_sys(half %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_half_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_half_global_sys_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_global_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr addrspace(1) %addr syncscope("") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_half_global_cta(half %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_half_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_half_global_cta_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_global_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr addrspace(1) %addr syncscope("block") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_half_global_cluster(half %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_half_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_half_global_cluster_param_0];
+; CHECK-NEXT: fence.sc.cluster;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_global_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cluster.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr addrspace(1) %addr syncscope("cluster") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_half_global_gpu(half %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_half_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_half_global_gpu_param_0];
+; CHECK-NEXT: fence.sc.gpu;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_global_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr addrspace(1) %addr syncscope("device") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_half_global_sys_volatile(half %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_half_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_half_global_sys_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_global_sys_volatile_param_1];
+; CHECK-NEXT: st.mmio.relaxed.sys.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr addrspace(1) %addr syncscope("") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_half_global_cta_volatile(half %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_half_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_half_global_cta_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_global_cta_volatile_param_1];
+; CHECK-NEXT: st.mmio.relaxed.sys.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr addrspace(1) %addr syncscope("block") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_half_global_cluster_volatile(half %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_half_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_half_global_cluster_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_global_cluster_volatile_param_1];
+; CHECK-NEXT: st.mmio.relaxed.sys.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr addrspace(1) %addr syncscope("cluster") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_half_global_gpu_volatile(half %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_half_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_half_global_gpu_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_global_gpu_volatile_param_1];
+; CHECK-NEXT: st.mmio.relaxed.sys.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr addrspace(1) %addr syncscope("device") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_bfloat_global_sys(bfloat %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_bfloat_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_bfloat_global_sys_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_global_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr addrspace(1) %addr syncscope("") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_bfloat_global_cta(bfloat %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_bfloat_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_bfloat_global_cta_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_global_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr addrspace(1) %addr syncscope("block") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_bfloat_global_cluster(bfloat %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_bfloat_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_bfloat_global_cluster_param_0];
+; CHECK-NEXT: fence.sc.cluster;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_global_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cluster.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr addrspace(1) %addr syncscope("cluster") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_bfloat_global_gpu(bfloat %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_bfloat_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_bfloat_global_gpu_param_0];
+; CHECK-NEXT: fence.sc.gpu;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_global_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr addrspace(1) %addr syncscope("device") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_bfloat_global_sys_volatile(bfloat %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_bfloat_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_bfloat_global_sys_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_global_sys_volatile_param_1];
+; CHECK-NEXT: st.mmio.relaxed.sys.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr addrspace(1) %addr syncscope("") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_bfloat_global_cta_volatile(bfloat %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_bfloat_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_bfloat_global_cta_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_global_cta_volatile_param_1];
+; CHECK-NEXT: st.mmio.relaxed.sys.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr addrspace(1) %addr syncscope("block") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_bfloat_global_cluster_volatile(bfloat %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_bfloat_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_bfloat_global_cluster_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_global_cluster_volatile_param_1];
+; CHECK-NEXT: st.mmio.relaxed.sys.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr addrspace(1) %addr syncscope("cluster") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_bfloat_global_gpu_volatile(bfloat %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_bfloat_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_bfloat_global_gpu_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_global_gpu_volatile_param_1];
+; CHECK-NEXT: st.mmio.relaxed.sys.global.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr addrspace(1) %addr syncscope("device") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_float_global_sys(float %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_float_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_float_global_sys_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_global_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr addrspace(1) %addr syncscope("") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_float_global_cta(float %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_float_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_float_global_cta_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_global_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr addrspace(1) %addr syncscope("block") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_float_global_cluster(float %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_float_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_float_global_cluster_param_0];
+; CHECK-NEXT: fence.sc.cluster;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_global_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cluster.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr addrspace(1) %addr syncscope("cluster") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_float_global_gpu(float %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_float_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_float_global_gpu_param_0];
+; CHECK-NEXT: fence.sc.gpu;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_global_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr addrspace(1) %addr syncscope("device") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_float_global_sys_volatile(float %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_float_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_float_global_sys_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_global_sys_volatile_param_1];
+; CHECK-NEXT: st.mmio.relaxed.sys.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr addrspace(1) %addr syncscope("") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_float_global_cta_volatile(float %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_float_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_float_global_cta_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_global_cta_volatile_param_1];
+; CHECK-NEXT: st.mmio.relaxed.sys.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr addrspace(1) %addr syncscope("block") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_float_global_cluster_volatile(float %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_float_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_float_global_cluster_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_global_cluster_volatile_param_1];
+; CHECK-NEXT: st.mmio.relaxed.sys.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr addrspace(1) %addr syncscope("cluster") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_float_global_gpu_volatile(float %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_float_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_float_global_gpu_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_global_gpu_volatile_param_1];
+; CHECK-NEXT: st.mmio.relaxed.sys.global.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr addrspace(1) %addr syncscope("device") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_double_global_sys(double %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_double_global_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_global_sys_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_double_global_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr addrspace(1) %addr syncscope("") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_double_global_cta(double %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_double_global_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_global_cta_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_double_global_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr addrspace(1) %addr syncscope("block") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_double_global_cluster(double %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_double_global_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_global_cluster_param_0];
+; CHECK-NEXT: fence.sc.cluster;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_double_global_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cluster.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr addrspace(1) %addr syncscope("cluster") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_double_global_gpu(double %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_double_global_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_global_gpu_param_0];
+; CHECK-NEXT: fence.sc.gpu;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_double_global_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr addrspace(1) %addr syncscope("device") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_double_global_sys_volatile(double %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_double_global_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_global_sys_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_double_global_sys_volatile_param_1];
+; CHECK-NEXT: st.mmio.relaxed.sys.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr addrspace(1) %addr syncscope("") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_double_global_cta_volatile(double %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_double_global_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_global_cta_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_double_global_cta_volatile_param_1];
+; CHECK-NEXT: st.mmio.relaxed.sys.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr addrspace(1) %addr syncscope("block") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_double_global_cluster_volatile(double %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_double_global_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_global_cluster_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_double_global_cluster_volatile_param_1];
+; CHECK-NEXT: st.mmio.relaxed.sys.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr addrspace(1) %addr syncscope("cluster") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_double_global_gpu_volatile(double %val, ptr addrspace(1) %addr) {
+; CHECK-LABEL: seq_cst_double_global_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_global_gpu_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_double_global_gpu_volatile_param_1];
+; CHECK-NEXT: st.mmio.relaxed.sys.global.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr addrspace(1) %addr syncscope("device") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_i8_shared_sys(i8 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i8_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [seq_cst_i8_shared_sys_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_shared_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.shared.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr addrspace(3) %addr syncscope("") seq_cst, align 1
+ ret void
+}
+
+define void @seq_cst_i8_shared_cta(i8 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i8_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [seq_cst_i8_shared_cta_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_shared_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.shared.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr addrspace(3) %addr syncscope("block") seq_cst, align 1
+ ret void
+}
+
+define void @seq_cst_i8_shared_cluster(i8 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i8_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [seq_cst_i8_shared_cluster_param_0];
+; CHECK-NEXT: fence.sc.cluster;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_shared_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cluster.shared.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr addrspace(3) %addr syncscope("cluster") seq_cst, align 1
+ ret void
+}
+
+define void @seq_cst_i8_shared_gpu(i8 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i8_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [seq_cst_i8_shared_gpu_param_0];
+; CHECK-NEXT: fence.sc.gpu;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_shared_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.shared.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i8 %val, ptr addrspace(3) %addr syncscope("device") seq_cst, align 1
+ ret void
+}
+
+define void @seq_cst_i8_shared_sys_volatile(i8 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i8_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [seq_cst_i8_shared_sys_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_shared_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr addrspace(3) %addr syncscope("") seq_cst, align 1
+ ret void
+}
+
+define void @seq_cst_i8_shared_cta_volatile(i8 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i8_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [seq_cst_i8_shared_cta_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_shared_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr addrspace(3) %addr syncscope("block") seq_cst, align 1
+ ret void
+}
+
+define void @seq_cst_i8_shared_cluster_volatile(i8 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i8_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [seq_cst_i8_shared_cluster_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_shared_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr addrspace(3) %addr syncscope("cluster") seq_cst, align 1
+ ret void
+}
+
+define void @seq_cst_i8_shared_gpu_volatile(i8 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i8_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b8 %rs1, [seq_cst_i8_shared_gpu_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i8_shared_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b8 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i8 %val, ptr addrspace(3) %addr syncscope("device") seq_cst, align 1
+ ret void
+}
+
+define void @seq_cst_i16_shared_sys(i16 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i16_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_i16_shared_sys_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_shared_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr addrspace(3) %addr syncscope("") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_i16_shared_cta(i16 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i16_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_i16_shared_cta_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_shared_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr addrspace(3) %addr syncscope("block") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_i16_shared_cluster(i16 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i16_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_i16_shared_cluster_param_0];
+; CHECK-NEXT: fence.sc.cluster;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_shared_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cluster.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr addrspace(3) %addr syncscope("cluster") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_i16_shared_gpu(i16 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i16_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_i16_shared_gpu_param_0];
+; CHECK-NEXT: fence.sc.gpu;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_shared_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic i16 %val, ptr addrspace(3) %addr syncscope("device") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_i16_shared_sys_volatile(i16 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i16_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_i16_shared_sys_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_shared_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr addrspace(3) %addr syncscope("") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_i16_shared_cta_volatile(i16 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i16_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_i16_shared_cta_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_shared_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr addrspace(3) %addr syncscope("block") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_i16_shared_cluster_volatile(i16 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i16_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_i16_shared_cluster_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_shared_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr addrspace(3) %addr syncscope("cluster") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_i16_shared_gpu_volatile(i16 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i16_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_i16_shared_gpu_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i16_shared_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile i16 %val, ptr addrspace(3) %addr syncscope("device") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_i32_shared_sys(i32 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i32_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_i32_shared_sys_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_shared_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr addrspace(3) %addr syncscope("") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_i32_shared_cta(i32 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i32_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_i32_shared_cta_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_shared_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr addrspace(3) %addr syncscope("block") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_i32_shared_cluster(i32 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i32_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_i32_shared_cluster_param_0];
+; CHECK-NEXT: fence.sc.cluster;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_shared_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cluster.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr addrspace(3) %addr syncscope("cluster") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_i32_shared_gpu(i32 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i32_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_i32_shared_gpu_param_0];
+; CHECK-NEXT: fence.sc.gpu;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_shared_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic i32 %val, ptr addrspace(3) %addr syncscope("device") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_i32_shared_sys_volatile(i32 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i32_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_i32_shared_sys_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_shared_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr addrspace(3) %addr syncscope("") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_i32_shared_cta_volatile(i32 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i32_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_i32_shared_cta_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_shared_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr addrspace(3) %addr syncscope("block") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_i32_shared_cluster_volatile(i32 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i32_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_i32_shared_cluster_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_shared_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr addrspace(3) %addr syncscope("cluster") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_i32_shared_gpu_volatile(i32 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i32_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_i32_shared_gpu_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i32_shared_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile i32 %val, ptr addrspace(3) %addr syncscope("device") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_i64_shared_sys(i64 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i64_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_shared_sys_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_i64_shared_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr addrspace(3) %addr syncscope("") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_i64_shared_cta(i64 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i64_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_shared_cta_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_i64_shared_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr addrspace(3) %addr syncscope("block") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_i64_shared_cluster(i64 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i64_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_shared_cluster_param_0];
+; CHECK-NEXT: fence.sc.cluster;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_i64_shared_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cluster.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr addrspace(3) %addr syncscope("cluster") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_i64_shared_gpu(i64 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i64_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_shared_gpu_param_0];
+; CHECK-NEXT: fence.sc.gpu;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_i64_shared_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic i64 %val, ptr addrspace(3) %addr syncscope("device") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_i64_shared_sys_volatile(i64 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i64_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_shared_sys_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_i64_shared_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr addrspace(3) %addr syncscope("") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_i64_shared_cta_volatile(i64 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i64_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_shared_cta_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_i64_shared_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr addrspace(3) %addr syncscope("block") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_i64_shared_cluster_volatile(i64 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i64_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_shared_cluster_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_i64_shared_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr addrspace(3) %addr syncscope("cluster") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_i64_shared_gpu_volatile(i64 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i64_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_shared_gpu_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_i64_shared_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile i64 %val, ptr addrspace(3) %addr syncscope("device") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_i128_shared_sys(i128 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i128_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [seq_cst_i128_shared_sys_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd3, [seq_cst_i128_shared_sys_param_1];
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 amt, dst;
+; CHECK-NEXT: mov.b128 amt, {%rd1, %rd2};
+; CHECK-NEXT: atom.relaxed.sys.shared.exch.b128 dst, [%rd3], amt;
+; CHECK-NEXT: mov.b128 {%rd4, %rd5}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: ret;
+ store atomic i128 %val, ptr addrspace(3) %addr syncscope("") seq_cst, align 16
+ ret void
+}
+
+define void @seq_cst_i128_shared_cta(i128 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i128_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [seq_cst_i128_shared_cta_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.param.b64 %rd3, [seq_cst_i128_shared_cta_param_1];
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 amt, dst;
+; CHECK-NEXT: mov.b128 amt, {%rd1, %rd2};
+; CHECK-NEXT: atom.relaxed.cta.shared.exch.b128 dst, [%rd3], amt;
+; CHECK-NEXT: mov.b128 {%rd4, %rd5}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: ret;
+ store atomic i128 %val, ptr addrspace(3) %addr syncscope("block") seq_cst, align 16
+ ret void
+}
+
+define void @seq_cst_i128_shared_cluster(i128 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i128_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [seq_cst_i128_shared_cluster_param_0];
+; CHECK-NEXT: fence.sc.cluster;
+; CHECK-NEXT: ld.param.b64 %rd3, [seq_cst_i128_shared_cluster_param_1];
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 amt, dst;
+; CHECK-NEXT: mov.b128 amt, {%rd1, %rd2};
+; CHECK-NEXT: atom.relaxed.cluster.shared.exch.b128 dst, [%rd3], amt;
+; CHECK-NEXT: mov.b128 {%rd4, %rd5}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: ret;
+ store atomic i128 %val, ptr addrspace(3) %addr syncscope("cluster") seq_cst, align 16
+ ret void
+}
+
+define void @seq_cst_i128_shared_gpu(i128 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i128_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [seq_cst_i128_shared_gpu_param_0];
+; CHECK-NEXT: fence.sc.gpu;
+; CHECK-NEXT: ld.param.b64 %rd3, [seq_cst_i128_shared_gpu_param_1];
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 amt, dst;
+; CHECK-NEXT: mov.b128 amt, {%rd1, %rd2};
+; CHECK-NEXT: atom.relaxed.gpu.shared.exch.b128 dst, [%rd3], amt;
+; CHECK-NEXT: mov.b128 {%rd4, %rd5}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: ret;
+ store atomic i128 %val, ptr addrspace(3) %addr syncscope("device") seq_cst, align 16
+ ret void
+}
+
+define void @seq_cst_i128_shared_sys_volatile(i128 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i128_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [seq_cst_i128_shared_sys_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd3, [seq_cst_i128_shared_sys_volatile_param_1];
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 amt, dst;
+; CHECK-NEXT: mov.b128 amt, {%rd1, %rd2};
+; CHECK-NEXT: atom.relaxed.sys.shared.exch.b128 dst, [%rd3], amt;
+; CHECK-NEXT: mov.b128 {%rd4, %rd5}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: ret;
+ store atomic volatile i128 %val, ptr addrspace(3) %addr syncscope("") seq_cst, align 16
+ ret void
+}
+
+define void @seq_cst_i128_shared_cta_volatile(i128 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i128_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [seq_cst_i128_shared_cta_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd3, [seq_cst_i128_shared_cta_volatile_param_1];
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 amt, dst;
+; CHECK-NEXT: mov.b128 amt, {%rd1, %rd2};
+; CHECK-NEXT: atom.relaxed.cta.shared.exch.b128 dst, [%rd3], amt;
+; CHECK-NEXT: mov.b128 {%rd4, %rd5}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: ret;
+ store atomic volatile i128 %val, ptr addrspace(3) %addr syncscope("block") seq_cst, align 16
+ ret void
+}
+
+define void @seq_cst_i128_shared_cluster_volatile(i128 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i128_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [seq_cst_i128_shared_cluster_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd3, [seq_cst_i128_shared_cluster_volatile_param_1];
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 amt, dst;
+; CHECK-NEXT: mov.b128 amt, {%rd1, %rd2};
+; CHECK-NEXT: atom.relaxed.cluster.shared.exch.b128 dst, [%rd3], amt;
+; CHECK-NEXT: mov.b128 {%rd4, %rd5}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: ret;
+ store atomic volatile i128 %val, ptr addrspace(3) %addr syncscope("cluster") seq_cst, align 16
+ ret void
+}
+
+define void @seq_cst_i128_shared_gpu_volatile(i128 %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_i128_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [seq_cst_i128_shared_gpu_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd3, [seq_cst_i128_shared_gpu_volatile_param_1];
+; CHECK-NEXT: {
+; CHECK-NEXT: .reg .b128 amt, dst;
+; CHECK-NEXT: mov.b128 amt, {%rd1, %rd2};
+; CHECK-NEXT: atom.relaxed.gpu.shared.exch.b128 dst, [%rd3], amt;
+; CHECK-NEXT: mov.b128 {%rd4, %rd5}, dst;
+; CHECK-NEXT: }
+; CHECK-NEXT: ret;
+ store atomic volatile i128 %val, ptr addrspace(3) %addr syncscope("device") seq_cst, align 16
+ ret void
+}
+
+define void @seq_cst_half_shared_sys(half %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_half_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_half_shared_sys_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_shared_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr addrspace(3) %addr syncscope("") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_half_shared_cta(half %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_half_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_half_shared_cta_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_shared_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr addrspace(3) %addr syncscope("block") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_half_shared_cluster(half %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_half_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_half_shared_cluster_param_0];
+; CHECK-NEXT: fence.sc.cluster;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_shared_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cluster.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr addrspace(3) %addr syncscope("cluster") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_half_shared_gpu(half %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_half_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_half_shared_gpu_param_0];
+; CHECK-NEXT: fence.sc.gpu;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_shared_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic half %val, ptr addrspace(3) %addr syncscope("device") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_half_shared_sys_volatile(half %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_half_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_half_shared_sys_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_shared_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr addrspace(3) %addr syncscope("") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_half_shared_cta_volatile(half %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_half_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_half_shared_cta_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_shared_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr addrspace(3) %addr syncscope("block") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_half_shared_cluster_volatile(half %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_half_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_half_shared_cluster_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_shared_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr addrspace(3) %addr syncscope("cluster") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_half_shared_gpu_volatile(half %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_half_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_half_shared_gpu_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_half_shared_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile half %val, ptr addrspace(3) %addr syncscope("device") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_bfloat_shared_sys(bfloat %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_bfloat_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_bfloat_shared_sys_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_shared_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr addrspace(3) %addr syncscope("") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_bfloat_shared_cta(bfloat %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_bfloat_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_bfloat_shared_cta_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_shared_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr addrspace(3) %addr syncscope("block") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_bfloat_shared_cluster(bfloat %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_bfloat_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_bfloat_shared_cluster_param_0];
+; CHECK-NEXT: fence.sc.cluster;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_shared_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cluster.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr addrspace(3) %addr syncscope("cluster") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_bfloat_shared_gpu(bfloat %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_bfloat_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_bfloat_shared_gpu_param_0];
+; CHECK-NEXT: fence.sc.gpu;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_shared_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic bfloat %val, ptr addrspace(3) %addr syncscope("device") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_bfloat_shared_sys_volatile(bfloat %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_bfloat_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_bfloat_shared_sys_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_shared_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr addrspace(3) %addr syncscope("") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_bfloat_shared_cta_volatile(bfloat %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_bfloat_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_bfloat_shared_cta_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_shared_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr addrspace(3) %addr syncscope("block") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_bfloat_shared_cluster_volatile(bfloat %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_bfloat_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_bfloat_shared_cluster_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_shared_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr addrspace(3) %addr syncscope("cluster") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_bfloat_shared_gpu_volatile(bfloat %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_bfloat_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_bfloat_shared_gpu_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_bfloat_shared_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b16 [%rd1], %rs1;
+; CHECK-NEXT: ret;
+ store atomic volatile bfloat %val, ptr addrspace(3) %addr syncscope("device") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_float_shared_sys(float %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_float_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_float_shared_sys_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_shared_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr addrspace(3) %addr syncscope("") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_float_shared_cta(float %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_float_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_float_shared_cta_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_shared_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr addrspace(3) %addr syncscope("block") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_float_shared_cluster(float %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_float_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_float_shared_cluster_param_0];
+; CHECK-NEXT: fence.sc.cluster;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_shared_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cluster.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr addrspace(3) %addr syncscope("cluster") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_float_shared_gpu(float %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_float_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_float_shared_gpu_param_0];
+; CHECK-NEXT: fence.sc.gpu;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_shared_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic float %val, ptr addrspace(3) %addr syncscope("device") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_float_shared_sys_volatile(float %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_float_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_float_shared_sys_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_shared_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr addrspace(3) %addr syncscope("") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_float_shared_cta_volatile(float %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_float_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_float_shared_cta_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_shared_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr addrspace(3) %addr syncscope("block") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_float_shared_cluster_volatile(float %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_float_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_float_shared_cluster_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_shared_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr addrspace(3) %addr syncscope("cluster") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_float_shared_gpu_volatile(float %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_float_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [seq_cst_float_shared_gpu_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_float_shared_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b32 [%rd1], %r1;
+; CHECK-NEXT: ret;
+ store atomic volatile float %val, ptr addrspace(3) %addr syncscope("device") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_double_shared_sys(double %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_double_shared_sys(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_shared_sys_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_double_shared_sys_param_1];
+; CHECK-NEXT: st.relaxed.sys.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr addrspace(3) %addr syncscope("") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_double_shared_cta(double %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_double_shared_cta(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_shared_cta_param_0];
+; CHECK-NEXT: fence.sc.cta;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_double_shared_cta_param_1];
+; CHECK-NEXT: st.relaxed.cta.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr addrspace(3) %addr syncscope("block") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_double_shared_cluster(double %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_double_shared_cluster(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_shared_cluster_param_0];
+; CHECK-NEXT: fence.sc.cluster;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_double_shared_cluster_param_1];
+; CHECK-NEXT: st.relaxed.cluster.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr addrspace(3) %addr syncscope("cluster") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_double_shared_gpu(double %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_double_shared_gpu(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_shared_gpu_param_0];
+; CHECK-NEXT: fence.sc.gpu;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_double_shared_gpu_param_1];
+; CHECK-NEXT: st.relaxed.gpu.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic double %val, ptr addrspace(3) %addr syncscope("device") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_double_shared_sys_volatile(double %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_double_shared_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_shared_sys_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_double_shared_sys_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr addrspace(3) %addr syncscope("") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_double_shared_cta_volatile(double %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_double_shared_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_shared_cta_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_double_shared_cta_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr addrspace(3) %addr syncscope("block") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_double_shared_cluster_volatile(double %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_double_shared_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_shared_cluster_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_double_shared_cluster_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr addrspace(3) %addr syncscope("cluster") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_double_shared_gpu_volatile(double %val, ptr addrspace(3) %addr) {
+; CHECK-LABEL: seq_cst_double_shared_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_shared_gpu_volatile_param_0];
+; CHECK-NEXT: fence.sc.sys;
+; CHECK-NEXT: ld.param.b64 %rd2, [seq_cst_double_shared_gpu_volatile_param_1];
+; CHECK-NEXT: st.volatile.shared.b64 [%rd2], %rd1;
+; CHECK-NEXT: ret;
+ store atomic volatile double %val, ptr addrspace(3) %addr syncscope("device") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_i8_local_sys(i8 %val) {
+; CHECK-LABEL: seq_cst_i8_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot1152[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1152;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b8 %rs1, [seq_cst_i8_local_sys_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b8 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i8 %val, ptr addrspace(5) %slot syncscope("") seq_cst, align 1
+ ret void
+}
+
+define void @seq_cst_i8_local_cta(i8 %val) {
+; CHECK-LABEL: seq_cst_i8_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot1153[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1153;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b8 %rs1, [seq_cst_i8_local_cta_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b8 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i8 %val, ptr addrspace(5) %slot syncscope("block") seq_cst, align 1
+ ret void
+}
+
+define void @seq_cst_i8_local_cluster(i8 %val) {
+; CHECK-LABEL: seq_cst_i8_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot1154[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1154;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b8 %rs1, [seq_cst_i8_local_cluster_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b8 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i8 %val, ptr addrspace(5) %slot syncscope("cluster") seq_cst, align 1
+ ret void
+}
+
+define void @seq_cst_i8_local_gpu(i8 %val) {
+; CHECK-LABEL: seq_cst_i8_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot1155[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1155;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b8 %rs1, [seq_cst_i8_local_gpu_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b8 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i8 %val, ptr addrspace(5) %slot syncscope("device") seq_cst, align 1
+ ret void
+}
+
+define void @seq_cst_i8_local_sys_volatile(i8 %val) {
+; CHECK-LABEL: seq_cst_i8_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot1156[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1156;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b8 %rs1, [seq_cst_i8_local_sys_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b8 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i8 %val, ptr addrspace(5) %slot syncscope("") seq_cst, align 1
+ ret void
+}
+
+define void @seq_cst_i8_local_cta_volatile(i8 %val) {
+; CHECK-LABEL: seq_cst_i8_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot1157[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1157;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b8 %rs1, [seq_cst_i8_local_cta_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b8 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i8 %val, ptr addrspace(5) %slot syncscope("block") seq_cst, align 1
+ ret void
+}
+
+define void @seq_cst_i8_local_cluster_volatile(i8 %val) {
+; CHECK-LABEL: seq_cst_i8_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot1158[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1158;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b8 %rs1, [seq_cst_i8_local_cluster_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b8 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i8 %val, ptr addrspace(5) %slot syncscope("cluster") seq_cst, align 1
+ ret void
+}
+
+define void @seq_cst_i8_local_gpu_volatile(i8 %val) {
+; CHECK-LABEL: seq_cst_i8_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 1 .b8 __local_depot1159[1];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1159;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b8 %rs1, [seq_cst_i8_local_gpu_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b8 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i8, align 1, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i8 %val, ptr addrspace(5) %slot syncscope("device") seq_cst, align 1
+ ret void
+}
+
+define void @seq_cst_i16_local_sys(i16 %val) {
+; CHECK-LABEL: seq_cst_i16_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1160[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1160;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_i16_local_sys_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i16 %val, ptr addrspace(5) %slot syncscope("") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_i16_local_cta(i16 %val) {
+; CHECK-LABEL: seq_cst_i16_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1161[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1161;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_i16_local_cta_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i16 %val, ptr addrspace(5) %slot syncscope("block") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_i16_local_cluster(i16 %val) {
+; CHECK-LABEL: seq_cst_i16_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1162[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1162;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_i16_local_cluster_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i16 %val, ptr addrspace(5) %slot syncscope("cluster") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_i16_local_gpu(i16 %val) {
+; CHECK-LABEL: seq_cst_i16_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1163[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1163;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_i16_local_gpu_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i16 %val, ptr addrspace(5) %slot syncscope("device") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_i16_local_sys_volatile(i16 %val) {
+; CHECK-LABEL: seq_cst_i16_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1164[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1164;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_i16_local_sys_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i16 %val, ptr addrspace(5) %slot syncscope("") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_i16_local_cta_volatile(i16 %val) {
+; CHECK-LABEL: seq_cst_i16_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1165[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1165;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_i16_local_cta_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i16 %val, ptr addrspace(5) %slot syncscope("block") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_i16_local_cluster_volatile(i16 %val) {
+; CHECK-LABEL: seq_cst_i16_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1166[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1166;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_i16_local_cluster_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i16 %val, ptr addrspace(5) %slot syncscope("cluster") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_i16_local_gpu_volatile(i16 %val) {
+; CHECK-LABEL: seq_cst_i16_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1167[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1167;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_i16_local_gpu_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca i16, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i16 %val, ptr addrspace(5) %slot syncscope("device") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_i32_local_sys(i32 %val) {
+; CHECK-LABEL: seq_cst_i32_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot1168[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1168;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [seq_cst_i32_local_sys_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i32 %val, ptr addrspace(5) %slot syncscope("") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_i32_local_cta(i32 %val) {
+; CHECK-LABEL: seq_cst_i32_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot1169[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1169;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [seq_cst_i32_local_cta_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i32 %val, ptr addrspace(5) %slot syncscope("block") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_i32_local_cluster(i32 %val) {
+; CHECK-LABEL: seq_cst_i32_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot1170[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1170;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [seq_cst_i32_local_cluster_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i32 %val, ptr addrspace(5) %slot syncscope("cluster") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_i32_local_gpu(i32 %val) {
+; CHECK-LABEL: seq_cst_i32_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot1171[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1171;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [seq_cst_i32_local_gpu_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i32 %val, ptr addrspace(5) %slot syncscope("device") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_i32_local_sys_volatile(i32 %val) {
+; CHECK-LABEL: seq_cst_i32_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot1172[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1172;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [seq_cst_i32_local_sys_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i32 %val, ptr addrspace(5) %slot syncscope("") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_i32_local_cta_volatile(i32 %val) {
+; CHECK-LABEL: seq_cst_i32_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot1173[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1173;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [seq_cst_i32_local_cta_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i32 %val, ptr addrspace(5) %slot syncscope("block") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_i32_local_cluster_volatile(i32 %val) {
+; CHECK-LABEL: seq_cst_i32_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot1174[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1174;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [seq_cst_i32_local_cluster_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i32 %val, ptr addrspace(5) %slot syncscope("cluster") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_i32_local_gpu_volatile(i32 %val) {
+; CHECK-LABEL: seq_cst_i32_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot1175[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1175;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [seq_cst_i32_local_gpu_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca i32, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i32 %val, ptr addrspace(5) %slot syncscope("device") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_i64_local_sys(i64 %val) {
+; CHECK-LABEL: seq_cst_i64_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot1176[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1176;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_local_sys_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i64 %val, ptr addrspace(5) %slot syncscope("") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_i64_local_cta(i64 %val) {
+; CHECK-LABEL: seq_cst_i64_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot1177[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1177;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_local_cta_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i64 %val, ptr addrspace(5) %slot syncscope("block") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_i64_local_cluster(i64 %val) {
+; CHECK-LABEL: seq_cst_i64_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot1178[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1178;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_local_cluster_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i64 %val, ptr addrspace(5) %slot syncscope("cluster") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_i64_local_gpu(i64 %val) {
+; CHECK-LABEL: seq_cst_i64_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot1179[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1179;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_local_gpu_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i64 %val, ptr addrspace(5) %slot syncscope("device") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_i64_local_sys_volatile(i64 %val) {
+; CHECK-LABEL: seq_cst_i64_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot1180[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1180;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_local_sys_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i64 %val, ptr addrspace(5) %slot syncscope("") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_i64_local_cta_volatile(i64 %val) {
+; CHECK-LABEL: seq_cst_i64_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot1181[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1181;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_local_cta_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i64 %val, ptr addrspace(5) %slot syncscope("block") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_i64_local_cluster_volatile(i64 %val) {
+; CHECK-LABEL: seq_cst_i64_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot1182[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1182;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_local_cluster_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i64 %val, ptr addrspace(5) %slot syncscope("cluster") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_i64_local_gpu_volatile(i64 %val) {
+; CHECK-LABEL: seq_cst_i64_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot1183[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1183;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_i64_local_gpu_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca i64, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i64 %val, ptr addrspace(5) %slot syncscope("device") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_i128_local_sys(i128 %val) {
+; CHECK-LABEL: seq_cst_i128_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 16 .b8 __local_depot1184[16];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1184;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [seq_cst_i128_local_sys_param_0];
+; CHECK-NEXT: add.u64 %rd3, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd3;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.v2.b64 [%SP], {%rd1, %rd2};
+; CHECK-NEXT: ret;
+ %slot = alloca i128, align 16, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i128 %val, ptr addrspace(5) %slot syncscope("") seq_cst, align 16
+ ret void
+}
+
+define void @seq_cst_i128_local_cta(i128 %val) {
+; CHECK-LABEL: seq_cst_i128_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 16 .b8 __local_depot1185[16];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1185;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [seq_cst_i128_local_cta_param_0];
+; CHECK-NEXT: add.u64 %rd3, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd3;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.v2.b64 [%SP], {%rd1, %rd2};
+; CHECK-NEXT: ret;
+ %slot = alloca i128, align 16, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i128 %val, ptr addrspace(5) %slot syncscope("block") seq_cst, align 16
+ ret void
+}
+
+define void @seq_cst_i128_local_cluster(i128 %val) {
+; CHECK-LABEL: seq_cst_i128_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 16 .b8 __local_depot1186[16];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1186;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [seq_cst_i128_local_cluster_param_0];
+; CHECK-NEXT: add.u64 %rd3, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd3;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.v2.b64 [%SP], {%rd1, %rd2};
+; CHECK-NEXT: ret;
+ %slot = alloca i128, align 16, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i128 %val, ptr addrspace(5) %slot syncscope("cluster") seq_cst, align 16
+ ret void
+}
+
+define void @seq_cst_i128_local_gpu(i128 %val) {
+; CHECK-LABEL: seq_cst_i128_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 16 .b8 __local_depot1187[16];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1187;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [seq_cst_i128_local_gpu_param_0];
+; CHECK-NEXT: add.u64 %rd3, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd3;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.v2.b64 [%SP], {%rd1, %rd2};
+; CHECK-NEXT: ret;
+ %slot = alloca i128, align 16, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic i128 %val, ptr addrspace(5) %slot syncscope("device") seq_cst, align 16
+ ret void
+}
+
+define void @seq_cst_i128_local_sys_volatile(i128 %val) {
+; CHECK-LABEL: seq_cst_i128_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 16 .b8 __local_depot1188[16];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1188;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [seq_cst_i128_local_sys_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd3, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd3;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.v2.b64 [%SP], {%rd1, %rd2};
+; CHECK-NEXT: ret;
+ %slot = alloca i128, align 16, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i128 %val, ptr addrspace(5) %slot syncscope("") seq_cst, align 16
+ ret void
+}
+
+define void @seq_cst_i128_local_cta_volatile(i128 %val) {
+; CHECK-LABEL: seq_cst_i128_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 16 .b8 __local_depot1189[16];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1189;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [seq_cst_i128_local_cta_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd3, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd3;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.v2.b64 [%SP], {%rd1, %rd2};
+; CHECK-NEXT: ret;
+ %slot = alloca i128, align 16, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i128 %val, ptr addrspace(5) %slot syncscope("block") seq_cst, align 16
+ ret void
+}
+
+define void @seq_cst_i128_local_cluster_volatile(i128 %val) {
+; CHECK-LABEL: seq_cst_i128_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 16 .b8 __local_depot1190[16];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1190;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [seq_cst_i128_local_cluster_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd3, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd3;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.v2.b64 [%SP], {%rd1, %rd2};
+; CHECK-NEXT: ret;
+ %slot = alloca i128, align 16, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i128 %val, ptr addrspace(5) %slot syncscope("cluster") seq_cst, align 16
+ ret void
+}
+
+define void @seq_cst_i128_local_gpu_volatile(i128 %val) {
+; CHECK-LABEL: seq_cst_i128_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 16 .b8 __local_depot1191[16];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1191;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [seq_cst_i128_local_gpu_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd3, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd3;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.v2.b64 [%SP], {%rd1, %rd2};
+; CHECK-NEXT: ret;
+ %slot = alloca i128, align 16, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile i128 %val, ptr addrspace(5) %slot syncscope("device") seq_cst, align 16
+ ret void
+}
+
+define void @seq_cst_half_local_sys(half %val) {
+; CHECK-LABEL: seq_cst_half_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1192[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1192;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_half_local_sys_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic half %val, ptr addrspace(5) %slot syncscope("") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_half_local_cta(half %val) {
+; CHECK-LABEL: seq_cst_half_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1193[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1193;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_half_local_cta_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic half %val, ptr addrspace(5) %slot syncscope("block") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_half_local_cluster(half %val) {
+; CHECK-LABEL: seq_cst_half_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1194[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1194;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_half_local_cluster_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic half %val, ptr addrspace(5) %slot syncscope("cluster") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_half_local_gpu(half %val) {
+; CHECK-LABEL: seq_cst_half_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1195[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1195;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_half_local_gpu_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic half %val, ptr addrspace(5) %slot syncscope("device") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_half_local_sys_volatile(half %val) {
+; CHECK-LABEL: seq_cst_half_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1196[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1196;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_half_local_sys_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile half %val, ptr addrspace(5) %slot syncscope("") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_half_local_cta_volatile(half %val) {
+; CHECK-LABEL: seq_cst_half_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1197[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1197;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_half_local_cta_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile half %val, ptr addrspace(5) %slot syncscope("block") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_half_local_cluster_volatile(half %val) {
+; CHECK-LABEL: seq_cst_half_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1198[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1198;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_half_local_cluster_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile half %val, ptr addrspace(5) %slot syncscope("cluster") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_half_local_gpu_volatile(half %val) {
+; CHECK-LABEL: seq_cst_half_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1199[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1199;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_half_local_gpu_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca half, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile half %val, ptr addrspace(5) %slot syncscope("device") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_bfloat_local_sys(bfloat %val) {
+; CHECK-LABEL: seq_cst_bfloat_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1200[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1200;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_bfloat_local_sys_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic bfloat %val, ptr addrspace(5) %slot syncscope("") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_bfloat_local_cta(bfloat %val) {
+; CHECK-LABEL: seq_cst_bfloat_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1201[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1201;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_bfloat_local_cta_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic bfloat %val, ptr addrspace(5) %slot syncscope("block") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_bfloat_local_cluster(bfloat %val) {
+; CHECK-LABEL: seq_cst_bfloat_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1202[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1202;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_bfloat_local_cluster_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic bfloat %val, ptr addrspace(5) %slot syncscope("cluster") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_bfloat_local_gpu(bfloat %val) {
+; CHECK-LABEL: seq_cst_bfloat_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1203[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1203;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_bfloat_local_gpu_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic bfloat %val, ptr addrspace(5) %slot syncscope("device") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_bfloat_local_sys_volatile(bfloat %val) {
+; CHECK-LABEL: seq_cst_bfloat_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1204[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1204;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_bfloat_local_sys_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile bfloat %val, ptr addrspace(5) %slot syncscope("") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_bfloat_local_cta_volatile(bfloat %val) {
+; CHECK-LABEL: seq_cst_bfloat_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1205[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1205;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_bfloat_local_cta_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile bfloat %val, ptr addrspace(5) %slot syncscope("block") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_bfloat_local_cluster_volatile(bfloat %val) {
+; CHECK-LABEL: seq_cst_bfloat_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1206[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1206;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_bfloat_local_cluster_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile bfloat %val, ptr addrspace(5) %slot syncscope("cluster") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_bfloat_local_gpu_volatile(bfloat %val) {
+; CHECK-LABEL: seq_cst_bfloat_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 2 .b8 __local_depot1207[2];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1207;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b16 %rs1, [seq_cst_bfloat_local_gpu_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b16 [%SP], %rs1;
+; CHECK-NEXT: ret;
+ %slot = alloca bfloat, align 2, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile bfloat %val, ptr addrspace(5) %slot syncscope("device") seq_cst, align 2
+ ret void
+}
+
+define void @seq_cst_float_local_sys(float %val) {
+; CHECK-LABEL: seq_cst_float_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot1208[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1208;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [seq_cst_float_local_sys_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic float %val, ptr addrspace(5) %slot syncscope("") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_float_local_cta(float %val) {
+; CHECK-LABEL: seq_cst_float_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot1209[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1209;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [seq_cst_float_local_cta_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic float %val, ptr addrspace(5) %slot syncscope("block") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_float_local_cluster(float %val) {
+; CHECK-LABEL: seq_cst_float_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot1210[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1210;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [seq_cst_float_local_cluster_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic float %val, ptr addrspace(5) %slot syncscope("cluster") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_float_local_gpu(float %val) {
+; CHECK-LABEL: seq_cst_float_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot1211[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1211;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [seq_cst_float_local_gpu_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic float %val, ptr addrspace(5) %slot syncscope("device") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_float_local_sys_volatile(float %val) {
+; CHECK-LABEL: seq_cst_float_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot1212[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1212;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [seq_cst_float_local_sys_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile float %val, ptr addrspace(5) %slot syncscope("") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_float_local_cta_volatile(float %val) {
+; CHECK-LABEL: seq_cst_float_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot1213[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1213;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [seq_cst_float_local_cta_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile float %val, ptr addrspace(5) %slot syncscope("block") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_float_local_cluster_volatile(float %val) {
+; CHECK-LABEL: seq_cst_float_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot1214[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1214;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [seq_cst_float_local_cluster_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile float %val, ptr addrspace(5) %slot syncscope("cluster") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_float_local_gpu_volatile(float %val) {
+; CHECK-LABEL: seq_cst_float_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 4 .b8 __local_depot1215[4];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1215;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b32 %r2, [seq_cst_float_local_gpu_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd1, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd1;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b32 [%SP], %r2;
+; CHECK-NEXT: ret;
+ %slot = alloca float, align 4, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile float %val, ptr addrspace(5) %slot syncscope("device") seq_cst, align 4
+ ret void
+}
+
+define void @seq_cst_double_local_sys(double %val) {
+; CHECK-LABEL: seq_cst_double_local_sys(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot1216[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1216;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_local_sys_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic double %val, ptr addrspace(5) %slot syncscope("") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_double_local_cta(double %val) {
+; CHECK-LABEL: seq_cst_double_local_cta(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot1217[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1217;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_local_cta_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic double %val, ptr addrspace(5) %slot syncscope("block") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_double_local_cluster(double %val) {
+; CHECK-LABEL: seq_cst_double_local_cluster(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot1218[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1218;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_local_cluster_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic double %val, ptr addrspace(5) %slot syncscope("cluster") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_double_local_gpu(double %val) {
+; CHECK-LABEL: seq_cst_double_local_gpu(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot1219[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1219;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_local_gpu_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic double %val, ptr addrspace(5) %slot syncscope("device") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_double_local_sys_volatile(double %val) {
+; CHECK-LABEL: seq_cst_double_local_sys_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot1220[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1220;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_local_sys_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile double %val, ptr addrspace(5) %slot syncscope("") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_double_local_cta_volatile(double %val) {
+; CHECK-LABEL: seq_cst_double_local_cta_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot1221[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1221;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_local_cta_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile double %val, ptr addrspace(5) %slot syncscope("block") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_double_local_cluster_volatile(double %val) {
+; CHECK-LABEL: seq_cst_double_local_cluster_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot1222[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1222;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_local_cluster_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile double %val, ptr addrspace(5) %slot syncscope("cluster") seq_cst, align 8
+ ret void
+}
+
+define void @seq_cst_double_local_gpu_volatile(double %val) {
+; CHECK-LABEL: seq_cst_double_local_gpu_volatile(
+; CHECK: {
+; CHECK-NEXT: .local .align 8 .b8 __local_depot1223[8];
+; CHECK-NEXT: .reg .b64 %SP;
+; CHECK-NEXT: .reg .b64 %SPL;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov.b64 %SPL, __local_depot1223;
+; CHECK-NEXT: cvta.local.u64 %SP, %SPL;
+; CHECK-NEXT: ld.param.b64 %rd1, [seq_cst_double_local_gpu_volatile_param_0];
+; CHECK-NEXT: add.u64 %rd2, %SP, 0;
+; CHECK-NEXT: cvt.u32.u64 %r1, %rd2;
+; CHECK-NEXT: // begin inline asm
+; CHECK-NEXT: // end inline asm
+; CHECK-NEXT: st.local.b64 [%SP], %rd1;
+; CHECK-NEXT: ret;
+ %slot = alloca double, align 8, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic volatile double %val, ptr addrspace(5) %slot syncscope("device") seq_cst, align 8
+ ret void
+}
+
diff --git a/llvm/test/CodeGen/NVPTX/store.py b/llvm/test/CodeGen/NVPTX/store.py
new file mode 100644
index 0000000000000..29159bd04a795
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/store.py
@@ -0,0 +1,206 @@
+from string import Template
+from itertools import product
+
+# From https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-st, Target ISA notes:
+
+# PTX
+# st introduced in PTX ISA version 1.0.
+# st.volatile introduced in PTX ISA version 1.1.
+# Generic addressing and cache operations introduced in PTX ISA version 2.0.
+# Support for scope qualifier, .relaxed, .release, .weak qualifiers introduced in PTX ISA version 6.0.
+# Support for .level1::eviction_priority and .level::cache_hint qualifiers introduced in PTX ISA version 7.4.
+# Support for .cluster scope qualifier introduced in PTX ISA version 7.8.
+# Support for ::cta and ::cluster sub-qualifiers introduced in PTX ISA version 7.8.
+# Support for .mmio qualifier introduced in PTX ISA version 8.2.
+# Support for ::func sub-qualifier on .param space introduced in PTX ISA version 8.3.
+# Support for .b128 type introduced in PTX ISA version 8.3.
+# Support for .sys scope with .b128 type introduced in PTX ISA version 8.4.
+# Support for .level2::eviction_priority qualifier and .v8.b32/.v4.b64 introduced in PTX ISA version 8.8.
+# Support for .volatile qualifier with .local state space introduced in PTX ISA version 9.1.
+
+# SM
+# st.f64 requires sm_13 or higher.
+# Generic addressing requires sm_20 or higher.
+# Cache operations require sm_20 or higher.
+# Sub-qualifier ::cta requires sm_30 or higher.
+# Support for scope qualifier, .relaxed, .release, .weak qualifiers require sm_70 or higher.
+# Support for .level1::eviction_priority qualifier requires sm_70 or higher.
+# Support for .mmio qualifier requires sm_70 or higher.
+# Support for .b128 type requires sm_70 or higher.
+# Support for .level::cache_hint qualifier requires sm_80 or higher.
+# Support for .cluster scope qualifier requires sm_90 or higher.
+# Sub-qualifier ::cluster requires sm_90 or higher.
+# Support for .level2::eviction_priority qualifier and .v8.b32/.v4.b64 require sm_100 or higher.
+
+# Feature compatibility matrix
+# ----------------------------
+# Features actually exercised by this test (each gated by both an SM and a PTX
+# threshold; the feature lights up only when both thresholds are satisfied):
+# A = scope qualifier / .relaxed / .release / .weak sm_70 + PTX 6.0
+# B = .cluster scope sm_90 + PTX 7.8
+# C = .b128 type sm_70 + PTX 8.3
+# D = .sys + .b128 sm_70 + PTX 8.4
+# E = .volatile + .local (any) + PTX 9.1
+#
+# PTX 5.0 PTX 6.0 PTX 7.8 PTX 8.3 PTX 8.4 PTX 9.1
+# sm_30 -- -- -- -- -- E
+# sm_70 -- A A A C A C D A C D E
+# sm_90 -- A A B A B C A B C D A B C D E
+#
+# Each cell lists the features valid at that (SM, PTX); blank means no feature
+# from this set is supported. A feature's "subgrid" is the rectangle of cells
+# containing its letter; the subgrid's top-left cell is the minimal (SM, PTX)
+# pair that turns the feature on.
+
+# One PTX version per SM (matching cmpxchg.py / atomicrmw.py). The previous
+# multi-PTX sweep produced identical codegen on every slice axis.
+TEST_SM_PTX_ARCHS = [(60, 50), (70, 63), (90, 87)]
+
+SCOPE_LLVM_TO_PTX = {"": "sys", "block": "cta", "cluster": "cluster", "device": "gpu"}
+
+ORDERINGS = ["notatomic", "unordered", "monotonic", "release", "seq_cst"]
+
+ADDRSPACE_NUM_TO_ADDRSPACE = {0: "generic", 1: "global", 3: "shared", 4: "const", 5: "local", 101: "param"}
+
+DATATYPE_SIZE_BITS = {
+ "i8": 8, "i16": 16, "i32": 32, "i64": 64, "i128": 128,
+ "half": 16, "bfloat": 16, "float": 32, "double": 64,
+}
+DATATYPES = list(DATATYPE_SIZE_BITS.keys())
+
+VOLATILES = [False, True]
+
+
+def get_addrspace_cast(addrspace):
+ if addrspace == 0:
+ return ""
+ return " addrspace({})".format(addrspace)
+
+
+store_atomic_func = Template(
+ """define void @${func_name}(${datatype} %val, ptr${addrspace_cast} %addr) {
+ store atomic ${volatile_kw}${datatype} %val, ptr${addrspace_cast} %addr syncscope("${llvm_scope}") ${ordering}, align ${align}
+ ret void
+}
+"""
+)
+
+store_nonatomic_func = Template(
+ """define void @${func_name}(${datatype} %val, ptr${addrspace_cast} %addr) {
+ store ${volatile_kw}${datatype} %val, ptr${addrspace_cast} %addr
+ ret void
+}
+"""
+)
+
+local_atomic_store_func = Template(
+ """define void @${func_name}(${datatype} %val) {
+ %slot = alloca ${datatype}, align ${align}, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store atomic ${volatile_kw}${datatype} %val, ptr addrspace(5) %slot syncscope("${llvm_scope}") ${ordering}, align ${align}
+ ret void
+}
+"""
+)
+
+local_nonatomic_store_func = Template(
+ """define void @${func_name}(${datatype} %val) {
+ %slot = alloca ${datatype}, addrspace(5)
+ call void asm sideeffect "", "r"(ptr addrspace(5) %slot)
+ store ${volatile_kw}${datatype} %val, ptr addrspace(5) %slot
+ ret void
+}
+"""
+)
+
+
+def get_store_func(datatype, addrspace, scope, ordering, volatile):
+ """Pick template + fill slots from (datatype, addrspace, scope, ordering, volatile).
+
+ - ordering == "notatomic" -> non-atomic (scope unused)
+ - addrspace == 5 -> local (alloca + inline-asm address capture)
+ - volatile -> emits `volatile` keyword and suffixes the name
+ """
+ size = DATATYPE_SIZE_BITS[datatype]
+ align = size // 8
+ addrspace_name = ADDRSPACE_NUM_TO_ADDRSPACE[addrspace]
+ is_atomic = ordering != "notatomic"
+ is_local = addrspace == 5
+ volatile_kw = "volatile " if volatile else ""
+ volatile_suffix = "_volatile" if volatile else ""
+
+ if is_atomic:
+ ptx_scope = SCOPE_LLVM_TO_PTX[scope]
+ func_name = "{}_{}_{}_{}{}".format(
+ ordering, datatype, addrspace_name, ptx_scope, volatile_suffix
+ )
+ if is_local:
+ return local_atomic_store_func.substitute(
+ datatype=datatype,
+ func_name=func_name,
+ llvm_scope=scope,
+ ordering=ordering,
+ align=align,
+ volatile_kw=volatile_kw,
+ )
+ return store_atomic_func.substitute(
+ datatype=datatype,
+ func_name=func_name,
+ addrspace_cast=get_addrspace_cast(addrspace),
+ llvm_scope=scope,
+ ordering=ordering,
+ align=align,
+ volatile_kw=volatile_kw,
+ )
+
+ func_name = "notatomic_{}_{}{}".format(datatype, addrspace_name, volatile_suffix)
+ if is_local:
+ return local_nonatomic_store_func.substitute(
+ datatype=datatype,
+ func_name=func_name,
+ volatile_kw=volatile_kw,
+ )
+ return store_nonatomic_func.substitute(
+ datatype=datatype,
+ func_name=func_name,
+ addrspace_cast=get_addrspace_cast(addrspace),
+ volatile_kw=volatile_kw,
+ )
+
+
+run_statement = Template(
+ """; RUN: llc < %s -march=nvptx64 -mcpu=sm_${sm} -mattr=+ptx${ptx} | FileCheck %s
+; RUN: %if ptxas-sm_${sm} && ptxas-isa-${ptxfp} %{ llc < %s -march=nvptx64 -mcpu=sm_${sm} -mattr=+ptx${ptx} | %ptxas-verify -arch=sm_${sm} %}
+"""
+)
+
+if __name__ == "__main__":
+ # NOTE: stores can't be `acquire` or `acq_rel` per LangRef. On sm < 70 the
+ # backend used to reject orderings other than notatomic/monotonic, but
+ # AtomicExpandPass now emulates them, so the same axes work uniformly.
+
+ for sm, ptx in TEST_SM_PTX_ARCHS:
+ with open("store-sm{}.ll".format(str(sm)), "w") as fp:
+ print(run_statement.substitute(sm=sm, ptx=ptx, ptxfp=ptx / 10.0), file=fp)
+
+ # All combinations of (ordering x addrspace x datatype x volatile x
+ # scope). Stores to .const/.param are skipped entirely (read-only
+ # spaces). i128 only at sm_90 -- atom.b128 requires sm_90+/PTX
+ # 8.3+, and plain st.b128 requires sm_70+/PTX 8.3+; neither holds
+ # at sm_60/70 with our test matrix.
+ # For notatomic, syncscope is illegal -- emit only at scope="".
+ for ordering, addrspace, datatype, volatile, scope in product(
+ ORDERINGS, ADDRSPACE_NUM_TO_ADDRSPACE, DATATYPES, VOLATILES,
+ SCOPE_LLVM_TO_PTX,
+ ):
+ if datatype == "i128" and sm < 90:
+ continue
+ if addrspace in (4, 101):
+ continue
+ is_atomic = ordering != "notatomic"
+ if not is_atomic and scope != "":
+ continue
+ print(
+ get_store_func(datatype, addrspace, scope, ordering, volatile),
+ file=fp,
+ )
diff --git a/llvm/test/lit.cfg.py b/llvm/test/lit.cfg.py
index 09df1e3fd6281..b8c3ac4f8ae9a 100644
--- a/llvm/test/lit.cfg.py
+++ b/llvm/test/lit.cfg.py
@@ -448,8 +448,12 @@ def ptxas_supported_sms(ptxas_executable):
def ptxas_supports_address_size_32(ptxas_executable):
# Linux outputs the error message to stderr, while Windows outputs to stdout.
# Pipe both to stdout to make sure we get the error message.
+ # Pass "-" + empty stdin so ptxas attempts to read input rather than falling
+ # back to a usage banner (which some builds do when an injected default option
+ # like -std-elf conflicts with -m 32).
result = subprocess.run(
- [ptxas_executable, "-m 32"],
+ [ptxas_executable, "-m 32", "-"],
+ input="",
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
text=True,
More information about the llvm-commits
mailing list