[llvm] 3743a27 - [NVPTX] Properly support acquire/release/acq_rel atomics pre-SM70 by emitting membars (#222449)
via llvm-commits
llvm-commits at lists.llvm.org
Wed Sep 30 18:02:07 PDT 2026
Author: Yonah Goldberg
Date: 2026-09-30T18:01:59-07:00
New Revision: 3743a27e0c75e19c28478a4a9248e0bab68f3347
URL: https://github.com/llvm/llvm-project/commit/3743a27e0c75e19c28478a4a9248e0bab68f3347
DIFF: https://github.com/llvm/llvm-project/commit/3743a27e0c75e19c28478a4a9248e0bab68f3347.diff
LOG: [NVPTX] Properly support acquire/release/acq_rel atomics pre-SM70 by emitting membars (#222449)
acquire/release/acq_rel are not supported semantics on pre-SM70 atomics.
We need to implement these by relaxing the atomic to "relaxed" and then
surounding it with membars. This makes the atomic sequentially
consistent, which is a superset of acquire-release behavior. We already
do this for pre-SM70 atomic that cmpxchg expand, just not for atomics
that otherwise are natively supported.
Added:
Modified:
llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
llvm/test/CodeGen/NVPTX/atomicrmw-sm60.ll
llvm/test/CodeGen/NVPTX/cmpxchg-sm60.ll
llvm/test/CodeGen/NVPTX/cmpxchg.ll
Removed:
################################################################################
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
index cde47c7ab3d9a..33ff7f8180d65 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
@@ -7834,8 +7834,9 @@ NVPTXTargetLowering::shouldExpandAtomicRMWInIR(const AtomicRMWInst *AI) const {
bool NVPTXTargetLowering::shouldInsertFencesForAtomic(
const Instruction *I) const {
// This function returns true iff the operation is emulated using a CAS-loop,
- // or if it has the memory order seq_cst (which is not natively supported in
- // the PTX `atom` instruction).
+ // the target does not support memory-order qualifiers, or the operation has
+ // the memory order seq_cst (which is not natively supported in the PTX
+ // `atom` instruction).
//
// atomicrmw and cmpxchg instructions not efficiently supported by PTX
// are lowered to CAS emulation loops that preserve their memory order,
@@ -7843,26 +7844,29 @@ bool NVPTXTargetLowering::shouldInsertFencesForAtomic(
// atom.cas.relaxed.sco instructions within the loop, and fences before and
// after the loop to restore order.
//
- // Atomic instructions efficiently supported by PTX are lowered to
- // `atom.<op>.<sem>.<scope` instruction with their corresponding memory order
- // and scope. Since PTX does not support seq_cst, we emulate it by lowering to
- // a fence.sc followed by an atom according to the PTX atomics ABI
+ // On targets with memory-order qualifiers, atomic instructions efficiently
+ // supported by PTX are lowered to `atom.<op>.<sem>.<scope>` instructions with
+ // their corresponding memory order and scope. Since PTX does not support
+ // seq_cst, we emulate it by lowering to a fence.sc followed by an atom
+ // according to the PTX atomics ABI.
// https://docs.nvidia.com/cuda/ptx-writers-guide-to-interoperability/atomic-abi.html
if (auto *CI = dyn_cast<AtomicCmpXchgInst>(I))
return (cast<IntegerType>(CI->getCompareOperand()->getType())
->getBitWidth() < STI.getMinCmpXchgSizeInBits()) ||
+ !STI.hasMemoryOrdering() ||
CI->getMergedOrdering() == AtomicOrdering::SequentiallyConsistent;
if (auto *RI = dyn_cast<AtomicRMWInst>(I))
return shouldExpandAtomicRMWInIR(RI) == AtomicExpansionKind::CmpXChg ||
+ !STI.hasMemoryOrdering() ||
RI->getOrdering() == AtomicOrdering::SequentiallyConsistent;
return false;
}
AtomicOrdering NVPTXTargetLowering::atomicOperationOrderAfterFenceSplit(
const Instruction *I) const {
- // If the operation is emulated by a CAS-loop, we lower the instruction to
- // atom.<op>.relaxed, since AtomicExpandPass will insert fences for enforcing
- // the correct memory ordering around the CAS loop.
+ // If the operation is emulated by a CAS-loop, or the target does not support
+ // memory-order qualifiers, we set its IR ordering to monotonic.
+ // AtomicExpandPass inserts fences to enforce the original memory ordering.
//
// When the operation is not emulated, but the memory order is seq_cst,
// we must lower to "fence.sc.<scope>; atom.<op>.acquire.<scope>;" to conform
@@ -7878,6 +7882,9 @@ AtomicOrdering NVPTXTargetLowering::atomicOperationOrderAfterFenceSplit(
// will NOT be called.
// prerequisite: shouldInsertFencesForAtomic() should have returned `true` for
// I before its memory order was modified.
+ if (!STI.hasMemoryOrdering())
+ return AtomicOrdering::Monotonic;
+
if (auto *CI = dyn_cast<AtomicCmpXchgInst>(I);
CI && CI->getMergedOrdering() == AtomicOrdering::SequentiallyConsistent &&
cast<IntegerType>(CI->getCompareOperand()->getType())->getBitWidth() >=
@@ -7931,9 +7938,10 @@ Instruction *NVPTXTargetLowering::emitTrailingFence(IRBuilderBase &Builder,
assert(SSID.has_value() && "Expected an atomic operation");
bool IsEmulated =
- CI ? cast<IntegerType>(CI->getCompareOperand()->getType())
- ->getBitWidth() < STI.getMinCmpXchgSizeInBits()
- : shouldExpandAtomicRMWInIR(RI) == AtomicExpansionKind::CmpXChg;
+ !STI.hasMemoryOrdering() ||
+ (CI ? cast<IntegerType>(CI->getCompareOperand()->getType())
+ ->getBitWidth() < STI.getMinCmpXchgSizeInBits()
+ : shouldExpandAtomicRMWInIR(RI) == AtomicExpansionKind::CmpXChg);
if (isAcquireOrStronger(Ord) && IsEmulated)
return Builder.CreateFence(AtomicOrdering::Acquire, SSID.value());
diff --git a/llvm/test/CodeGen/NVPTX/atomicrmw-sm60.ll b/llvm/test/CodeGen/NVPTX/atomicrmw-sm60.ll
index 5a43d78501ae0..9f3f24902100e 100644
--- a/llvm/test/CodeGen/NVPTX/atomicrmw-sm60.ll
+++ b/llvm/test/CodeGen/NVPTX/atomicrmw-sm60.ll
@@ -94,8 +94,10 @@ define i32 @xchg_acq_rel_i32_global_cta(ptr addrspace(1) %addr, i32 %val) {
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd1, [xchg_acq_rel_i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
; SM60-NEXT: ld.param.b32 %r1, [xchg_acq_rel_i32_global_cta_param_1];
; SM60-NEXT: atom.cta.global.exch.b32 %r2, [%rd1], %r1;
+; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b32 [func_retval0], %r2;
; SM60-NEXT: ret;
%retval = atomicrmw xchg ptr addrspace(1) %addr, i32 %val syncscope("block") acq_rel
@@ -109,8 +111,10 @@ define i64 @xchg_acq_rel_i64_global_cta(ptr addrspace(1) %addr, i64 %val) {
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd1, [xchg_acq_rel_i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
; SM60-NEXT: ld.param.b64 %rd2, [xchg_acq_rel_i64_global_cta_param_1];
; SM60-NEXT: atom.cta.global.exch.b64 %rd3, [%rd1], %rd2;
+; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b64 [func_retval0], %rd3;
; SM60-NEXT: ret;
%retval = atomicrmw xchg ptr addrspace(1) %addr, i64 %val syncscope("block") acq_rel
@@ -203,8 +207,10 @@ define i32 @add_acq_rel_i32_global_cta(ptr addrspace(1) %addr, i32 %val) {
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd1, [add_acq_rel_i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
; SM60-NEXT: ld.param.b32 %r1, [add_acq_rel_i32_global_cta_param_1];
; SM60-NEXT: atom.cta.global.add.u32 %r2, [%rd1], %r1;
+; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b32 [func_retval0], %r2;
; SM60-NEXT: ret;
%retval = atomicrmw add ptr addrspace(1) %addr, i32 %val syncscope("block") acq_rel
@@ -218,8 +224,10 @@ define i64 @add_acq_rel_i64_global_cta(ptr addrspace(1) %addr, i64 %val) {
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd1, [add_acq_rel_i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
; SM60-NEXT: ld.param.b64 %rd2, [add_acq_rel_i64_global_cta_param_1];
; SM60-NEXT: atom.cta.global.add.u64 %rd3, [%rd1], %rd2;
+; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b64 [func_retval0], %rd3;
; SM60-NEXT: ret;
%retval = atomicrmw add ptr addrspace(1) %addr, i64 %val syncscope("block") acq_rel
@@ -312,9 +320,11 @@ define i32 @sub_acq_rel_i32_global_cta(ptr addrspace(1) %addr, i32 %val) {
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd1, [sub_acq_rel_i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
; SM60-NEXT: ld.param.b32 %r1, [sub_acq_rel_i32_global_cta_param_1];
; SM60-NEXT: neg.s32 %r2, %r1;
; SM60-NEXT: atom.cta.global.add.u32 %r3, [%rd1], %r2;
+; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b32 [func_retval0], %r3;
; SM60-NEXT: ret;
%retval = atomicrmw sub ptr addrspace(1) %addr, i32 %val syncscope("block") acq_rel
@@ -328,9 +338,11 @@ define i64 @sub_acq_rel_i64_global_cta(ptr addrspace(1) %addr, i64 %val) {
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd1, [sub_acq_rel_i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
; SM60-NEXT: ld.param.b64 %rd2, [sub_acq_rel_i64_global_cta_param_1];
; SM60-NEXT: neg.s64 %rd3, %rd2;
; SM60-NEXT: atom.cta.global.add.u64 %rd4, [%rd1], %rd3;
+; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b64 [func_retval0], %rd4;
; SM60-NEXT: ret;
%retval = atomicrmw sub ptr addrspace(1) %addr, i64 %val syncscope("block") acq_rel
@@ -401,8 +413,10 @@ define i32 @and_acq_rel_i32_global_cta(ptr addrspace(1) %addr, i32 %val) {
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd1, [and_acq_rel_i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
; SM60-NEXT: ld.param.b32 %r1, [and_acq_rel_i32_global_cta_param_1];
; SM60-NEXT: atom.cta.global.and.b32 %r2, [%rd1], %r1;
+; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b32 [func_retval0], %r2;
; SM60-NEXT: ret;
%retval = atomicrmw and ptr addrspace(1) %addr, i32 %val syncscope("block") acq_rel
@@ -416,8 +430,10 @@ define i64 @and_acq_rel_i64_global_cta(ptr addrspace(1) %addr, i64 %val) {
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd1, [and_acq_rel_i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
; SM60-NEXT: ld.param.b64 %rd2, [and_acq_rel_i64_global_cta_param_1];
; SM60-NEXT: atom.cta.global.and.b64 %rd3, [%rd1], %rd2;
+; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b64 [func_retval0], %rd3;
; SM60-NEXT: ret;
%retval = atomicrmw and ptr addrspace(1) %addr, i64 %val syncscope("block") acq_rel
@@ -615,8 +631,10 @@ define i32 @or_acq_rel_i32_global_cta(ptr addrspace(1) %addr, i32 %val) {
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd1, [or_acq_rel_i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
; SM60-NEXT: ld.param.b32 %r1, [or_acq_rel_i32_global_cta_param_1];
; SM60-NEXT: atom.cta.global.or.b32 %r2, [%rd1], %r1;
+; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b32 [func_retval0], %r2;
; SM60-NEXT: ret;
%retval = atomicrmw or ptr addrspace(1) %addr, i32 %val syncscope("block") acq_rel
@@ -630,8 +648,10 @@ define i64 @or_acq_rel_i64_global_cta(ptr addrspace(1) %addr, i64 %val) {
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd1, [or_acq_rel_i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
; SM60-NEXT: ld.param.b64 %rd2, [or_acq_rel_i64_global_cta_param_1];
; SM60-NEXT: atom.cta.global.or.b64 %rd3, [%rd1], %rd2;
+; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b64 [func_retval0], %rd3;
; SM60-NEXT: ret;
%retval = atomicrmw or ptr addrspace(1) %addr, i64 %val syncscope("block") acq_rel
@@ -694,8 +714,10 @@ define i32 @xor_acq_rel_i32_global_cta(ptr addrspace(1) %addr, i32 %val) {
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd1, [xor_acq_rel_i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
; SM60-NEXT: ld.param.b32 %r1, [xor_acq_rel_i32_global_cta_param_1];
; SM60-NEXT: atom.cta.global.xor.b32 %r2, [%rd1], %r1;
+; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b32 [func_retval0], %r2;
; SM60-NEXT: ret;
%retval = atomicrmw xor ptr addrspace(1) %addr, i32 %val syncscope("block") acq_rel
@@ -709,8 +731,10 @@ define i64 @xor_acq_rel_i64_global_cta(ptr addrspace(1) %addr, i64 %val) {
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd1, [xor_acq_rel_i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
; SM60-NEXT: ld.param.b64 %rd2, [xor_acq_rel_i64_global_cta_param_1];
; SM60-NEXT: atom.cta.global.xor.b64 %rd3, [%rd1], %rd2;
+; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b64 [func_retval0], %rd3;
; SM60-NEXT: ret;
%retval = atomicrmw xor ptr addrspace(1) %addr, i64 %val syncscope("block") acq_rel
@@ -811,8 +835,10 @@ define i32 @max_acq_rel_i32_global_cta(ptr addrspace(1) %addr, i32 %val) {
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd1, [max_acq_rel_i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
; SM60-NEXT: ld.param.b32 %r1, [max_acq_rel_i32_global_cta_param_1];
; SM60-NEXT: atom.cta.global.max.s32 %r2, [%rd1], %r1;
+; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b32 [func_retval0], %r2;
; SM60-NEXT: ret;
%retval = atomicrmw max ptr addrspace(1) %addr, i32 %val syncscope("block") acq_rel
@@ -826,8 +852,10 @@ define i64 @max_acq_rel_i64_global_cta(ptr addrspace(1) %addr, i64 %val) {
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd1, [max_acq_rel_i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
; SM60-NEXT: ld.param.b64 %rd2, [max_acq_rel_i64_global_cta_param_1];
; SM60-NEXT: atom.cta.global.max.s64 %rd3, [%rd1], %rd2;
+; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b64 [func_retval0], %rd3;
; SM60-NEXT: ret;
%retval = atomicrmw max ptr addrspace(1) %addr, i64 %val syncscope("block") acq_rel
@@ -928,8 +956,10 @@ define i32 @min_acq_rel_i32_global_cta(ptr addrspace(1) %addr, i32 %val) {
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd1, [min_acq_rel_i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
; SM60-NEXT: ld.param.b32 %r1, [min_acq_rel_i32_global_cta_param_1];
; SM60-NEXT: atom.cta.global.min.s32 %r2, [%rd1], %r1;
+; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b32 [func_retval0], %r2;
; SM60-NEXT: ret;
%retval = atomicrmw min ptr addrspace(1) %addr, i32 %val syncscope("block") acq_rel
@@ -943,8 +973,10 @@ define i64 @min_acq_rel_i64_global_cta(ptr addrspace(1) %addr, i64 %val) {
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd1, [min_acq_rel_i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
; SM60-NEXT: ld.param.b64 %rd2, [min_acq_rel_i64_global_cta_param_1];
; SM60-NEXT: atom.cta.global.min.s64 %rd3, [%rd1], %rd2;
+; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b64 [func_retval0], %rd3;
; SM60-NEXT: ret;
%retval = atomicrmw min ptr addrspace(1) %addr, i64 %val syncscope("block") acq_rel
@@ -1044,8 +1076,10 @@ define i32 @umax_acq_rel_i32_global_cta(ptr addrspace(1) %addr, i32 %val) {
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd1, [umax_acq_rel_i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
; SM60-NEXT: ld.param.b32 %r1, [umax_acq_rel_i32_global_cta_param_1];
; SM60-NEXT: atom.cta.global.max.u32 %r2, [%rd1], %r1;
+; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b32 [func_retval0], %r2;
; SM60-NEXT: ret;
%retval = atomicrmw umax ptr addrspace(1) %addr, i32 %val syncscope("block") acq_rel
@@ -1059,8 +1093,10 @@ define i64 @umax_acq_rel_i64_global_cta(ptr addrspace(1) %addr, i64 %val) {
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd1, [umax_acq_rel_i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
; SM60-NEXT: ld.param.b64 %rd2, [umax_acq_rel_i64_global_cta_param_1];
; SM60-NEXT: atom.cta.global.max.u64 %rd3, [%rd1], %rd2;
+; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b64 [func_retval0], %rd3;
; SM60-NEXT: ret;
%retval = atomicrmw umax ptr addrspace(1) %addr, i64 %val syncscope("block") acq_rel
@@ -1160,8 +1196,10 @@ define i32 @umin_acq_rel_i32_global_cta(ptr addrspace(1) %addr, i32 %val) {
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd1, [umin_acq_rel_i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
; SM60-NEXT: ld.param.b32 %r1, [umin_acq_rel_i32_global_cta_param_1];
; SM60-NEXT: atom.cta.global.min.u32 %r2, [%rd1], %r1;
+; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b32 [func_retval0], %r2;
; SM60-NEXT: ret;
%retval = atomicrmw umin ptr addrspace(1) %addr, i32 %val syncscope("block") acq_rel
@@ -1175,8 +1213,10 @@ define i64 @umin_acq_rel_i64_global_cta(ptr addrspace(1) %addr, i64 %val) {
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd1, [umin_acq_rel_i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
; SM60-NEXT: ld.param.b64 %rd2, [umin_acq_rel_i64_global_cta_param_1];
; SM60-NEXT: atom.cta.global.min.u64 %rd3, [%rd1], %rd2;
+; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b64 [func_retval0], %rd3;
; SM60-NEXT: ret;
%retval = atomicrmw umin ptr addrspace(1) %addr, i64 %val syncscope("block") acq_rel
@@ -1281,8 +1321,10 @@ define i32 @uinc_wrap_acq_rel_i32_global_cta(ptr addrspace(1) %addr, i32 %val) {
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd1, [uinc_wrap_acq_rel_i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
; SM60-NEXT: ld.param.b32 %r1, [uinc_wrap_acq_rel_i32_global_cta_param_1];
; SM60-NEXT: atom.cta.global.inc.u32 %r2, [%rd1], %r1;
+; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b32 [func_retval0], %r2;
; SM60-NEXT: ret;
%retval = atomicrmw uinc_wrap ptr addrspace(1) %addr, i32 %val syncscope("block") acq_rel
@@ -1419,8 +1461,10 @@ define i32 @udec_wrap_acq_rel_i32_global_cta(ptr addrspace(1) %addr, i32 %val) {
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd1, [udec_wrap_acq_rel_i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
; SM60-NEXT: ld.param.b32 %r1, [udec_wrap_acq_rel_i32_global_cta_param_1];
; SM60-NEXT: atom.cta.global.dec.u32 %r2, [%rd1], %r1;
+; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b32 [func_retval0], %r2;
; SM60-NEXT: ret;
%retval = atomicrmw udec_wrap ptr addrspace(1) %addr, i32 %val syncscope("block") acq_rel
@@ -1777,8 +1821,10 @@ define float @fadd_acq_rel_float_global_cta(ptr addrspace(1) %addr, float %val)
; SM60-NOFTZ-ALLOW-EMPTY:
; SM60-NOFTZ-ALLOW-NEXT: // %bb.0:
; SM60-NOFTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_cta_param_0];
+; SM60-NOFTZ-ALLOW-NEXT: membar.cta;
; SM60-NOFTZ-ALLOW-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_global_cta_param_1];
; SM60-NOFTZ-ALLOW-NEXT: atom.cta.global.add.f32 %r2, [%rd1], %r1;
+; SM60-NOFTZ-ALLOW-NEXT: membar.cta;
; SM60-NOFTZ-ALLOW-NEXT: st.param.b32 [func_retval0], %r2;
; SM60-NOFTZ-ALLOW-NEXT: ret;
;
@@ -1789,8 +1835,10 @@ define float @fadd_acq_rel_float_global_cta(ptr addrspace(1) %addr, float %val)
; SM60-FTZ-DISALLOW-EMPTY:
; SM60-FTZ-DISALLOW-NEXT: // %bb.0:
; SM60-FTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_cta_param_0];
+; SM60-FTZ-DISALLOW-NEXT: membar.cta;
; SM60-FTZ-DISALLOW-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_global_cta_param_1];
; SM60-FTZ-DISALLOW-NEXT: atom.cta.global.add.f32 %r2, [%rd1], %r1;
+; SM60-FTZ-DISALLOW-NEXT: membar.cta;
; SM60-FTZ-DISALLOW-NEXT: st.param.b32 [func_retval0], %r2;
; SM60-FTZ-DISALLOW-NEXT: ret;
;
@@ -1801,8 +1849,10 @@ define float @fadd_acq_rel_float_global_cta(ptr addrspace(1) %addr, float %val)
; SM60-FTZ-ALLOW-EMPTY:
; SM60-FTZ-ALLOW-NEXT: // %bb.0:
; SM60-FTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_cta_param_0];
+; SM60-FTZ-ALLOW-NEXT: membar.cta;
; SM60-FTZ-ALLOW-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_global_cta_param_1];
; SM60-FTZ-ALLOW-NEXT: atom.cta.global.add.f32 %r2, [%rd1], %r1;
+; SM60-FTZ-ALLOW-NEXT: membar.cta;
; SM60-FTZ-ALLOW-NEXT: st.param.b32 [func_retval0], %r2;
; SM60-FTZ-ALLOW-NEXT: ret;
%retval = atomicrmw fadd ptr addrspace(1) %addr, float %val syncscope("block") acq_rel
@@ -2344,8 +2394,10 @@ define double @fadd_acq_rel_double_global_cta(ptr addrspace(1) %addr, double %va
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_double_global_cta_param_0];
+; SM60-NEXT: membar.cta;
; SM60-NEXT: ld.param.b64 %rd2, [fadd_acq_rel_double_global_cta_param_1];
; SM60-NEXT: atom.cta.global.add.f64 %rd3, [%rd1], %rd2;
+; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b64 [func_retval0], %rd3;
; SM60-NEXT: ret;
%retval = atomicrmw fadd ptr addrspace(1) %addr, double %val syncscope("block") acq_rel
@@ -4832,6 +4884,7 @@ define i32 @add_acquire_i32_global_cta(ptr addrspace(1) %addr, i32 %val) {
; SM60-NEXT: ld.param.b64 %rd1, [add_acquire_i32_global_cta_param_0];
; SM60-NEXT: ld.param.b32 %r1, [add_acquire_i32_global_cta_param_1];
; SM60-NEXT: atom.cta.global.add.u32 %r2, [%rd1], %r1;
+; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b32 [func_retval0], %r2;
; SM60-NEXT: ret;
%retval = atomicrmw add ptr addrspace(1) %addr, i32 %val syncscope("block") acquire
@@ -4846,6 +4899,7 @@ define i32 @add_release_i32_global_cta(ptr addrspace(1) %addr, i32 %val) {
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd1, [add_release_i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
; SM60-NEXT: ld.param.b32 %r1, [add_release_i32_global_cta_param_1];
; SM60-NEXT: atom.cta.global.add.u32 %r2, [%rd1], %r1;
; SM60-NEXT: st.param.b32 [func_retval0], %r2;
@@ -4865,6 +4919,7 @@ define i32 @add_seq_cst_i32_global_cta(ptr addrspace(1) %addr, i32 %val) {
; SM60-NEXT: membar.cta;
; SM60-NEXT: ld.param.b32 %r1, [add_seq_cst_i32_global_cta_param_1];
; SM60-NEXT: atom.cta.global.add.u32 %r2, [%rd1], %r1;
+; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b32 [func_retval0], %r2;
; SM60-NEXT: ret;
%retval = atomicrmw add ptr addrspace(1) %addr, i32 %val syncscope("block") seq_cst
@@ -5143,8 +5198,10 @@ define i32 @add_acq_rel_i32_global_sys(ptr addrspace(1) %addr, i32 %val) {
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd1, [add_acq_rel_i32_global_sys_param_0];
+; SM60-NEXT: membar.sys;
; SM60-NEXT: ld.param.b32 %r1, [add_acq_rel_i32_global_sys_param_1];
; SM60-NEXT: atom.sys.global.add.u32 %r2, [%rd1], %r1;
+; SM60-NEXT: membar.sys;
; SM60-NEXT: st.param.b32 [func_retval0], %r2;
; SM60-NEXT: ret;
%retval = atomicrmw add ptr addrspace(1) %addr, i32 %val syncscope("") acq_rel
@@ -5158,8 +5215,10 @@ define i64 @add_acq_rel_i64_global_sys(ptr addrspace(1) %addr, i64 %val) {
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd1, [add_acq_rel_i64_global_sys_param_0];
+; SM60-NEXT: membar.sys;
; SM60-NEXT: ld.param.b64 %rd2, [add_acq_rel_i64_global_sys_param_1];
; SM60-NEXT: atom.sys.global.add.u64 %rd3, [%rd1], %rd2;
+; SM60-NEXT: membar.sys;
; SM60-NEXT: st.param.b64 [func_retval0], %rd3;
; SM60-NEXT: ret;
%retval = atomicrmw add ptr addrspace(1) %addr, i64 %val syncscope("") acq_rel
@@ -5174,8 +5233,10 @@ define i32 @min_acq_rel_i32_global_sys(ptr addrspace(1) %addr, i32 %val) {
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd1, [min_acq_rel_i32_global_sys_param_0];
+; SM60-NEXT: membar.sys;
; SM60-NEXT: ld.param.b32 %r1, [min_acq_rel_i32_global_sys_param_1];
; SM60-NEXT: atom.sys.global.min.s32 %r2, [%rd1], %r1;
+; SM60-NEXT: membar.sys;
; SM60-NEXT: st.param.b32 [func_retval0], %r2;
; SM60-NEXT: ret;
%retval = atomicrmw min ptr addrspace(1) %addr, i32 %val syncscope("") acq_rel
@@ -5190,8 +5251,10 @@ define i32 @umax_acq_rel_i32_global_sys(ptr addrspace(1) %addr, i32 %val) {
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd1, [umax_acq_rel_i32_global_sys_param_0];
+; SM60-NEXT: membar.sys;
; SM60-NEXT: ld.param.b32 %r1, [umax_acq_rel_i32_global_sys_param_1];
; SM60-NEXT: atom.sys.global.max.u32 %r2, [%rd1], %r1;
+; SM60-NEXT: membar.sys;
; SM60-NEXT: st.param.b32 [func_retval0], %r2;
; SM60-NEXT: ret;
%retval = atomicrmw umax ptr addrspace(1) %addr, i32 %val syncscope("") acq_rel
@@ -5284,8 +5347,10 @@ define float @fadd_acq_rel_float_global_sys(ptr addrspace(1) %addr, float %val)
; SM60-NOFTZ-ALLOW-EMPTY:
; SM60-NOFTZ-ALLOW-NEXT: // %bb.0:
; SM60-NOFTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_sys_param_0];
+; SM60-NOFTZ-ALLOW-NEXT: membar.sys;
; SM60-NOFTZ-ALLOW-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_global_sys_param_1];
; SM60-NOFTZ-ALLOW-NEXT: atom.sys.global.add.f32 %r2, [%rd1], %r1;
+; SM60-NOFTZ-ALLOW-NEXT: membar.sys;
; SM60-NOFTZ-ALLOW-NEXT: st.param.b32 [func_retval0], %r2;
; SM60-NOFTZ-ALLOW-NEXT: ret;
;
@@ -5296,8 +5361,10 @@ define float @fadd_acq_rel_float_global_sys(ptr addrspace(1) %addr, float %val)
; SM60-FTZ-DISALLOW-EMPTY:
; SM60-FTZ-DISALLOW-NEXT: // %bb.0:
; SM60-FTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_sys_param_0];
+; SM60-FTZ-DISALLOW-NEXT: membar.sys;
; SM60-FTZ-DISALLOW-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_global_sys_param_1];
; SM60-FTZ-DISALLOW-NEXT: atom.sys.global.add.f32 %r2, [%rd1], %r1;
+; SM60-FTZ-DISALLOW-NEXT: membar.sys;
; SM60-FTZ-DISALLOW-NEXT: st.param.b32 [func_retval0], %r2;
; SM60-FTZ-DISALLOW-NEXT: ret;
;
@@ -5308,8 +5375,10 @@ define float @fadd_acq_rel_float_global_sys(ptr addrspace(1) %addr, float %val)
; SM60-FTZ-ALLOW-EMPTY:
; SM60-FTZ-ALLOW-NEXT: // %bb.0:
; SM60-FTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_sys_param_0];
+; SM60-FTZ-ALLOW-NEXT: membar.sys;
; SM60-FTZ-ALLOW-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_global_sys_param_1];
; SM60-FTZ-ALLOW-NEXT: atom.sys.global.add.f32 %r2, [%rd1], %r1;
+; SM60-FTZ-ALLOW-NEXT: membar.sys;
; SM60-FTZ-ALLOW-NEXT: st.param.b32 [func_retval0], %r2;
; SM60-FTZ-ALLOW-NEXT: ret;
%retval = atomicrmw fadd ptr addrspace(1) %addr, float %val syncscope("") acq_rel
@@ -5323,8 +5392,10 @@ define double @fadd_acq_rel_double_global_sys(ptr addrspace(1) %addr, double %va
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_double_global_sys_param_0];
+; SM60-NEXT: membar.sys;
; SM60-NEXT: ld.param.b64 %rd2, [fadd_acq_rel_double_global_sys_param_1];
; SM60-NEXT: atom.sys.global.add.f64 %rd3, [%rd1], %rd2;
+; SM60-NEXT: membar.sys;
; SM60-NEXT: st.param.b64 [func_retval0], %rd3;
; SM60-NEXT: ret;
%retval = atomicrmw fadd ptr addrspace(1) %addr, double %val syncscope("") acq_rel
@@ -5339,8 +5410,10 @@ define i32 @add_acq_rel_i32_global_cluster(ptr addrspace(1) %addr, i32 %val) {
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd1, [add_acq_rel_i32_global_cluster_param_0];
+; SM60-NEXT: membar.cta;
; SM60-NEXT: ld.param.b32 %r1, [add_acq_rel_i32_global_cluster_param_1];
; SM60-NEXT: atom.cta.global.add.u32 %r2, [%rd1], %r1;
+; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b32 [func_retval0], %r2;
; SM60-NEXT: ret;
%retval = atomicrmw add ptr addrspace(1) %addr, i32 %val syncscope("cluster") acq_rel
@@ -5354,8 +5427,10 @@ define i64 @add_acq_rel_i64_global_cluster(ptr addrspace(1) %addr, i64 %val) {
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd1, [add_acq_rel_i64_global_cluster_param_0];
+; SM60-NEXT: membar.cta;
; SM60-NEXT: ld.param.b64 %rd2, [add_acq_rel_i64_global_cluster_param_1];
; SM60-NEXT: atom.cta.global.add.u64 %rd3, [%rd1], %rd2;
+; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b64 [func_retval0], %rd3;
; SM60-NEXT: ret;
%retval = atomicrmw add ptr addrspace(1) %addr, i64 %val syncscope("cluster") acq_rel
@@ -5370,8 +5445,10 @@ define i32 @min_acq_rel_i32_global_cluster(ptr addrspace(1) %addr, i32 %val) {
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd1, [min_acq_rel_i32_global_cluster_param_0];
+; SM60-NEXT: membar.cta;
; SM60-NEXT: ld.param.b32 %r1, [min_acq_rel_i32_global_cluster_param_1];
; SM60-NEXT: atom.cta.global.min.s32 %r2, [%rd1], %r1;
+; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b32 [func_retval0], %r2;
; SM60-NEXT: ret;
%retval = atomicrmw min ptr addrspace(1) %addr, i32 %val syncscope("cluster") acq_rel
@@ -5386,8 +5463,10 @@ define i32 @umax_acq_rel_i32_global_cluster(ptr addrspace(1) %addr, i32 %val) {
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd1, [umax_acq_rel_i32_global_cluster_param_0];
+; SM60-NEXT: membar.cta;
; SM60-NEXT: ld.param.b32 %r1, [umax_acq_rel_i32_global_cluster_param_1];
; SM60-NEXT: atom.cta.global.max.u32 %r2, [%rd1], %r1;
+; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b32 [func_retval0], %r2;
; SM60-NEXT: ret;
%retval = atomicrmw umax ptr addrspace(1) %addr, i32 %val syncscope("cluster") acq_rel
@@ -5480,8 +5559,10 @@ define float @fadd_acq_rel_float_global_cluster(ptr addrspace(1) %addr, float %v
; SM60-NOFTZ-ALLOW-EMPTY:
; SM60-NOFTZ-ALLOW-NEXT: // %bb.0:
; SM60-NOFTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_cluster_param_0];
+; SM60-NOFTZ-ALLOW-NEXT: membar.cta;
; SM60-NOFTZ-ALLOW-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_global_cluster_param_1];
; SM60-NOFTZ-ALLOW-NEXT: atom.cta.global.add.f32 %r2, [%rd1], %r1;
+; SM60-NOFTZ-ALLOW-NEXT: membar.cta;
; SM60-NOFTZ-ALLOW-NEXT: st.param.b32 [func_retval0], %r2;
; SM60-NOFTZ-ALLOW-NEXT: ret;
;
@@ -5492,8 +5573,10 @@ define float @fadd_acq_rel_float_global_cluster(ptr addrspace(1) %addr, float %v
; SM60-FTZ-DISALLOW-EMPTY:
; SM60-FTZ-DISALLOW-NEXT: // %bb.0:
; SM60-FTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_cluster_param_0];
+; SM60-FTZ-DISALLOW-NEXT: membar.cta;
; SM60-FTZ-DISALLOW-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_global_cluster_param_1];
; SM60-FTZ-DISALLOW-NEXT: atom.cta.global.add.f32 %r2, [%rd1], %r1;
+; SM60-FTZ-DISALLOW-NEXT: membar.cta;
; SM60-FTZ-DISALLOW-NEXT: st.param.b32 [func_retval0], %r2;
; SM60-FTZ-DISALLOW-NEXT: ret;
;
@@ -5504,8 +5587,10 @@ define float @fadd_acq_rel_float_global_cluster(ptr addrspace(1) %addr, float %v
; SM60-FTZ-ALLOW-EMPTY:
; SM60-FTZ-ALLOW-NEXT: // %bb.0:
; SM60-FTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_cluster_param_0];
+; SM60-FTZ-ALLOW-NEXT: membar.cta;
; SM60-FTZ-ALLOW-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_global_cluster_param_1];
; SM60-FTZ-ALLOW-NEXT: atom.cta.global.add.f32 %r2, [%rd1], %r1;
+; SM60-FTZ-ALLOW-NEXT: membar.cta;
; SM60-FTZ-ALLOW-NEXT: st.param.b32 [func_retval0], %r2;
; SM60-FTZ-ALLOW-NEXT: ret;
%retval = atomicrmw fadd ptr addrspace(1) %addr, float %val syncscope("cluster") acq_rel
@@ -5519,8 +5604,10 @@ define double @fadd_acq_rel_double_global_cluster(ptr addrspace(1) %addr, double
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_double_global_cluster_param_0];
+; SM60-NEXT: membar.cta;
; SM60-NEXT: ld.param.b64 %rd2, [fadd_acq_rel_double_global_cluster_param_1];
; SM60-NEXT: atom.cta.global.add.f64 %rd3, [%rd1], %rd2;
+; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b64 [func_retval0], %rd3;
; SM60-NEXT: ret;
%retval = atomicrmw fadd ptr addrspace(1) %addr, double %val syncscope("cluster") acq_rel
@@ -5535,8 +5622,10 @@ define i32 @add_acq_rel_i32_global_gpu(ptr addrspace(1) %addr, i32 %val) {
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd1, [add_acq_rel_i32_global_gpu_param_0];
+; SM60-NEXT: membar.gl;
; SM60-NEXT: ld.param.b32 %r1, [add_acq_rel_i32_global_gpu_param_1];
; SM60-NEXT: atom.gpu.global.add.u32 %r2, [%rd1], %r1;
+; SM60-NEXT: membar.gl;
; SM60-NEXT: st.param.b32 [func_retval0], %r2;
; SM60-NEXT: ret;
%retval = atomicrmw add ptr addrspace(1) %addr, i32 %val syncscope("device") acq_rel
@@ -5550,8 +5639,10 @@ define i64 @add_acq_rel_i64_global_gpu(ptr addrspace(1) %addr, i64 %val) {
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd1, [add_acq_rel_i64_global_gpu_param_0];
+; SM60-NEXT: membar.gl;
; SM60-NEXT: ld.param.b64 %rd2, [add_acq_rel_i64_global_gpu_param_1];
; SM60-NEXT: atom.gpu.global.add.u64 %rd3, [%rd1], %rd2;
+; SM60-NEXT: membar.gl;
; SM60-NEXT: st.param.b64 [func_retval0], %rd3;
; SM60-NEXT: ret;
%retval = atomicrmw add ptr addrspace(1) %addr, i64 %val syncscope("device") acq_rel
@@ -5566,8 +5657,10 @@ define i32 @min_acq_rel_i32_global_gpu(ptr addrspace(1) %addr, i32 %val) {
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd1, [min_acq_rel_i32_global_gpu_param_0];
+; SM60-NEXT: membar.gl;
; SM60-NEXT: ld.param.b32 %r1, [min_acq_rel_i32_global_gpu_param_1];
; SM60-NEXT: atom.gpu.global.min.s32 %r2, [%rd1], %r1;
+; SM60-NEXT: membar.gl;
; SM60-NEXT: st.param.b32 [func_retval0], %r2;
; SM60-NEXT: ret;
%retval = atomicrmw min ptr addrspace(1) %addr, i32 %val syncscope("device") acq_rel
@@ -5582,8 +5675,10 @@ define i32 @umax_acq_rel_i32_global_gpu(ptr addrspace(1) %addr, i32 %val) {
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd1, [umax_acq_rel_i32_global_gpu_param_0];
+; SM60-NEXT: membar.gl;
; SM60-NEXT: ld.param.b32 %r1, [umax_acq_rel_i32_global_gpu_param_1];
; SM60-NEXT: atom.gpu.global.max.u32 %r2, [%rd1], %r1;
+; SM60-NEXT: membar.gl;
; SM60-NEXT: st.param.b32 [func_retval0], %r2;
; SM60-NEXT: ret;
%retval = atomicrmw umax ptr addrspace(1) %addr, i32 %val syncscope("device") acq_rel
@@ -5676,8 +5771,10 @@ define float @fadd_acq_rel_float_global_gpu(ptr addrspace(1) %addr, float %val)
; SM60-NOFTZ-ALLOW-EMPTY:
; SM60-NOFTZ-ALLOW-NEXT: // %bb.0:
; SM60-NOFTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_gpu_param_0];
+; SM60-NOFTZ-ALLOW-NEXT: membar.gl;
; SM60-NOFTZ-ALLOW-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_global_gpu_param_1];
; SM60-NOFTZ-ALLOW-NEXT: atom.gpu.global.add.f32 %r2, [%rd1], %r1;
+; SM60-NOFTZ-ALLOW-NEXT: membar.gl;
; SM60-NOFTZ-ALLOW-NEXT: st.param.b32 [func_retval0], %r2;
; SM60-NOFTZ-ALLOW-NEXT: ret;
;
@@ -5688,8 +5785,10 @@ define float @fadd_acq_rel_float_global_gpu(ptr addrspace(1) %addr, float %val)
; SM60-FTZ-DISALLOW-EMPTY:
; SM60-FTZ-DISALLOW-NEXT: // %bb.0:
; SM60-FTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_gpu_param_0];
+; SM60-FTZ-DISALLOW-NEXT: membar.gl;
; SM60-FTZ-DISALLOW-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_global_gpu_param_1];
; SM60-FTZ-DISALLOW-NEXT: atom.gpu.global.add.f32 %r2, [%rd1], %r1;
+; SM60-FTZ-DISALLOW-NEXT: membar.gl;
; SM60-FTZ-DISALLOW-NEXT: st.param.b32 [func_retval0], %r2;
; SM60-FTZ-DISALLOW-NEXT: ret;
;
@@ -5700,8 +5799,10 @@ define float @fadd_acq_rel_float_global_gpu(ptr addrspace(1) %addr, float %val)
; SM60-FTZ-ALLOW-EMPTY:
; SM60-FTZ-ALLOW-NEXT: // %bb.0:
; SM60-FTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_gpu_param_0];
+; SM60-FTZ-ALLOW-NEXT: membar.gl;
; SM60-FTZ-ALLOW-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_global_gpu_param_1];
; SM60-FTZ-ALLOW-NEXT: atom.gpu.global.add.f32 %r2, [%rd1], %r1;
+; SM60-FTZ-ALLOW-NEXT: membar.gl;
; SM60-FTZ-ALLOW-NEXT: st.param.b32 [func_retval0], %r2;
; SM60-FTZ-ALLOW-NEXT: ret;
%retval = atomicrmw fadd ptr addrspace(1) %addr, float %val syncscope("device") acq_rel
@@ -5715,8 +5816,10 @@ define double @fadd_acq_rel_double_global_gpu(ptr addrspace(1) %addr, double %va
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_double_global_gpu_param_0];
+; SM60-NEXT: membar.gl;
; SM60-NEXT: ld.param.b64 %rd2, [fadd_acq_rel_double_global_gpu_param_1];
; SM60-NEXT: atom.gpu.global.add.f64 %rd3, [%rd1], %rd2;
+; SM60-NEXT: membar.gl;
; SM60-NEXT: st.param.b64 [func_retval0], %rd3;
; SM60-NEXT: ret;
%retval = atomicrmw fadd ptr addrspace(1) %addr, double %val syncscope("device") acq_rel
@@ -5731,8 +5834,10 @@ define i32 @add_acq_rel_i32_generic_cta(ptr %addr, i32 %val) {
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd1, [add_acq_rel_i32_generic_cta_param_0];
+; SM60-NEXT: membar.cta;
; SM60-NEXT: ld.param.b32 %r1, [add_acq_rel_i32_generic_cta_param_1];
; SM60-NEXT: atom.cta.add.u32 %r2, [%rd1], %r1;
+; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b32 [func_retval0], %r2;
; SM60-NEXT: ret;
%retval = atomicrmw add ptr %addr, i32 %val syncscope("block") acq_rel
@@ -5746,8 +5851,10 @@ define i64 @add_acq_rel_i64_generic_cta(ptr %addr, i64 %val) {
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd1, [add_acq_rel_i64_generic_cta_param_0];
+; SM60-NEXT: membar.cta;
; SM60-NEXT: ld.param.b64 %rd2, [add_acq_rel_i64_generic_cta_param_1];
; SM60-NEXT: atom.cta.add.u64 %rd3, [%rd1], %rd2;
+; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b64 [func_retval0], %rd3;
; SM60-NEXT: ret;
%retval = atomicrmw add ptr %addr, i64 %val syncscope("block") acq_rel
@@ -5762,8 +5869,10 @@ define i32 @min_acq_rel_i32_generic_cta(ptr %addr, i32 %val) {
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd1, [min_acq_rel_i32_generic_cta_param_0];
+; SM60-NEXT: membar.cta;
; SM60-NEXT: ld.param.b32 %r1, [min_acq_rel_i32_generic_cta_param_1];
; SM60-NEXT: atom.cta.min.s32 %r2, [%rd1], %r1;
+; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b32 [func_retval0], %r2;
; SM60-NEXT: ret;
%retval = atomicrmw min ptr %addr, i32 %val syncscope("block") acq_rel
@@ -5778,8 +5887,10 @@ define i32 @umax_acq_rel_i32_generic_cta(ptr %addr, i32 %val) {
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd1, [umax_acq_rel_i32_generic_cta_param_0];
+; SM60-NEXT: membar.cta;
; SM60-NEXT: ld.param.b32 %r1, [umax_acq_rel_i32_generic_cta_param_1];
; SM60-NEXT: atom.cta.max.u32 %r2, [%rd1], %r1;
+; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b32 [func_retval0], %r2;
; SM60-NEXT: ret;
%retval = atomicrmw umax ptr %addr, i32 %val syncscope("block") acq_rel
@@ -5872,8 +5983,10 @@ define float @fadd_acq_rel_float_generic_cta(ptr %addr, float %val) {
; SM60-NOFTZ-ALLOW-EMPTY:
; SM60-NOFTZ-ALLOW-NEXT: // %bb.0:
; SM60-NOFTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_generic_cta_param_0];
+; SM60-NOFTZ-ALLOW-NEXT: membar.cta;
; SM60-NOFTZ-ALLOW-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_generic_cta_param_1];
; SM60-NOFTZ-ALLOW-NEXT: atom.cta.add.f32 %r2, [%rd1], %r1;
+; SM60-NOFTZ-ALLOW-NEXT: membar.cta;
; SM60-NOFTZ-ALLOW-NEXT: st.param.b32 [func_retval0], %r2;
; SM60-NOFTZ-ALLOW-NEXT: ret;
;
@@ -5907,8 +6020,10 @@ define float @fadd_acq_rel_float_generic_cta(ptr %addr, float %val) {
; SM60-FTZ-ALLOW-EMPTY:
; SM60-FTZ-ALLOW-NEXT: // %bb.0:
; SM60-FTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_generic_cta_param_0];
+; SM60-FTZ-ALLOW-NEXT: membar.cta;
; SM60-FTZ-ALLOW-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_generic_cta_param_1];
; SM60-FTZ-ALLOW-NEXT: atom.cta.add.f32 %r2, [%rd1], %r1;
+; SM60-FTZ-ALLOW-NEXT: membar.cta;
; SM60-FTZ-ALLOW-NEXT: st.param.b32 [func_retval0], %r2;
; SM60-FTZ-ALLOW-NEXT: ret;
%retval = atomicrmw fadd ptr %addr, float %val syncscope("block") acq_rel
@@ -5922,8 +6037,10 @@ define double @fadd_acq_rel_double_generic_cta(ptr %addr, double %val) {
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_double_generic_cta_param_0];
+; SM60-NEXT: membar.cta;
; SM60-NEXT: ld.param.b64 %rd2, [fadd_acq_rel_double_generic_cta_param_1];
; SM60-NEXT: atom.cta.add.f64 %rd3, [%rd1], %rd2;
+; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b64 [func_retval0], %rd3;
; SM60-NEXT: ret;
%retval = atomicrmw fadd ptr %addr, double %val syncscope("block") acq_rel
@@ -5938,8 +6055,10 @@ define i32 @add_acq_rel_i32_shared_cta(ptr addrspace(3) %addr, i32 %val) {
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd1, [add_acq_rel_i32_shared_cta_param_0];
+; SM60-NEXT: membar.cta;
; SM60-NEXT: ld.param.b32 %r1, [add_acq_rel_i32_shared_cta_param_1];
; SM60-NEXT: atom.cta.shared.add.u32 %r2, [%rd1], %r1;
+; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b32 [func_retval0], %r2;
; SM60-NEXT: ret;
%retval = atomicrmw add ptr addrspace(3) %addr, i32 %val syncscope("block") acq_rel
@@ -5953,8 +6072,10 @@ define i64 @add_acq_rel_i64_shared_cta(ptr addrspace(3) %addr, i64 %val) {
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd1, [add_acq_rel_i64_shared_cta_param_0];
+; SM60-NEXT: membar.cta;
; SM60-NEXT: ld.param.b64 %rd2, [add_acq_rel_i64_shared_cta_param_1];
; SM60-NEXT: atom.cta.shared.add.u64 %rd3, [%rd1], %rd2;
+; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b64 [func_retval0], %rd3;
; SM60-NEXT: ret;
%retval = atomicrmw add ptr addrspace(3) %addr, i64 %val syncscope("block") acq_rel
@@ -5969,8 +6090,10 @@ define i32 @min_acq_rel_i32_shared_cta(ptr addrspace(3) %addr, i32 %val) {
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd1, [min_acq_rel_i32_shared_cta_param_0];
+; SM60-NEXT: membar.cta;
; SM60-NEXT: ld.param.b32 %r1, [min_acq_rel_i32_shared_cta_param_1];
; SM60-NEXT: atom.cta.shared.min.s32 %r2, [%rd1], %r1;
+; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b32 [func_retval0], %r2;
; SM60-NEXT: ret;
%retval = atomicrmw min ptr addrspace(3) %addr, i32 %val syncscope("block") acq_rel
@@ -5985,8 +6108,10 @@ define i32 @umax_acq_rel_i32_shared_cta(ptr addrspace(3) %addr, i32 %val) {
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd1, [umax_acq_rel_i32_shared_cta_param_0];
+; SM60-NEXT: membar.cta;
; SM60-NEXT: ld.param.b32 %r1, [umax_acq_rel_i32_shared_cta_param_1];
; SM60-NEXT: atom.cta.shared.max.u32 %r2, [%rd1], %r1;
+; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b32 [func_retval0], %r2;
; SM60-NEXT: ret;
%retval = atomicrmw umax ptr addrspace(3) %addr, i32 %val syncscope("block") acq_rel
@@ -6056,8 +6181,10 @@ define float @fadd_acq_rel_float_shared_cta(ptr addrspace(3) %addr, float %val)
; SM60-NOFTZ-DISALLOW-EMPTY:
; SM60-NOFTZ-DISALLOW-NEXT: // %bb.0:
; SM60-NOFTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_shared_cta_param_0];
+; SM60-NOFTZ-DISALLOW-NEXT: membar.cta;
; SM60-NOFTZ-DISALLOW-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_shared_cta_param_1];
; SM60-NOFTZ-DISALLOW-NEXT: atom.cta.shared.add.f32 %r2, [%rd1], %r1;
+; SM60-NOFTZ-DISALLOW-NEXT: membar.cta;
; SM60-NOFTZ-DISALLOW-NEXT: st.param.b32 [func_retval0], %r2;
; SM60-NOFTZ-DISALLOW-NEXT: ret;
;
@@ -6068,8 +6195,10 @@ define float @fadd_acq_rel_float_shared_cta(ptr addrspace(3) %addr, float %val)
; SM60-NOFTZ-ALLOW-EMPTY:
; SM60-NOFTZ-ALLOW-NEXT: // %bb.0:
; SM60-NOFTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_shared_cta_param_0];
+; SM60-NOFTZ-ALLOW-NEXT: membar.cta;
; SM60-NOFTZ-ALLOW-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_shared_cta_param_1];
; SM60-NOFTZ-ALLOW-NEXT: atom.cta.shared.add.f32 %r2, [%rd1], %r1;
+; SM60-NOFTZ-ALLOW-NEXT: membar.cta;
; SM60-NOFTZ-ALLOW-NEXT: st.param.b32 [func_retval0], %r2;
; SM60-NOFTZ-ALLOW-NEXT: ret;
;
@@ -6103,8 +6232,10 @@ define float @fadd_acq_rel_float_shared_cta(ptr addrspace(3) %addr, float %val)
; SM60-FTZ-ALLOW-EMPTY:
; SM60-FTZ-ALLOW-NEXT: // %bb.0:
; SM60-FTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_shared_cta_param_0];
+; SM60-FTZ-ALLOW-NEXT: membar.cta;
; SM60-FTZ-ALLOW-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_shared_cta_param_1];
; SM60-FTZ-ALLOW-NEXT: atom.cta.shared.add.f32 %r2, [%rd1], %r1;
+; SM60-FTZ-ALLOW-NEXT: membar.cta;
; SM60-FTZ-ALLOW-NEXT: st.param.b32 [func_retval0], %r2;
; SM60-FTZ-ALLOW-NEXT: ret;
%retval = atomicrmw fadd ptr addrspace(3) %addr, float %val syncscope("block") acq_rel
@@ -6118,8 +6249,10 @@ define double @fadd_acq_rel_double_shared_cta(ptr addrspace(3) %addr, double %va
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_double_shared_cta_param_0];
+; SM60-NEXT: membar.cta;
; SM60-NEXT: ld.param.b64 %rd2, [fadd_acq_rel_double_shared_cta_param_1];
; SM60-NEXT: atom.cta.shared.add.f64 %rd3, [%rd1], %rd2;
+; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b64 [func_retval0], %rd3;
; SM60-NEXT: ret;
%retval = atomicrmw fadd ptr addrspace(3) %addr, double %val syncscope("block") acq_rel
diff --git a/llvm/test/CodeGen/NVPTX/cmpxchg-sm60.ll b/llvm/test/CodeGen/NVPTX/cmpxchg-sm60.ll
index b3ce18f1d23af..379e86e0d1a41 100644
--- a/llvm/test/CodeGen/NVPTX/cmpxchg-sm60.ll
+++ b/llvm/test/CodeGen/NVPTX/cmpxchg-sm60.ll
@@ -1399,6 +1399,7 @@ define i32 @monotonic_acquire_i32_global_cta(ptr addrspace(1) %addr, i32 %cmp, i
; SM60-NEXT: ld.param.b32 %r1, [monotonic_acquire_i32_global_cta_param_1];
; SM60-NEXT: ld.param.b32 %r2, [monotonic_acquire_i32_global_cta_param_2];
; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r1, %r2;
+; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b32 [func_retval0], %r2;
; SM60-NEXT: ret;
%pairold = cmpxchg ptr addrspace(1) %addr, i32 %cmp, i32 %new syncscope("block") monotonic acquire
@@ -1417,6 +1418,7 @@ define i32 @monotonic_seq_cst_i32_global_cta(ptr addrspace(1) %addr, i32 %cmp, i
; SM60-NEXT: ld.param.b32 %r1, [monotonic_seq_cst_i32_global_cta_param_1];
; SM60-NEXT: ld.param.b32 %r2, [monotonic_seq_cst_i32_global_cta_param_2];
; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r1, %r2;
+; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b32 [func_retval0], %r2;
; SM60-NEXT: ret;
%pairold = cmpxchg ptr addrspace(1) %addr, i32 %cmp, i32 %new syncscope("block") monotonic seq_cst
@@ -1434,6 +1436,7 @@ define i32 @acquire_monotonic_i32_global_cta(ptr addrspace(1) %addr, i32 %cmp, i
; SM60-NEXT: ld.param.b32 %r1, [acquire_monotonic_i32_global_cta_param_1];
; SM60-NEXT: ld.param.b32 %r2, [acquire_monotonic_i32_global_cta_param_2];
; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r1, %r2;
+; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b32 [func_retval0], %r2;
; SM60-NEXT: ret;
%pairold = cmpxchg ptr addrspace(1) %addr, i32 %cmp, i32 %new syncscope("block") acquire monotonic
@@ -1451,6 +1454,7 @@ define i32 @acquire_acquire_i32_global_cta(ptr addrspace(1) %addr, i32 %cmp, i32
; SM60-NEXT: ld.param.b32 %r1, [acquire_acquire_i32_global_cta_param_1];
; SM60-NEXT: ld.param.b32 %r2, [acquire_acquire_i32_global_cta_param_2];
; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r1, %r2;
+; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b32 [func_retval0], %r2;
; SM60-NEXT: ret;
%pairold = cmpxchg ptr addrspace(1) %addr, i32 %cmp, i32 %new syncscope("block") acquire acquire
@@ -1469,6 +1473,7 @@ define i32 @acquire_seq_cst_i32_global_cta(ptr addrspace(1) %addr, i32 %cmp, i32
; SM60-NEXT: ld.param.b32 %r1, [acquire_seq_cst_i32_global_cta_param_1];
; SM60-NEXT: ld.param.b32 %r2, [acquire_seq_cst_i32_global_cta_param_2];
; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r1, %r2;
+; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b32 [func_retval0], %r2;
; SM60-NEXT: ret;
%pairold = cmpxchg ptr addrspace(1) %addr, i32 %cmp, i32 %new syncscope("block") acquire seq_cst
@@ -1483,6 +1488,7 @@ define i32 @release_monotonic_i32_global_cta(ptr addrspace(1) %addr, i32 %cmp, i
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd1, [release_monotonic_i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
; SM60-NEXT: ld.param.b32 %r1, [release_monotonic_i32_global_cta_param_1];
; SM60-NEXT: ld.param.b32 %r2, [release_monotonic_i32_global_cta_param_2];
; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r1, %r2;
@@ -1500,9 +1506,11 @@ define i32 @release_acquire_i32_global_cta(ptr addrspace(1) %addr, i32 %cmp, i32
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd1, [release_acquire_i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
; SM60-NEXT: ld.param.b32 %r1, [release_acquire_i32_global_cta_param_1];
; SM60-NEXT: ld.param.b32 %r2, [release_acquire_i32_global_cta_param_2];
; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r1, %r2;
+; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b32 [func_retval0], %r2;
; SM60-NEXT: ret;
%pairold = cmpxchg ptr addrspace(1) %addr, i32 %cmp, i32 %new syncscope("block") release acquire
@@ -1521,6 +1529,7 @@ define i32 @release_seq_cst_i32_global_cta(ptr addrspace(1) %addr, i32 %cmp, i32
; SM60-NEXT: ld.param.b32 %r1, [release_seq_cst_i32_global_cta_param_1];
; SM60-NEXT: ld.param.b32 %r2, [release_seq_cst_i32_global_cta_param_2];
; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r1, %r2;
+; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b32 [func_retval0], %r2;
; SM60-NEXT: ret;
%pairold = cmpxchg ptr addrspace(1) %addr, i32 %cmp, i32 %new syncscope("block") release seq_cst
@@ -1535,9 +1544,11 @@ define i32 @acq_rel_monotonic_i32_global_cta(ptr addrspace(1) %addr, i32 %cmp, i
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd1, [acq_rel_monotonic_i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
; SM60-NEXT: ld.param.b32 %r1, [acq_rel_monotonic_i32_global_cta_param_1];
; SM60-NEXT: ld.param.b32 %r2, [acq_rel_monotonic_i32_global_cta_param_2];
; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r1, %r2;
+; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b32 [func_retval0], %r2;
; SM60-NEXT: ret;
%pairold = cmpxchg ptr addrspace(1) %addr, i32 %cmp, i32 %new syncscope("block") acq_rel monotonic
@@ -1552,9 +1563,11 @@ define i32 @acq_rel_acquire_i32_global_cta(ptr addrspace(1) %addr, i32 %cmp, i32
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd1, [acq_rel_acquire_i32_global_cta_param_0];
+; SM60-NEXT: membar.cta;
; SM60-NEXT: ld.param.b32 %r1, [acq_rel_acquire_i32_global_cta_param_1];
; SM60-NEXT: ld.param.b32 %r2, [acq_rel_acquire_i32_global_cta_param_2];
; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r1, %r2;
+; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b32 [func_retval0], %r2;
; SM60-NEXT: ret;
%pairold = cmpxchg ptr addrspace(1) %addr, i32 %cmp, i32 %new syncscope("block") acq_rel acquire
@@ -1573,6 +1586,7 @@ define i32 @acq_rel_seq_cst_i32_global_cta(ptr addrspace(1) %addr, i32 %cmp, i32
; SM60-NEXT: ld.param.b32 %r1, [acq_rel_seq_cst_i32_global_cta_param_1];
; SM60-NEXT: ld.param.b32 %r2, [acq_rel_seq_cst_i32_global_cta_param_2];
; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r1, %r2;
+; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b32 [func_retval0], %r2;
; SM60-NEXT: ret;
%pairold = cmpxchg ptr addrspace(1) %addr, i32 %cmp, i32 %new syncscope("block") acq_rel seq_cst
@@ -1591,6 +1605,7 @@ define i32 @seq_cst_monotonic_i32_global_cta(ptr addrspace(1) %addr, i32 %cmp, i
; SM60-NEXT: ld.param.b32 %r1, [seq_cst_monotonic_i32_global_cta_param_1];
; SM60-NEXT: ld.param.b32 %r2, [seq_cst_monotonic_i32_global_cta_param_2];
; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r1, %r2;
+; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b32 [func_retval0], %r2;
; SM60-NEXT: ret;
%pairold = cmpxchg ptr addrspace(1) %addr, i32 %cmp, i32 %new syncscope("block") seq_cst monotonic
@@ -1609,6 +1624,7 @@ define i32 @seq_cst_acquire_i32_global_cta(ptr addrspace(1) %addr, i32 %cmp, i32
; SM60-NEXT: ld.param.b32 %r1, [seq_cst_acquire_i32_global_cta_param_1];
; SM60-NEXT: ld.param.b32 %r2, [seq_cst_acquire_i32_global_cta_param_2];
; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r1, %r2;
+; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b32 [func_retval0], %r2;
; SM60-NEXT: ret;
%pairold = cmpxchg ptr addrspace(1) %addr, i32 %cmp, i32 %new syncscope("block") seq_cst acquire
@@ -1627,6 +1643,7 @@ define i32 @seq_cst_seq_cst_i32_global_cta(ptr addrspace(1) %addr, i32 %cmp, i32
; SM60-NEXT: ld.param.b32 %r1, [seq_cst_seq_cst_i32_global_cta_param_1];
; SM60-NEXT: ld.param.b32 %r2, [seq_cst_seq_cst_i32_global_cta_param_2];
; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r1, %r2;
+; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b32 [func_retval0], %r2;
; SM60-NEXT: ret;
%pairold = cmpxchg ptr addrspace(1) %addr, i32 %cmp, i32 %new syncscope("block") seq_cst seq_cst
@@ -1659,6 +1676,7 @@ define i64 @monotonic_acquire_i64_global_cta(ptr addrspace(1) %addr, i64 %cmp, i
; SM60-NEXT: ld.param.b64 %rd2, [monotonic_acquire_i64_global_cta_param_1];
; SM60-NEXT: ld.param.b64 %rd3, [monotonic_acquire_i64_global_cta_param_2];
; SM60-NEXT: atom.cta.global.cas.b64 %rd4, [%rd1], %rd2, %rd3;
+; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b64 [func_retval0], %rd3;
; SM60-NEXT: ret;
%pairold = cmpxchg ptr addrspace(1) %addr, i64 %cmp, i64 %new syncscope("block") monotonic acquire
@@ -1676,6 +1694,7 @@ define i64 @monotonic_seq_cst_i64_global_cta(ptr addrspace(1) %addr, i64 %cmp, i
; SM60-NEXT: ld.param.b64 %rd2, [monotonic_seq_cst_i64_global_cta_param_1];
; SM60-NEXT: ld.param.b64 %rd3, [monotonic_seq_cst_i64_global_cta_param_2];
; SM60-NEXT: atom.cta.global.cas.b64 %rd4, [%rd1], %rd2, %rd3;
+; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b64 [func_retval0], %rd3;
; SM60-NEXT: ret;
%pairold = cmpxchg ptr addrspace(1) %addr, i64 %cmp, i64 %new syncscope("block") monotonic seq_cst
@@ -1692,6 +1711,7 @@ define i64 @acquire_monotonic_i64_global_cta(ptr addrspace(1) %addr, i64 %cmp, i
; SM60-NEXT: ld.param.b64 %rd2, [acquire_monotonic_i64_global_cta_param_1];
; SM60-NEXT: ld.param.b64 %rd3, [acquire_monotonic_i64_global_cta_param_2];
; SM60-NEXT: atom.cta.global.cas.b64 %rd4, [%rd1], %rd2, %rd3;
+; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b64 [func_retval0], %rd3;
; SM60-NEXT: ret;
%pairold = cmpxchg ptr addrspace(1) %addr, i64 %cmp, i64 %new syncscope("block") acquire monotonic
@@ -1708,6 +1728,7 @@ define i64 @acquire_acquire_i64_global_cta(ptr addrspace(1) %addr, i64 %cmp, i64
; SM60-NEXT: ld.param.b64 %rd2, [acquire_acquire_i64_global_cta_param_1];
; SM60-NEXT: ld.param.b64 %rd3, [acquire_acquire_i64_global_cta_param_2];
; SM60-NEXT: atom.cta.global.cas.b64 %rd4, [%rd1], %rd2, %rd3;
+; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b64 [func_retval0], %rd3;
; SM60-NEXT: ret;
%pairold = cmpxchg ptr addrspace(1) %addr, i64 %cmp, i64 %new syncscope("block") acquire acquire
@@ -1725,6 +1746,7 @@ define i64 @acquire_seq_cst_i64_global_cta(ptr addrspace(1) %addr, i64 %cmp, i64
; SM60-NEXT: ld.param.b64 %rd2, [acquire_seq_cst_i64_global_cta_param_1];
; SM60-NEXT: ld.param.b64 %rd3, [acquire_seq_cst_i64_global_cta_param_2];
; SM60-NEXT: atom.cta.global.cas.b64 %rd4, [%rd1], %rd2, %rd3;
+; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b64 [func_retval0], %rd3;
; SM60-NEXT: ret;
%pairold = cmpxchg ptr addrspace(1) %addr, i64 %cmp, i64 %new syncscope("block") acquire seq_cst
@@ -1738,6 +1760,7 @@ define i64 @release_monotonic_i64_global_cta(ptr addrspace(1) %addr, i64 %cmp, i
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd1, [release_monotonic_i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
; SM60-NEXT: ld.param.b64 %rd2, [release_monotonic_i64_global_cta_param_1];
; SM60-NEXT: ld.param.b64 %rd3, [release_monotonic_i64_global_cta_param_2];
; SM60-NEXT: atom.cta.global.cas.b64 %rd4, [%rd1], %rd2, %rd3;
@@ -1754,9 +1777,11 @@ define i64 @release_acquire_i64_global_cta(ptr addrspace(1) %addr, i64 %cmp, i64
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd1, [release_acquire_i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
; SM60-NEXT: ld.param.b64 %rd2, [release_acquire_i64_global_cta_param_1];
; SM60-NEXT: ld.param.b64 %rd3, [release_acquire_i64_global_cta_param_2];
; SM60-NEXT: atom.cta.global.cas.b64 %rd4, [%rd1], %rd2, %rd3;
+; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b64 [func_retval0], %rd3;
; SM60-NEXT: ret;
%pairold = cmpxchg ptr addrspace(1) %addr, i64 %cmp, i64 %new syncscope("block") release acquire
@@ -1774,6 +1799,7 @@ define i64 @release_seq_cst_i64_global_cta(ptr addrspace(1) %addr, i64 %cmp, i64
; SM60-NEXT: ld.param.b64 %rd2, [release_seq_cst_i64_global_cta_param_1];
; SM60-NEXT: ld.param.b64 %rd3, [release_seq_cst_i64_global_cta_param_2];
; SM60-NEXT: atom.cta.global.cas.b64 %rd4, [%rd1], %rd2, %rd3;
+; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b64 [func_retval0], %rd3;
; SM60-NEXT: ret;
%pairold = cmpxchg ptr addrspace(1) %addr, i64 %cmp, i64 %new syncscope("block") release seq_cst
@@ -1787,9 +1813,11 @@ define i64 @acq_rel_monotonic_i64_global_cta(ptr addrspace(1) %addr, i64 %cmp, i
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd1, [acq_rel_monotonic_i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
; SM60-NEXT: ld.param.b64 %rd2, [acq_rel_monotonic_i64_global_cta_param_1];
; SM60-NEXT: ld.param.b64 %rd3, [acq_rel_monotonic_i64_global_cta_param_2];
; SM60-NEXT: atom.cta.global.cas.b64 %rd4, [%rd1], %rd2, %rd3;
+; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b64 [func_retval0], %rd3;
; SM60-NEXT: ret;
%pairold = cmpxchg ptr addrspace(1) %addr, i64 %cmp, i64 %new syncscope("block") acq_rel monotonic
@@ -1803,9 +1831,11 @@ define i64 @acq_rel_acquire_i64_global_cta(ptr addrspace(1) %addr, i64 %cmp, i64
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd1, [acq_rel_acquire_i64_global_cta_param_0];
+; SM60-NEXT: membar.cta;
; SM60-NEXT: ld.param.b64 %rd2, [acq_rel_acquire_i64_global_cta_param_1];
; SM60-NEXT: ld.param.b64 %rd3, [acq_rel_acquire_i64_global_cta_param_2];
; SM60-NEXT: atom.cta.global.cas.b64 %rd4, [%rd1], %rd2, %rd3;
+; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b64 [func_retval0], %rd3;
; SM60-NEXT: ret;
%pairold = cmpxchg ptr addrspace(1) %addr, i64 %cmp, i64 %new syncscope("block") acq_rel acquire
@@ -1823,6 +1853,7 @@ define i64 @acq_rel_seq_cst_i64_global_cta(ptr addrspace(1) %addr, i64 %cmp, i64
; SM60-NEXT: ld.param.b64 %rd2, [acq_rel_seq_cst_i64_global_cta_param_1];
; SM60-NEXT: ld.param.b64 %rd3, [acq_rel_seq_cst_i64_global_cta_param_2];
; SM60-NEXT: atom.cta.global.cas.b64 %rd4, [%rd1], %rd2, %rd3;
+; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b64 [func_retval0], %rd3;
; SM60-NEXT: ret;
%pairold = cmpxchg ptr addrspace(1) %addr, i64 %cmp, i64 %new syncscope("block") acq_rel seq_cst
@@ -1840,6 +1871,7 @@ define i64 @seq_cst_monotonic_i64_global_cta(ptr addrspace(1) %addr, i64 %cmp, i
; SM60-NEXT: ld.param.b64 %rd2, [seq_cst_monotonic_i64_global_cta_param_1];
; SM60-NEXT: ld.param.b64 %rd3, [seq_cst_monotonic_i64_global_cta_param_2];
; SM60-NEXT: atom.cta.global.cas.b64 %rd4, [%rd1], %rd2, %rd3;
+; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b64 [func_retval0], %rd3;
; SM60-NEXT: ret;
%pairold = cmpxchg ptr addrspace(1) %addr, i64 %cmp, i64 %new syncscope("block") seq_cst monotonic
@@ -1857,6 +1889,7 @@ define i64 @seq_cst_acquire_i64_global_cta(ptr addrspace(1) %addr, i64 %cmp, i64
; SM60-NEXT: ld.param.b64 %rd2, [seq_cst_acquire_i64_global_cta_param_1];
; SM60-NEXT: ld.param.b64 %rd3, [seq_cst_acquire_i64_global_cta_param_2];
; SM60-NEXT: atom.cta.global.cas.b64 %rd4, [%rd1], %rd2, %rd3;
+; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b64 [func_retval0], %rd3;
; SM60-NEXT: ret;
%pairold = cmpxchg ptr addrspace(1) %addr, i64 %cmp, i64 %new syncscope("block") seq_cst acquire
@@ -1874,6 +1907,7 @@ define i64 @seq_cst_seq_cst_i64_global_cta(ptr addrspace(1) %addr, i64 %cmp, i64
; SM60-NEXT: ld.param.b64 %rd2, [seq_cst_seq_cst_i64_global_cta_param_1];
; SM60-NEXT: ld.param.b64 %rd3, [seq_cst_seq_cst_i64_global_cta_param_2];
; SM60-NEXT: atom.cta.global.cas.b64 %rd4, [%rd1], %rd2, %rd3;
+; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b64 [func_retval0], %rd3;
; SM60-NEXT: ret;
%pairold = cmpxchg ptr addrspace(1) %addr, i64 %cmp, i64 %new syncscope("block") seq_cst seq_cst
@@ -1934,9 +1968,11 @@ define i32 @acq_rel_acquire_i32_global(ptr addrspace(1) %addr, i32 %cmp, i32 %ne
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd1, [acq_rel_acquire_i32_global_param_0];
+; SM60-NEXT: membar.sys;
; SM60-NEXT: ld.param.b32 %r1, [acq_rel_acquire_i32_global_param_1];
; SM60-NEXT: ld.param.b32 %r2, [acq_rel_acquire_i32_global_param_2];
; SM60-NEXT: atom.sys.global.cas.b32 %r3, [%rd1], %r1, %r2;
+; SM60-NEXT: membar.sys;
; SM60-NEXT: st.param.b32 [func_retval0], %r2;
; SM60-NEXT: ret;
%pairold = cmpxchg ptr addrspace(1) %addr, i32 %cmp, i32 %new acq_rel acquire
@@ -1951,9 +1987,11 @@ define i32 @acq_rel_acquire_i32_global_sys(ptr addrspace(1) %addr, i32 %cmp, i32
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd1, [acq_rel_acquire_i32_global_sys_param_0];
+; SM60-NEXT: membar.sys;
; SM60-NEXT: ld.param.b32 %r1, [acq_rel_acquire_i32_global_sys_param_1];
; SM60-NEXT: ld.param.b32 %r2, [acq_rel_acquire_i32_global_sys_param_2];
; SM60-NEXT: atom.sys.global.cas.b32 %r3, [%rd1], %r1, %r2;
+; SM60-NEXT: membar.sys;
; SM60-NEXT: st.param.b32 [func_retval0], %r2;
; SM60-NEXT: ret;
%pairold = cmpxchg ptr addrspace(1) %addr, i32 %cmp, i32 %new syncscope("") acq_rel acquire
@@ -1968,9 +2006,11 @@ define i32 @acq_rel_acquire_i32_global_gpu(ptr addrspace(1) %addr, i32 %cmp, i32
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd1, [acq_rel_acquire_i32_global_gpu_param_0];
+; SM60-NEXT: membar.gl;
; SM60-NEXT: ld.param.b32 %r1, [acq_rel_acquire_i32_global_gpu_param_1];
; SM60-NEXT: ld.param.b32 %r2, [acq_rel_acquire_i32_global_gpu_param_2];
; SM60-NEXT: atom.gpu.global.cas.b32 %r3, [%rd1], %r1, %r2;
+; SM60-NEXT: membar.gl;
; SM60-NEXT: st.param.b32 [func_retval0], %r2;
; SM60-NEXT: ret;
%pairold = cmpxchg ptr addrspace(1) %addr, i32 %cmp, i32 %new syncscope("device") acq_rel acquire
@@ -2077,9 +2117,11 @@ define i32 @acq_rel_acquire_i32_generic_cta(ptr %addr, i32 %cmp, i32 %new) {
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd1, [acq_rel_acquire_i32_generic_cta_param_0];
+; SM60-NEXT: membar.cta;
; SM60-NEXT: ld.param.b32 %r1, [acq_rel_acquire_i32_generic_cta_param_1];
; SM60-NEXT: ld.param.b32 %r2, [acq_rel_acquire_i32_generic_cta_param_2];
; SM60-NEXT: atom.cta.cas.b32 %r3, [%rd1], %r1, %r2;
+; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b32 [func_retval0], %r2;
; SM60-NEXT: ret;
%pairold = cmpxchg ptr %addr, i32 %cmp, i32 %new syncscope("block") acq_rel acquire
@@ -2094,9 +2136,11 @@ define i32 @acq_rel_acquire_i32_shared_cta(ptr addrspace(3) %addr, i32 %cmp, i32
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd1, [acq_rel_acquire_i32_shared_cta_param_0];
+; SM60-NEXT: membar.cta;
; SM60-NEXT: ld.param.b32 %r1, [acq_rel_acquire_i32_shared_cta_param_1];
; SM60-NEXT: ld.param.b32 %r2, [acq_rel_acquire_i32_shared_cta_param_2];
; SM60-NEXT: atom.cta.shared.cas.b32 %r3, [%rd1], %r1, %r2;
+; SM60-NEXT: membar.cta;
; SM60-NEXT: st.param.b32 [func_retval0], %r2;
; SM60-NEXT: ret;
%pairold = cmpxchg ptr addrspace(3) %addr, i32 %cmp, i32 %new syncscope("block") acq_rel acquire
diff --git a/llvm/test/CodeGen/NVPTX/cmpxchg.ll b/llvm/test/CodeGen/NVPTX/cmpxchg.ll
index d0b802a81b918..466bb62ed319e 100644
--- a/llvm/test/CodeGen/NVPTX/cmpxchg.ll
+++ b/llvm/test/CodeGen/NVPTX/cmpxchg.ll
@@ -4,7 +4,7 @@
; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_70 -mattr=+ptx63 | FileCheck %s --check-prefixes=SM70,CHECK
; RUN: %if ptxas-sm_70 && ptxas-isa-6.3 %{ llc < %s -mtriple=nvptx64 -mcpu=sm_70 -mattr=+ptx63 | %ptxas-verify -arch=sm_70 %}
-; TODO: these are seq_cst, but are compiled to relaxed..
+; TODO: these are system scope, but are compiled to gpu scope..
; CHECK-LABEL: relaxed_sys_i8
@@ -1344,9 +1344,11 @@ define i32 @acq_rel_sys_i32(ptr %addr, i32 %cmp, i32 %new) {
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd1, [acq_rel_sys_i32_param_0];
+; SM60-NEXT: membar.sys;
; SM60-NEXT: ld.param.b32 %r1, [acq_rel_sys_i32_param_1];
; SM60-NEXT: ld.param.b32 %r2, [acq_rel_sys_i32_param_2];
; SM60-NEXT: atom.sys.cas.b32 %r3, [%rd1], %r1, %r2;
+; SM60-NEXT: membar.sys;
; SM60-NEXT: st.param.b32 [func_retval0], %r2;
; SM60-NEXT: ret;
;
@@ -1390,6 +1392,7 @@ define i32 @acquire_sys_i32(ptr %addr, i32 %cmp, i32 %new) {
; SM60-NEXT: ld.param.b32 %r1, [acquire_sys_i32_param_1];
; SM60-NEXT: ld.param.b32 %r2, [acquire_sys_i32_param_2];
; SM60-NEXT: atom.sys.cas.b32 %r3, [%rd1], %r1, %r2;
+; SM60-NEXT: membar.sys;
; SM60-NEXT: st.param.b32 [func_retval0], %r2;
; SM60-NEXT: ret;
;
@@ -1430,6 +1433,7 @@ define i32 @release_sys_i32(ptr %addr, i32 %cmp, i32 %new) {
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd1, [release_sys_i32_param_0];
+; SM60-NEXT: membar.sys;
; SM60-NEXT: ld.param.b32 %r1, [release_sys_i32_param_1];
; SM60-NEXT: ld.param.b32 %r2, [release_sys_i32_param_2];
; SM60-NEXT: atom.sys.cas.b32 %r3, [%rd1], %r1, %r2;
@@ -1477,6 +1481,7 @@ define i32 @seq_cst_sys_i32(ptr %addr, i32 %cmp, i32 %new) {
; SM60-NEXT: ld.param.b32 %r1, [seq_cst_sys_i32_param_1];
; SM60-NEXT: ld.param.b32 %r2, [seq_cst_sys_i32_param_2];
; SM60-NEXT: atom.sys.cas.b32 %r3, [%rd1], %r1, %r2;
+; SM60-NEXT: membar.sys;
; SM60-NEXT: st.param.b32 [func_retval0], %r2;
; SM60-NEXT: ret;
;
@@ -1563,6 +1568,7 @@ define i64 @acquire_sys_i64(ptr %addr, i64 %cmp, i64 %new) {
; SM60-NEXT: ld.param.b64 %rd2, [acquire_sys_i64_param_1];
; SM60-NEXT: ld.param.b64 %rd3, [acquire_sys_i64_param_2];
; SM60-NEXT: atom.sys.cas.b64 %rd4, [%rd1], %rd2, %rd3;
+; SM60-NEXT: membar.sys;
; SM60-NEXT: st.param.b64 [func_retval0], %rd3;
; SM60-NEXT: ret;
;
@@ -1600,9 +1606,11 @@ define i64 @acq_rel_sys_i64(ptr %addr, i64 %cmp, i64 %new) {
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd1, [acq_rel_sys_i64_param_0];
+; SM60-NEXT: membar.sys;
; SM60-NEXT: ld.param.b64 %rd2, [acq_rel_sys_i64_param_1];
; SM60-NEXT: ld.param.b64 %rd3, [acq_rel_sys_i64_param_2];
; SM60-NEXT: atom.sys.cas.b64 %rd4, [%rd1], %rd2, %rd3;
+; SM60-NEXT: membar.sys;
; SM60-NEXT: st.param.b64 [func_retval0], %rd3;
; SM60-NEXT: ret;
;
@@ -1640,6 +1648,7 @@ define i64 @release_sys_i64(ptr %addr, i64 %cmp, i64 %new) {
; SM60-EMPTY:
; SM60-NEXT: // %bb.0:
; SM60-NEXT: ld.param.b64 %rd1, [release_sys_i64_param_0];
+; SM60-NEXT: membar.sys;
; SM60-NEXT: ld.param.b64 %rd2, [release_sys_i64_param_1];
; SM60-NEXT: ld.param.b64 %rd3, [release_sys_i64_param_2];
; SM60-NEXT: atom.sys.cas.b64 %rd4, [%rd1], %rd2, %rd3;
@@ -1684,6 +1693,7 @@ define i64 @seq_cst_sys_i64(ptr %addr, i64 %cmp, i64 %new) {
; SM60-NEXT: ld.param.b64 %rd2, [seq_cst_sys_i64_param_1];
; SM60-NEXT: ld.param.b64 %rd3, [seq_cst_sys_i64_param_2];
; SM60-NEXT: atom.sys.cas.b64 %rd4, [%rd1], %rd2, %rd3;
+; SM60-NEXT: membar.sys;
; SM60-NEXT: st.param.b64 [func_retval0], %rd3;
; SM60-NEXT: ret;
;
More information about the llvm-commits
mailing list