[llvm] [NVPTX] Respect FTZ flag when lowering atomicrmw fadd. (PR #200732)
Justin Lebar via llvm-commits
llvm-commits at lists.llvm.org
Mon Jun 1 22:59:22 PDT 2026
https://github.com/jlebar updated https://github.com/llvm/llvm-project/pull/200732
>From 76b368f40f1ea90f97bd262ff740ca353d804bbd Mon Sep 17 00:00:00 2001
From: Justin Lebar <justin.lebar at gmail.com>
Date: Mon, 1 Jun 2026 22:44:05 -0700
Subject: [PATCH] [NVPTX] Respect FTZ flag when lowering atomicrmw fadd.
Previously we unconditionally lowered LLVM atomicrmw fadd to PTX atom.add.
This is incorrect, because it ignores the FTZ behavior of the LLVM and PTX
instructions:
- atom.add.f32 on global memory flushes denormals
- atom.add.f32 on shared memory does not flush denormals
- atom.add.f16 and atom.add.bf16 never flush denormals
We now lower to atom.add only when the function's denormal (FTZ) mode matches
that of the instruction; otherwise we fall back to a CAS loop.
Add -nvptx-allow-ftz-atomics as an escape hatch to use atom.add regardless of
the FTZ mismatch, trading correct denormal handling for performance.
Tests are regenerated via the two-step process (atomicrmw.py then
update_llc_test_checks.py); the generator now emits each test with and without
-denormal-fp-math-f32=preserve-sign. The new flag is covered by
atomicrmw-allow-ftz-atomics.ll.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply at anthropic.com>
---
llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp | 66 +-
.../NVPTX/atomicrmw-allow-ftz-atomics.ll | 105 +
llvm/test/CodeGen/NVPTX/atomicrmw-sm60.ll | 2270 +++++++++++------
llvm/test/CodeGen/NVPTX/atomicrmw-sm70.ll | 2218 +++++++++++-----
llvm/test/CodeGen/NVPTX/atomicrmw-sm90.ll | 1240 ++++++---
llvm/test/CodeGen/NVPTX/atomicrmw.py | 9 +-
llvm/test/CodeGen/NVPTX/atomics.ll | 34 +-
7 files changed, 4128 insertions(+), 1814 deletions(-)
create mode 100644 llvm/test/CodeGen/NVPTX/atomicrmw-allow-ftz-atomics.ll
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
index 75b8e736251d8..1448cf7dea9c6 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
@@ -108,6 +108,16 @@ static cl::opt<bool> UsePrecSqrtF32(
cl::desc("NVPTX Specific: 0 use sqrt.approx, 1 use sqrt.rn."),
cl::init(true));
+// PTX atom.add.f32 has fixed FTZ behavior that may not match the function's
+// (see shouldExpandAtomicRMWInIR), so by default we fall back to a CAS loop
+// when they disagree. This flag is an escape hatch to use atom.add anyway,
+// trading correct denormal handling for the speed of the native instruction.
+static cl::opt<bool> AllowFTZAtomics(
+ "nvptx-allow-ftz-atomics", cl::Hidden,
+ cl::desc("NVPTX Specific: Lower atomicrmw fadd to atom.add even when its "
+ "FTZ behavior does not match the function's denormal mode."),
+ cl::init(false));
+
/// Whereas CUDA's implementation (see libdevice) uses ex2.approx for exp2(), it
/// does NOT use lg2.approx for log2, so this is disabled by default.
static cl::opt<bool> UseApproxLog2F32(
@@ -7466,22 +7476,54 @@ NVPTXTargetLowering::AtomicExpansionKind
NVPTXTargetLowering::shouldExpandAtomicRMWInIR(const AtomicRMWInst *AI) const {
Type *Ty = AI->getValOperand()->getType();
- if (AI->isFloatingPointOperation()) {
- if (AI->getOperation() == AtomicRMWInst::BinOp::FAdd) {
- if (Ty->isHalfTy() && STI.getSmVersion() >= 70 &&
- STI.getPTXVersion() >= 63)
- return AtomicExpansionKind::None;
- if (Ty->isBFloatTy() && STI.getSmVersion() >= 90 &&
- STI.getPTXVersion() >= 78)
- return AtomicExpansionKind::None;
- if (Ty->isFloatTy())
- return AtomicExpansionKind::None;
- if (Ty->isDoubleTy() && STI.hasAtomAddF64())
+ // Try to lower LLVM atomicrmw fadd to PTX atomic.add. This is complicated
+ // by the weird FTZ behavior PTX atom.add has:
+ // - atom.add.f32 on global memory flushes denormals
+ // - atom.add.f32 on shared memory does not flush denormals
+ // - atom.add.f16 and atomic.add.bf16 never flush denormals
+ //
+ // We lower to atom.add only if the function's FTZ behavior matches that of
+ // atom.add; otherwise, we lower to a CAS loop. But we always allow
+ // atomic.add.bf16; even though it never flushes denormals, we never flush
+ // bf16 denormals when doing regular arithmetic, even when FTZ is enabled.
+ if (AI->isFloatingPointOperation() &&
+ AI->getOperation() == AtomicRMWInst::BinOp::FAdd) {
+ const bool FTZ =
+ AI->getFunction()->getDenormalMode(APFloat::IEEEsingle()).Output ==
+ DenormalMode::PreserveSign;
+
+ // AllowFTZAtomics forces atom.add regardless of the FTZ mismatch.
+ if (Ty->isFloatTy()) {
+ bool UseNative = AllowFTZAtomics;
+ switch (AI->getPointerAddressSpace()) {
+ case llvm::ADDRESS_SPACE_GLOBAL:
+ UseNative |= FTZ;
+ break;
+ case llvm::ADDRESS_SPACE_SHARED:
+ case llvm::ADDRESS_SPACE_SHARED_CLUSTER:
+ UseNative |= !FTZ;
+ break;
+ }
+ if (UseNative)
return AtomicExpansionKind::None;
}
- return AtomicExpansionKind::CmpXChg;
+
+ if (Ty->isHalfTy() && (!FTZ || AllowFTZAtomics) &&
+ STI.getSmVersion() >= 70 && STI.getPTXVersion() >= 63)
+ return AtomicExpansionKind::None;
+
+ if (Ty->isBFloatTy() && STI.getSmVersion() >= 90 &&
+ STI.getPTXVersion() >= 78)
+ return AtomicExpansionKind::None;
+
+ if (Ty->isDoubleTy() && STI.hasAtomAddF64())
+ return AtomicExpansionKind::None;
}
+ // PTX's only atomic fp op is `add`; all other ops expand to a CAS loop.
+ if (AI->isFloatingPointOperation())
+ return AtomicExpansionKind::CmpXChg;
+
assert(Ty->isIntegerTy() && "Ty should be integer at this point");
const unsigned BitWidth = cast<IntegerType>(Ty)->getBitWidth();
diff --git a/llvm/test/CodeGen/NVPTX/atomicrmw-allow-ftz-atomics.ll b/llvm/test/CodeGen/NVPTX/atomicrmw-allow-ftz-atomics.ll
new file mode 100644
index 0000000000000..727c13bb4e47e
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/atomicrmw-allow-ftz-atomics.ll
@@ -0,0 +1,105 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; -nvptx-allow-ftz-atomics forces atomicrmw fadd to lower to atom.add even when
+; the hardware's FTZ behavior doesn't match the function's denormal mode, where
+; we'd otherwise emit a CAS loop. Compare the default lowering (DEFAULT) against
+; the flag (ALLOW).
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_90 -mattr=+ptx87 | FileCheck %s --check-prefixes=CHECK,DEFAULT
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_90 -mattr=+ptx87 -nvptx-allow-ftz-atomics | FileCheck %s --check-prefixes=CHECK,ALLOW
+; RUN: %if ptxas-sm_90 && ptxas-isa-8.7 %{ llc < %s -march=nvptx64 -mcpu=sm_90 -mattr=+ptx87 | %ptxas-verify -arch=sm_90 %}
+; RUN: %if ptxas-sm_90 && ptxas-isa-8.7 %{ llc < %s -march=nvptx64 -mcpu=sm_90 -mattr=+ptx87 -nvptx-allow-ftz-atomics | %ptxas-verify -arch=sm_90 %}
+
+; global f32: atom.add flushes, so default uses a CAS loop (function is not FTZ);
+; the flag forces atom.add.
+define float @fadd_f32_global(ptr addrspace(1) %addr, float %val) {
+; DEFAULT-LABEL: fadd_f32_global(
+; DEFAULT: {
+; DEFAULT-NEXT: .reg .pred %p<2>;
+; DEFAULT-NEXT: .reg .b32 %r<5>;
+; DEFAULT-NEXT: .reg .b64 %rd<2>;
+; DEFAULT-EMPTY:
+; DEFAULT-NEXT: // %bb.0:
+; DEFAULT-NEXT: ld.param.b32 %r2, [fadd_f32_global_param_1];
+; DEFAULT-NEXT: ld.param.b64 %rd1, [fadd_f32_global_param_0];
+; DEFAULT-NEXT: ld.relaxed.sys.global.b32 %r4, [%rd1];
+; DEFAULT-NEXT: $L__BB0_1: // %atomicrmw.start
+; DEFAULT-NEXT: // =>This Inner Loop Header: Depth=1
+; DEFAULT-NEXT: add.rn.f32 %r3, %r4, %r2;
+; DEFAULT-NEXT: atom.relaxed.sys.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; DEFAULT-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; DEFAULT-NEXT: mov.b32 %r4, %r1;
+; DEFAULT-NEXT: @%p1 bra $L__BB0_1;
+; DEFAULT-NEXT: // %bb.2: // %atomicrmw.end
+; DEFAULT-NEXT: st.param.b32 [func_retval0], %r1;
+; DEFAULT-NEXT: ret;
+;
+; ALLOW-LABEL: fadd_f32_global(
+; ALLOW: {
+; ALLOW-NEXT: .reg .b32 %r<3>;
+; ALLOW-NEXT: .reg .b64 %rd<2>;
+; ALLOW-EMPTY:
+; ALLOW-NEXT: // %bb.0:
+; ALLOW-NEXT: ld.param.b64 %rd1, [fadd_f32_global_param_0];
+; ALLOW-NEXT: ld.param.b32 %r1, [fadd_f32_global_param_1];
+; ALLOW-NEXT: atom.relaxed.sys.global.add.f32 %r2, [%rd1], %r1;
+; ALLOW-NEXT: st.param.b32 [func_retval0], %r2;
+; ALLOW-NEXT: ret;
+ %r = atomicrmw fadd ptr addrspace(1) %addr, float %val monotonic
+ ret float %r
+}
+
+; generic f32: we can't know the memory space, so default always uses a CAS loop;
+; the flag forces atom.add.
+define float @fadd_f32_generic(ptr %addr, float %val) {
+; DEFAULT-LABEL: fadd_f32_generic(
+; DEFAULT: {
+; DEFAULT-NEXT: .reg .pred %p<2>;
+; DEFAULT-NEXT: .reg .b32 %r<5>;
+; DEFAULT-NEXT: .reg .b64 %rd<2>;
+; DEFAULT-EMPTY:
+; DEFAULT-NEXT: // %bb.0:
+; DEFAULT-NEXT: ld.param.b32 %r2, [fadd_f32_generic_param_1];
+; DEFAULT-NEXT: ld.param.b64 %rd1, [fadd_f32_generic_param_0];
+; DEFAULT-NEXT: ld.relaxed.sys.b32 %r4, [%rd1];
+; DEFAULT-NEXT: $L__BB1_1: // %atomicrmw.start
+; DEFAULT-NEXT: // =>This Inner Loop Header: Depth=1
+; DEFAULT-NEXT: add.rn.f32 %r3, %r4, %r2;
+; DEFAULT-NEXT: atom.relaxed.sys.cas.b32 %r1, [%rd1], %r4, %r3;
+; DEFAULT-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; DEFAULT-NEXT: mov.b32 %r4, %r1;
+; DEFAULT-NEXT: @%p1 bra $L__BB1_1;
+; DEFAULT-NEXT: // %bb.2: // %atomicrmw.end
+; DEFAULT-NEXT: st.param.b32 [func_retval0], %r1;
+; DEFAULT-NEXT: ret;
+;
+; ALLOW-LABEL: fadd_f32_generic(
+; ALLOW: {
+; ALLOW-NEXT: .reg .b32 %r<3>;
+; ALLOW-NEXT: .reg .b64 %rd<2>;
+; ALLOW-EMPTY:
+; ALLOW-NEXT: // %bb.0:
+; ALLOW-NEXT: ld.param.b64 %rd1, [fadd_f32_generic_param_0];
+; ALLOW-NEXT: ld.param.b32 %r1, [fadd_f32_generic_param_1];
+; ALLOW-NEXT: atom.relaxed.sys.add.f32 %r2, [%rd1], %r1;
+; ALLOW-NEXT: st.param.b32 [func_retval0], %r2;
+; ALLOW-NEXT: ret;
+ %r = atomicrmw fadd ptr %addr, float %val monotonic
+ ret float %r
+}
+
+; shared f32: atom.add does not flush, matching the non-FTZ function, so this is
+; atom.add either way.
+define float @fadd_f32_shared(ptr addrspace(3) %addr, float %val) {
+; CHECK-LABEL: fadd_f32_shared(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [fadd_f32_shared_param_0];
+; CHECK-NEXT: ld.param.b32 %r1, [fadd_f32_shared_param_1];
+; CHECK-NEXT: atom.relaxed.sys.shared.add.f32 %r2, [%rd1], %r1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %r = atomicrmw fadd ptr addrspace(3) %addr, float %val monotonic
+ ret float %r
+}
diff --git a/llvm/test/CodeGen/NVPTX/atomicrmw-sm60.ll b/llvm/test/CodeGen/NVPTX/atomicrmw-sm60.ll
index 8886873fd5c2e..5eb508eb12e17 100644
--- a/llvm/test/CodeGen/NVPTX/atomicrmw-sm60.ll
+++ b/llvm/test/CodeGen/NVPTX/atomicrmw-sm60.ll
@@ -1,6 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
-; RUN: llc < %s -march=nvptx64 -mcpu=sm_60 -mattr=+ptx50 | FileCheck %s --check-prefix=SM60
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_60 -mattr=+ptx50 | FileCheck %s --check-prefixes=SM60,SM60-NOFTZ
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_60 -mattr=+ptx50 -denormal-fp-math-f32=preserve-sign | FileCheck %s --check-prefixes=SM60,SM60-FTZ
; RUN: %if ptxas-sm_60 && ptxas-isa-5.0 %{ llc < %s -march=nvptx64 -mcpu=sm_60 -mattr=+ptx50 | %ptxas-verify -arch=sm_60 %}
+; RUN: %if ptxas-sm_60 && ptxas-isa-5.0 %{ llc < %s -march=nvptx64 -mcpu=sm_60 -mattr=+ptx50 -denormal-fp-math-f32=preserve-sign | %ptxas-verify -arch=sm_60 %}
define i8 @xchg_acq_rel_i8_global_cta(ptr addrspace(1) %addr, i8 %val) {
; SM60-LABEL: xchg_acq_rel_i8_global_cta(
@@ -1737,168 +1739,322 @@ define i64 @usub_sat_acq_rel_i64_global_cta(ptr addrspace(1) %addr, i64 %val) {
}
define float @fadd_acq_rel_float_global_cta(ptr addrspace(1) %addr, float %val) {
-; SM60-LABEL: fadd_acq_rel_float_global_cta(
-; SM60: {
-; SM60-NEXT: .reg .b32 %r<3>;
-; SM60-NEXT: .reg .b64 %rd<2>;
-; SM60-EMPTY:
-; SM60-NEXT: // %bb.0:
-; SM60-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_cta_param_0];
-; SM60-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_global_cta_param_1];
-; SM60-NEXT: atom.cta.global.add.f32 %r2, [%rd1], %r1;
-; SM60-NEXT: st.param.b32 [func_retval0], %r2;
-; SM60-NEXT: ret;
+; SM60-NOFTZ-LABEL: fadd_acq_rel_float_global_cta(
+; SM60-NOFTZ: {
+; SM60-NOFTZ-NEXT: .reg .pred %p<2>;
+; SM60-NOFTZ-NEXT: .reg .b32 %r<5>;
+; SM60-NOFTZ-NEXT: .reg .b64 %rd<2>;
+; SM60-NOFTZ-EMPTY:
+; SM60-NOFTZ-NEXT: // %bb.0:
+; SM60-NOFTZ-NEXT: ld.param.b32 %r2, [fadd_acq_rel_float_global_cta_param_1];
+; SM60-NOFTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_cta_param_0];
+; SM60-NOFTZ-NEXT: membar.cta;
+; SM60-NOFTZ-NEXT: ld.volatile.global.b32 %r4, [%rd1];
+; SM60-NOFTZ-NEXT: $L__BB60_1: // %atomicrmw.start
+; SM60-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NOFTZ-NEXT: add.rn.f32 %r3, %r4, %r2;
+; SM60-NOFTZ-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM60-NOFTZ-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM60-NOFTZ-NEXT: mov.b32 %r4, %r1;
+; SM60-NOFTZ-NEXT: @%p1 bra $L__BB60_1;
+; SM60-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NOFTZ-NEXT: membar.cta;
+; SM60-NOFTZ-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NOFTZ-NEXT: ret;
+;
+; SM60-FTZ-LABEL: fadd_acq_rel_float_global_cta(
+; SM60-FTZ: {
+; SM60-FTZ-NEXT: .reg .b32 %r<3>;
+; SM60-FTZ-NEXT: .reg .b64 %rd<2>;
+; SM60-FTZ-EMPTY:
+; SM60-FTZ-NEXT: // %bb.0:
+; SM60-FTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_cta_param_0];
+; SM60-FTZ-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_global_cta_param_1];
+; SM60-FTZ-NEXT: atom.cta.global.add.f32 %r2, [%rd1], %r1;
+; SM60-FTZ-NEXT: st.param.b32 [func_retval0], %r2;
+; SM60-FTZ-NEXT: ret;
%retval = atomicrmw fadd ptr addrspace(1) %addr, float %val syncscope("block") acq_rel
ret float %retval
}
define float @fsub_acq_rel_float_global_cta(ptr addrspace(1) %addr, float %val) {
-; SM60-LABEL: fsub_acq_rel_float_global_cta(
-; SM60: {
-; SM60-NEXT: .reg .pred %p<2>;
-; SM60-NEXT: .reg .b32 %r<5>;
-; SM60-NEXT: .reg .b64 %rd<2>;
-; SM60-EMPTY:
-; SM60-NEXT: // %bb.0:
-; SM60-NEXT: ld.param.b32 %r2, [fsub_acq_rel_float_global_cta_param_1];
-; SM60-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_float_global_cta_param_0];
-; SM60-NEXT: membar.cta;
-; SM60-NEXT: ld.volatile.global.b32 %r4, [%rd1];
-; SM60-NEXT: $L__BB61_1: // %atomicrmw.start
-; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: sub.rn.f32 %r3, %r4, %r2;
-; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
-; SM60-NEXT: setp.ne.b32 %p1, %r1, %r4;
-; SM60-NEXT: mov.b32 %r4, %r1;
-; SM60-NEXT: @%p1 bra $L__BB61_1;
-; SM60-NEXT: // %bb.2: // %atomicrmw.end
-; SM60-NEXT: membar.cta;
-; SM60-NEXT: st.param.b32 [func_retval0], %r1;
-; SM60-NEXT: ret;
+; SM60-NOFTZ-LABEL: fsub_acq_rel_float_global_cta(
+; SM60-NOFTZ: {
+; SM60-NOFTZ-NEXT: .reg .pred %p<2>;
+; SM60-NOFTZ-NEXT: .reg .b32 %r<5>;
+; SM60-NOFTZ-NEXT: .reg .b64 %rd<2>;
+; SM60-NOFTZ-EMPTY:
+; SM60-NOFTZ-NEXT: // %bb.0:
+; SM60-NOFTZ-NEXT: ld.param.b32 %r2, [fsub_acq_rel_float_global_cta_param_1];
+; SM60-NOFTZ-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_float_global_cta_param_0];
+; SM60-NOFTZ-NEXT: membar.cta;
+; SM60-NOFTZ-NEXT: ld.volatile.global.b32 %r4, [%rd1];
+; SM60-NOFTZ-NEXT: $L__BB61_1: // %atomicrmw.start
+; SM60-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NOFTZ-NEXT: sub.rn.f32 %r3, %r4, %r2;
+; SM60-NOFTZ-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM60-NOFTZ-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM60-NOFTZ-NEXT: mov.b32 %r4, %r1;
+; SM60-NOFTZ-NEXT: @%p1 bra $L__BB61_1;
+; SM60-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NOFTZ-NEXT: membar.cta;
+; SM60-NOFTZ-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NOFTZ-NEXT: ret;
+;
+; SM60-FTZ-LABEL: fsub_acq_rel_float_global_cta(
+; SM60-FTZ: {
+; SM60-FTZ-NEXT: .reg .pred %p<2>;
+; SM60-FTZ-NEXT: .reg .b32 %r<5>;
+; SM60-FTZ-NEXT: .reg .b64 %rd<2>;
+; SM60-FTZ-EMPTY:
+; SM60-FTZ-NEXT: // %bb.0:
+; SM60-FTZ-NEXT: ld.param.b32 %r2, [fsub_acq_rel_float_global_cta_param_1];
+; SM60-FTZ-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_float_global_cta_param_0];
+; SM60-FTZ-NEXT: membar.cta;
+; SM60-FTZ-NEXT: ld.volatile.global.b32 %r4, [%rd1];
+; SM60-FTZ-NEXT: $L__BB61_1: // %atomicrmw.start
+; SM60-FTZ-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-FTZ-NEXT: sub.rn.ftz.f32 %r3, %r4, %r2;
+; SM60-FTZ-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM60-FTZ-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM60-FTZ-NEXT: mov.b32 %r4, %r1;
+; SM60-FTZ-NEXT: @%p1 bra $L__BB61_1;
+; SM60-FTZ-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-FTZ-NEXT: membar.cta;
+; SM60-FTZ-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-FTZ-NEXT: ret;
%retval = atomicrmw fsub ptr addrspace(1) %addr, float %val syncscope("block") acq_rel
ret float %retval
}
define float @fmin_acq_rel_float_global_cta(ptr addrspace(1) %addr, float %val) {
-; SM60-LABEL: fmin_acq_rel_float_global_cta(
-; SM60: {
-; SM60-NEXT: .reg .pred %p<2>;
-; SM60-NEXT: .reg .b32 %r<5>;
-; SM60-NEXT: .reg .b64 %rd<2>;
-; SM60-EMPTY:
-; SM60-NEXT: // %bb.0:
-; SM60-NEXT: ld.param.b32 %r2, [fmin_acq_rel_float_global_cta_param_1];
-; SM60-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_float_global_cta_param_0];
-; SM60-NEXT: membar.cta;
-; SM60-NEXT: ld.volatile.global.b32 %r4, [%rd1];
-; SM60-NEXT: $L__BB62_1: // %atomicrmw.start
-; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: min.f32 %r3, %r4, %r2;
-; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
-; SM60-NEXT: setp.ne.b32 %p1, %r1, %r4;
-; SM60-NEXT: mov.b32 %r4, %r1;
-; SM60-NEXT: @%p1 bra $L__BB62_1;
-; SM60-NEXT: // %bb.2: // %atomicrmw.end
-; SM60-NEXT: membar.cta;
-; SM60-NEXT: st.param.b32 [func_retval0], %r1;
-; SM60-NEXT: ret;
+; SM60-NOFTZ-LABEL: fmin_acq_rel_float_global_cta(
+; SM60-NOFTZ: {
+; SM60-NOFTZ-NEXT: .reg .pred %p<2>;
+; SM60-NOFTZ-NEXT: .reg .b32 %r<5>;
+; SM60-NOFTZ-NEXT: .reg .b64 %rd<2>;
+; SM60-NOFTZ-EMPTY:
+; SM60-NOFTZ-NEXT: // %bb.0:
+; SM60-NOFTZ-NEXT: ld.param.b32 %r2, [fmin_acq_rel_float_global_cta_param_1];
+; SM60-NOFTZ-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_float_global_cta_param_0];
+; SM60-NOFTZ-NEXT: membar.cta;
+; SM60-NOFTZ-NEXT: ld.volatile.global.b32 %r4, [%rd1];
+; SM60-NOFTZ-NEXT: $L__BB62_1: // %atomicrmw.start
+; SM60-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NOFTZ-NEXT: min.f32 %r3, %r4, %r2;
+; SM60-NOFTZ-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM60-NOFTZ-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM60-NOFTZ-NEXT: mov.b32 %r4, %r1;
+; SM60-NOFTZ-NEXT: @%p1 bra $L__BB62_1;
+; SM60-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NOFTZ-NEXT: membar.cta;
+; SM60-NOFTZ-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NOFTZ-NEXT: ret;
+;
+; SM60-FTZ-LABEL: fmin_acq_rel_float_global_cta(
+; SM60-FTZ: {
+; SM60-FTZ-NEXT: .reg .pred %p<2>;
+; SM60-FTZ-NEXT: .reg .b32 %r<5>;
+; SM60-FTZ-NEXT: .reg .b64 %rd<2>;
+; SM60-FTZ-EMPTY:
+; SM60-FTZ-NEXT: // %bb.0:
+; SM60-FTZ-NEXT: ld.param.b32 %r2, [fmin_acq_rel_float_global_cta_param_1];
+; SM60-FTZ-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_float_global_cta_param_0];
+; SM60-FTZ-NEXT: membar.cta;
+; SM60-FTZ-NEXT: ld.volatile.global.b32 %r4, [%rd1];
+; SM60-FTZ-NEXT: $L__BB62_1: // %atomicrmw.start
+; SM60-FTZ-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-FTZ-NEXT: min.ftz.f32 %r3, %r4, %r2;
+; SM60-FTZ-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM60-FTZ-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM60-FTZ-NEXT: mov.b32 %r4, %r1;
+; SM60-FTZ-NEXT: @%p1 bra $L__BB62_1;
+; SM60-FTZ-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-FTZ-NEXT: membar.cta;
+; SM60-FTZ-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-FTZ-NEXT: ret;
%retval = atomicrmw fmin ptr addrspace(1) %addr, float %val syncscope("block") acq_rel
ret float %retval
}
define float @fmax_acq_rel_float_global_cta(ptr addrspace(1) %addr, float %val) {
-; SM60-LABEL: fmax_acq_rel_float_global_cta(
-; SM60: {
-; SM60-NEXT: .reg .pred %p<2>;
-; SM60-NEXT: .reg .b32 %r<5>;
-; SM60-NEXT: .reg .b64 %rd<2>;
-; SM60-EMPTY:
-; SM60-NEXT: // %bb.0:
-; SM60-NEXT: ld.param.b32 %r2, [fmax_acq_rel_float_global_cta_param_1];
-; SM60-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_float_global_cta_param_0];
-; SM60-NEXT: membar.cta;
-; SM60-NEXT: ld.volatile.global.b32 %r4, [%rd1];
-; SM60-NEXT: $L__BB63_1: // %atomicrmw.start
-; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: max.f32 %r3, %r4, %r2;
-; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
-; SM60-NEXT: setp.ne.b32 %p1, %r1, %r4;
-; SM60-NEXT: mov.b32 %r4, %r1;
-; SM60-NEXT: @%p1 bra $L__BB63_1;
-; SM60-NEXT: // %bb.2: // %atomicrmw.end
-; SM60-NEXT: membar.cta;
-; SM60-NEXT: st.param.b32 [func_retval0], %r1;
-; SM60-NEXT: ret;
+; SM60-NOFTZ-LABEL: fmax_acq_rel_float_global_cta(
+; SM60-NOFTZ: {
+; SM60-NOFTZ-NEXT: .reg .pred %p<2>;
+; SM60-NOFTZ-NEXT: .reg .b32 %r<5>;
+; SM60-NOFTZ-NEXT: .reg .b64 %rd<2>;
+; SM60-NOFTZ-EMPTY:
+; SM60-NOFTZ-NEXT: // %bb.0:
+; SM60-NOFTZ-NEXT: ld.param.b32 %r2, [fmax_acq_rel_float_global_cta_param_1];
+; SM60-NOFTZ-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_float_global_cta_param_0];
+; SM60-NOFTZ-NEXT: membar.cta;
+; SM60-NOFTZ-NEXT: ld.volatile.global.b32 %r4, [%rd1];
+; SM60-NOFTZ-NEXT: $L__BB63_1: // %atomicrmw.start
+; SM60-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NOFTZ-NEXT: max.f32 %r3, %r4, %r2;
+; SM60-NOFTZ-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM60-NOFTZ-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM60-NOFTZ-NEXT: mov.b32 %r4, %r1;
+; SM60-NOFTZ-NEXT: @%p1 bra $L__BB63_1;
+; SM60-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NOFTZ-NEXT: membar.cta;
+; SM60-NOFTZ-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NOFTZ-NEXT: ret;
+;
+; SM60-FTZ-LABEL: fmax_acq_rel_float_global_cta(
+; SM60-FTZ: {
+; SM60-FTZ-NEXT: .reg .pred %p<2>;
+; SM60-FTZ-NEXT: .reg .b32 %r<5>;
+; SM60-FTZ-NEXT: .reg .b64 %rd<2>;
+; SM60-FTZ-EMPTY:
+; SM60-FTZ-NEXT: // %bb.0:
+; SM60-FTZ-NEXT: ld.param.b32 %r2, [fmax_acq_rel_float_global_cta_param_1];
+; SM60-FTZ-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_float_global_cta_param_0];
+; SM60-FTZ-NEXT: membar.cta;
+; SM60-FTZ-NEXT: ld.volatile.global.b32 %r4, [%rd1];
+; SM60-FTZ-NEXT: $L__BB63_1: // %atomicrmw.start
+; SM60-FTZ-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-FTZ-NEXT: max.ftz.f32 %r3, %r4, %r2;
+; SM60-FTZ-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM60-FTZ-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM60-FTZ-NEXT: mov.b32 %r4, %r1;
+; SM60-FTZ-NEXT: @%p1 bra $L__BB63_1;
+; SM60-FTZ-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-FTZ-NEXT: membar.cta;
+; SM60-FTZ-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-FTZ-NEXT: ret;
%retval = atomicrmw fmax ptr addrspace(1) %addr, float %val syncscope("block") acq_rel
ret float %retval
}
define float @fminimum_acq_rel_float_global_cta(ptr addrspace(1) %addr, float %val) {
-; SM60-LABEL: fminimum_acq_rel_float_global_cta(
-; SM60: {
-; SM60-NEXT: .reg .pred %p<6>;
-; SM60-NEXT: .reg .b32 %r<9>;
-; SM60-NEXT: .reg .b64 %rd<2>;
-; SM60-EMPTY:
-; SM60-NEXT: // %bb.0:
-; SM60-NEXT: ld.param.b32 %r2, [fminimum_acq_rel_float_global_cta_param_1];
-; SM60-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_float_global_cta_param_0];
-; SM60-NEXT: membar.cta;
-; SM60-NEXT: ld.volatile.global.b32 %r8, [%rd1];
-; SM60-NEXT: setp.eq.b32 %p3, %r2, -2147483648;
-; SM60-NEXT: $L__BB64_1: // %atomicrmw.start
-; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: setp.nan.f32 %p1, %r8, %r2;
-; SM60-NEXT: min.f32 %r3, %r8, %r2;
-; SM60-NEXT: selp.f32 %r4, 0f7FC00000, %r3, %p1;
-; SM60-NEXT: setp.eq.b32 %p2, %r8, -2147483648;
-; SM60-NEXT: selp.f32 %r5, %r8, %r4, %p2;
-; SM60-NEXT: selp.f32 %r6, %r2, %r5, %p3;
-; SM60-NEXT: setp.eq.f32 %p4, %r4, 0f00000000;
-; SM60-NEXT: selp.f32 %r7, %r6, %r4, %p4;
-; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r8, %r7;
-; SM60-NEXT: setp.ne.b32 %p5, %r1, %r8;
-; SM60-NEXT: mov.b32 %r8, %r1;
-; SM60-NEXT: @%p5 bra $L__BB64_1;
-; SM60-NEXT: // %bb.2: // %atomicrmw.end
-; SM60-NEXT: membar.cta;
-; SM60-NEXT: st.param.b32 [func_retval0], %r1;
-; SM60-NEXT: ret;
+; SM60-NOFTZ-LABEL: fminimum_acq_rel_float_global_cta(
+; SM60-NOFTZ: {
+; SM60-NOFTZ-NEXT: .reg .pred %p<6>;
+; SM60-NOFTZ-NEXT: .reg .b32 %r<9>;
+; SM60-NOFTZ-NEXT: .reg .b64 %rd<2>;
+; SM60-NOFTZ-EMPTY:
+; SM60-NOFTZ-NEXT: // %bb.0:
+; SM60-NOFTZ-NEXT: ld.param.b32 %r2, [fminimum_acq_rel_float_global_cta_param_1];
+; SM60-NOFTZ-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_float_global_cta_param_0];
+; SM60-NOFTZ-NEXT: membar.cta;
+; SM60-NOFTZ-NEXT: ld.volatile.global.b32 %r8, [%rd1];
+; SM60-NOFTZ-NEXT: setp.eq.b32 %p3, %r2, -2147483648;
+; SM60-NOFTZ-NEXT: $L__BB64_1: // %atomicrmw.start
+; SM60-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NOFTZ-NEXT: setp.nan.f32 %p1, %r8, %r2;
+; SM60-NOFTZ-NEXT: min.f32 %r3, %r8, %r2;
+; SM60-NOFTZ-NEXT: selp.f32 %r4, 0f7FC00000, %r3, %p1;
+; SM60-NOFTZ-NEXT: setp.eq.b32 %p2, %r8, -2147483648;
+; SM60-NOFTZ-NEXT: selp.f32 %r5, %r8, %r4, %p2;
+; SM60-NOFTZ-NEXT: selp.f32 %r6, %r2, %r5, %p3;
+; SM60-NOFTZ-NEXT: setp.eq.f32 %p4, %r4, 0f00000000;
+; SM60-NOFTZ-NEXT: selp.f32 %r7, %r6, %r4, %p4;
+; SM60-NOFTZ-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r8, %r7;
+; SM60-NOFTZ-NEXT: setp.ne.b32 %p5, %r1, %r8;
+; SM60-NOFTZ-NEXT: mov.b32 %r8, %r1;
+; SM60-NOFTZ-NEXT: @%p5 bra $L__BB64_1;
+; SM60-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NOFTZ-NEXT: membar.cta;
+; SM60-NOFTZ-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NOFTZ-NEXT: ret;
+;
+; SM60-FTZ-LABEL: fminimum_acq_rel_float_global_cta(
+; SM60-FTZ: {
+; SM60-FTZ-NEXT: .reg .pred %p<6>;
+; SM60-FTZ-NEXT: .reg .b32 %r<9>;
+; SM60-FTZ-NEXT: .reg .b64 %rd<2>;
+; SM60-FTZ-EMPTY:
+; SM60-FTZ-NEXT: // %bb.0:
+; SM60-FTZ-NEXT: ld.param.b32 %r2, [fminimum_acq_rel_float_global_cta_param_1];
+; SM60-FTZ-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_float_global_cta_param_0];
+; SM60-FTZ-NEXT: membar.cta;
+; SM60-FTZ-NEXT: ld.volatile.global.b32 %r8, [%rd1];
+; SM60-FTZ-NEXT: setp.eq.b32 %p3, %r2, -2147483648;
+; SM60-FTZ-NEXT: $L__BB64_1: // %atomicrmw.start
+; SM60-FTZ-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-FTZ-NEXT: setp.nan.ftz.f32 %p1, %r8, %r2;
+; SM60-FTZ-NEXT: min.ftz.f32 %r3, %r8, %r2;
+; SM60-FTZ-NEXT: selp.f32 %r4, 0f7FC00000, %r3, %p1;
+; SM60-FTZ-NEXT: setp.eq.b32 %p2, %r8, -2147483648;
+; SM60-FTZ-NEXT: selp.f32 %r5, %r8, %r4, %p2;
+; SM60-FTZ-NEXT: selp.f32 %r6, %r2, %r5, %p3;
+; SM60-FTZ-NEXT: setp.eq.ftz.f32 %p4, %r4, 0f00000000;
+; SM60-FTZ-NEXT: selp.f32 %r7, %r6, %r4, %p4;
+; SM60-FTZ-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r8, %r7;
+; SM60-FTZ-NEXT: setp.ne.b32 %p5, %r1, %r8;
+; SM60-FTZ-NEXT: mov.b32 %r8, %r1;
+; SM60-FTZ-NEXT: @%p5 bra $L__BB64_1;
+; SM60-FTZ-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-FTZ-NEXT: membar.cta;
+; SM60-FTZ-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-FTZ-NEXT: ret;
%retval = atomicrmw fminimum ptr addrspace(1) %addr, float %val syncscope("block") acq_rel
ret float %retval
}
define float @fmaximum_acq_rel_float_global_cta(ptr addrspace(1) %addr, float %val) {
-; SM60-LABEL: fmaximum_acq_rel_float_global_cta(
-; SM60: {
-; SM60-NEXT: .reg .pred %p<6>;
-; SM60-NEXT: .reg .b32 %r<9>;
-; SM60-NEXT: .reg .b64 %rd<2>;
-; SM60-EMPTY:
-; SM60-NEXT: // %bb.0:
-; SM60-NEXT: ld.param.b32 %r2, [fmaximum_acq_rel_float_global_cta_param_1];
-; SM60-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_float_global_cta_param_0];
-; SM60-NEXT: membar.cta;
-; SM60-NEXT: ld.volatile.global.b32 %r8, [%rd1];
-; SM60-NEXT: setp.eq.b32 %p3, %r2, 0;
-; SM60-NEXT: $L__BB65_1: // %atomicrmw.start
-; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: setp.nan.f32 %p1, %r8, %r2;
-; SM60-NEXT: max.f32 %r3, %r8, %r2;
-; SM60-NEXT: selp.f32 %r4, 0f7FC00000, %r3, %p1;
-; SM60-NEXT: setp.eq.b32 %p2, %r8, 0;
-; SM60-NEXT: selp.f32 %r5, %r8, %r4, %p2;
-; SM60-NEXT: selp.f32 %r6, %r2, %r5, %p3;
-; SM60-NEXT: setp.eq.f32 %p4, %r4, 0f00000000;
-; SM60-NEXT: selp.f32 %r7, %r6, %r4, %p4;
-; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r8, %r7;
-; SM60-NEXT: setp.ne.b32 %p5, %r1, %r8;
-; SM60-NEXT: mov.b32 %r8, %r1;
-; SM60-NEXT: @%p5 bra $L__BB65_1;
-; SM60-NEXT: // %bb.2: // %atomicrmw.end
-; SM60-NEXT: membar.cta;
-; SM60-NEXT: st.param.b32 [func_retval0], %r1;
-; SM60-NEXT: ret;
+; SM60-NOFTZ-LABEL: fmaximum_acq_rel_float_global_cta(
+; SM60-NOFTZ: {
+; SM60-NOFTZ-NEXT: .reg .pred %p<6>;
+; SM60-NOFTZ-NEXT: .reg .b32 %r<9>;
+; SM60-NOFTZ-NEXT: .reg .b64 %rd<2>;
+; SM60-NOFTZ-EMPTY:
+; SM60-NOFTZ-NEXT: // %bb.0:
+; SM60-NOFTZ-NEXT: ld.param.b32 %r2, [fmaximum_acq_rel_float_global_cta_param_1];
+; SM60-NOFTZ-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_float_global_cta_param_0];
+; SM60-NOFTZ-NEXT: membar.cta;
+; SM60-NOFTZ-NEXT: ld.volatile.global.b32 %r8, [%rd1];
+; SM60-NOFTZ-NEXT: setp.eq.b32 %p3, %r2, 0;
+; SM60-NOFTZ-NEXT: $L__BB65_1: // %atomicrmw.start
+; SM60-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NOFTZ-NEXT: setp.nan.f32 %p1, %r8, %r2;
+; SM60-NOFTZ-NEXT: max.f32 %r3, %r8, %r2;
+; SM60-NOFTZ-NEXT: selp.f32 %r4, 0f7FC00000, %r3, %p1;
+; SM60-NOFTZ-NEXT: setp.eq.b32 %p2, %r8, 0;
+; SM60-NOFTZ-NEXT: selp.f32 %r5, %r8, %r4, %p2;
+; SM60-NOFTZ-NEXT: selp.f32 %r6, %r2, %r5, %p3;
+; SM60-NOFTZ-NEXT: setp.eq.f32 %p4, %r4, 0f00000000;
+; SM60-NOFTZ-NEXT: selp.f32 %r7, %r6, %r4, %p4;
+; SM60-NOFTZ-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r8, %r7;
+; SM60-NOFTZ-NEXT: setp.ne.b32 %p5, %r1, %r8;
+; SM60-NOFTZ-NEXT: mov.b32 %r8, %r1;
+; SM60-NOFTZ-NEXT: @%p5 bra $L__BB65_1;
+; SM60-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NOFTZ-NEXT: membar.cta;
+; SM60-NOFTZ-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NOFTZ-NEXT: ret;
+;
+; SM60-FTZ-LABEL: fmaximum_acq_rel_float_global_cta(
+; SM60-FTZ: {
+; SM60-FTZ-NEXT: .reg .pred %p<6>;
+; SM60-FTZ-NEXT: .reg .b32 %r<9>;
+; SM60-FTZ-NEXT: .reg .b64 %rd<2>;
+; SM60-FTZ-EMPTY:
+; SM60-FTZ-NEXT: // %bb.0:
+; SM60-FTZ-NEXT: ld.param.b32 %r2, [fmaximum_acq_rel_float_global_cta_param_1];
+; SM60-FTZ-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_float_global_cta_param_0];
+; SM60-FTZ-NEXT: membar.cta;
+; SM60-FTZ-NEXT: ld.volatile.global.b32 %r8, [%rd1];
+; SM60-FTZ-NEXT: setp.eq.b32 %p3, %r2, 0;
+; SM60-FTZ-NEXT: $L__BB65_1: // %atomicrmw.start
+; SM60-FTZ-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-FTZ-NEXT: setp.nan.ftz.f32 %p1, %r8, %r2;
+; SM60-FTZ-NEXT: max.ftz.f32 %r3, %r8, %r2;
+; SM60-FTZ-NEXT: selp.f32 %r4, 0f7FC00000, %r3, %p1;
+; SM60-FTZ-NEXT: setp.eq.b32 %p2, %r8, 0;
+; SM60-FTZ-NEXT: selp.f32 %r5, %r8, %r4, %p2;
+; SM60-FTZ-NEXT: selp.f32 %r6, %r2, %r5, %p3;
+; SM60-FTZ-NEXT: setp.eq.ftz.f32 %p4, %r4, 0f00000000;
+; SM60-FTZ-NEXT: selp.f32 %r7, %r6, %r4, %p4;
+; SM60-FTZ-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r8, %r7;
+; SM60-FTZ-NEXT: setp.ne.b32 %p5, %r1, %r8;
+; SM60-FTZ-NEXT: mov.b32 %r8, %r1;
+; SM60-FTZ-NEXT: @%p5 bra $L__BB65_1;
+; SM60-FTZ-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-FTZ-NEXT: membar.cta;
+; SM60-FTZ-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-FTZ-NEXT: ret;
%retval = atomicrmw fmaximum ptr addrspace(1) %addr, float %val syncscope("block") acq_rel
ret float %retval
}
@@ -2065,591 +2221,1133 @@ define double @fmaximum_acq_rel_double_global_cta(ptr addrspace(1) %addr, double
}
define half @fadd_acq_rel_half_global_cta(ptr addrspace(1) %addr, half %val) {
-; SM60-LABEL: fadd_acq_rel_half_global_cta(
-; SM60: {
-; SM60-NEXT: .reg .pred %p<2>;
-; SM60-NEXT: .reg .b16 %rs<4>;
-; SM60-NEXT: .reg .b32 %r<15>;
-; SM60-NEXT: .reg .b64 %rd<3>;
-; SM60-EMPTY:
-; SM60-NEXT: // %bb.0:
-; SM60-NEXT: ld.param.b16 %rs1, [fadd_acq_rel_half_global_cta_param_1];
-; SM60-NEXT: ld.param.b64 %rd2, [fadd_acq_rel_half_global_cta_param_0];
-; SM60-NEXT: membar.cta;
-; SM60-NEXT: and.b64 %rd1, %rd2, -4;
-; SM60-NEXT: cvt.u32.u64 %r4, %rd2;
-; SM60-NEXT: and.b32 %r5, %r4, 3;
-; SM60-NEXT: shl.b32 %r1, %r5, 3;
-; SM60-NEXT: mov.b32 %r6, 65535;
-; SM60-NEXT: shl.b32 %r7, %r6, %r1;
-; SM60-NEXT: not.b32 %r2, %r7;
-; SM60-NEXT: ld.volatile.global.b32 %r14, [%rd1];
-; SM60-NEXT: $L__BB72_1: // %atomicrmw.start
-; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: shr.u32 %r8, %r14, %r1;
-; SM60-NEXT: cvt.u16.u32 %rs2, %r8;
-; SM60-NEXT: add.rn.f16 %rs3, %rs2, %rs1;
-; SM60-NEXT: cvt.u32.u16 %r9, %rs3;
-; SM60-NEXT: shl.b32 %r10, %r9, %r1;
-; SM60-NEXT: and.b32 %r11, %r14, %r2;
-; SM60-NEXT: or.b32 %r12, %r11, %r10;
-; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
-; SM60-NEXT: setp.ne.b32 %p1, %r3, %r14;
-; SM60-NEXT: mov.b32 %r14, %r3;
-; SM60-NEXT: @%p1 bra $L__BB72_1;
-; SM60-NEXT: // %bb.2: // %atomicrmw.end
-; SM60-NEXT: shr.u32 %r13, %r3, %r1;
-; SM60-NEXT: membar.cta;
-; SM60-NEXT: st.param.b16 [func_retval0], %r13;
-; SM60-NEXT: ret;
+; SM60-NOFTZ-LABEL: fadd_acq_rel_half_global_cta(
+; SM60-NOFTZ: {
+; SM60-NOFTZ-NEXT: .reg .pred %p<2>;
+; SM60-NOFTZ-NEXT: .reg .b16 %rs<4>;
+; SM60-NOFTZ-NEXT: .reg .b32 %r<15>;
+; SM60-NOFTZ-NEXT: .reg .b64 %rd<3>;
+; SM60-NOFTZ-EMPTY:
+; SM60-NOFTZ-NEXT: // %bb.0:
+; SM60-NOFTZ-NEXT: ld.param.b16 %rs1, [fadd_acq_rel_half_global_cta_param_1];
+; SM60-NOFTZ-NEXT: ld.param.b64 %rd2, [fadd_acq_rel_half_global_cta_param_0];
+; SM60-NOFTZ-NEXT: membar.cta;
+; SM60-NOFTZ-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NOFTZ-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-NOFTZ-NEXT: and.b32 %r5, %r4, 3;
+; SM60-NOFTZ-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-NOFTZ-NEXT: mov.b32 %r6, 65535;
+; SM60-NOFTZ-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-NOFTZ-NEXT: not.b32 %r2, %r7;
+; SM60-NOFTZ-NEXT: ld.volatile.global.b32 %r14, [%rd1];
+; SM60-NOFTZ-NEXT: $L__BB72_1: // %atomicrmw.start
+; SM60-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NOFTZ-NEXT: shr.u32 %r8, %r14, %r1;
+; SM60-NOFTZ-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM60-NOFTZ-NEXT: add.rn.f16 %rs3, %rs2, %rs1;
+; SM60-NOFTZ-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM60-NOFTZ-NEXT: shl.b32 %r10, %r9, %r1;
+; SM60-NOFTZ-NEXT: and.b32 %r11, %r14, %r2;
+; SM60-NOFTZ-NEXT: or.b32 %r12, %r11, %r10;
+; SM60-NOFTZ-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM60-NOFTZ-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM60-NOFTZ-NEXT: mov.b32 %r14, %r3;
+; SM60-NOFTZ-NEXT: @%p1 bra $L__BB72_1;
+; SM60-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NOFTZ-NEXT: shr.u32 %r13, %r3, %r1;
+; SM60-NOFTZ-NEXT: membar.cta;
+; SM60-NOFTZ-NEXT: st.param.b16 [func_retval0], %r13;
+; SM60-NOFTZ-NEXT: ret;
+;
+; SM60-FTZ-LABEL: fadd_acq_rel_half_global_cta(
+; SM60-FTZ: {
+; SM60-FTZ-NEXT: .reg .pred %p<2>;
+; SM60-FTZ-NEXT: .reg .b16 %rs<4>;
+; SM60-FTZ-NEXT: .reg .b32 %r<15>;
+; SM60-FTZ-NEXT: .reg .b64 %rd<3>;
+; SM60-FTZ-EMPTY:
+; SM60-FTZ-NEXT: // %bb.0:
+; SM60-FTZ-NEXT: ld.param.b16 %rs1, [fadd_acq_rel_half_global_cta_param_1];
+; SM60-FTZ-NEXT: ld.param.b64 %rd2, [fadd_acq_rel_half_global_cta_param_0];
+; SM60-FTZ-NEXT: membar.cta;
+; SM60-FTZ-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-FTZ-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-FTZ-NEXT: and.b32 %r5, %r4, 3;
+; SM60-FTZ-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-FTZ-NEXT: mov.b32 %r6, 65535;
+; SM60-FTZ-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-FTZ-NEXT: not.b32 %r2, %r7;
+; SM60-FTZ-NEXT: ld.volatile.global.b32 %r14, [%rd1];
+; SM60-FTZ-NEXT: $L__BB72_1: // %atomicrmw.start
+; SM60-FTZ-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-FTZ-NEXT: shr.u32 %r8, %r14, %r1;
+; SM60-FTZ-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM60-FTZ-NEXT: add.rn.ftz.f16 %rs3, %rs2, %rs1;
+; SM60-FTZ-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM60-FTZ-NEXT: shl.b32 %r10, %r9, %r1;
+; SM60-FTZ-NEXT: and.b32 %r11, %r14, %r2;
+; SM60-FTZ-NEXT: or.b32 %r12, %r11, %r10;
+; SM60-FTZ-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM60-FTZ-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM60-FTZ-NEXT: mov.b32 %r14, %r3;
+; SM60-FTZ-NEXT: @%p1 bra $L__BB72_1;
+; SM60-FTZ-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-FTZ-NEXT: shr.u32 %r13, %r3, %r1;
+; SM60-FTZ-NEXT: membar.cta;
+; SM60-FTZ-NEXT: st.param.b16 [func_retval0], %r13;
+; SM60-FTZ-NEXT: ret;
%retval = atomicrmw fadd ptr addrspace(1) %addr, half %val syncscope("block") acq_rel
ret half %retval
}
define half @fsub_acq_rel_half_global_cta(ptr addrspace(1) %addr, half %val) {
-; SM60-LABEL: fsub_acq_rel_half_global_cta(
-; SM60: {
-; SM60-NEXT: .reg .pred %p<2>;
-; SM60-NEXT: .reg .b16 %rs<4>;
-; SM60-NEXT: .reg .b32 %r<15>;
-; SM60-NEXT: .reg .b64 %rd<3>;
-; SM60-EMPTY:
-; SM60-NEXT: // %bb.0:
-; SM60-NEXT: ld.param.b16 %rs1, [fsub_acq_rel_half_global_cta_param_1];
-; SM60-NEXT: ld.param.b64 %rd2, [fsub_acq_rel_half_global_cta_param_0];
-; SM60-NEXT: membar.cta;
-; SM60-NEXT: and.b64 %rd1, %rd2, -4;
-; SM60-NEXT: cvt.u32.u64 %r4, %rd2;
-; SM60-NEXT: and.b32 %r5, %r4, 3;
-; SM60-NEXT: shl.b32 %r1, %r5, 3;
-; SM60-NEXT: mov.b32 %r6, 65535;
-; SM60-NEXT: shl.b32 %r7, %r6, %r1;
-; SM60-NEXT: not.b32 %r2, %r7;
-; SM60-NEXT: ld.volatile.global.b32 %r14, [%rd1];
-; SM60-NEXT: $L__BB73_1: // %atomicrmw.start
-; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: shr.u32 %r8, %r14, %r1;
-; SM60-NEXT: cvt.u16.u32 %rs2, %r8;
-; SM60-NEXT: sub.rn.f16 %rs3, %rs2, %rs1;
-; SM60-NEXT: cvt.u32.u16 %r9, %rs3;
-; SM60-NEXT: shl.b32 %r10, %r9, %r1;
-; SM60-NEXT: and.b32 %r11, %r14, %r2;
-; SM60-NEXT: or.b32 %r12, %r11, %r10;
-; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
-; SM60-NEXT: setp.ne.b32 %p1, %r3, %r14;
-; SM60-NEXT: mov.b32 %r14, %r3;
-; SM60-NEXT: @%p1 bra $L__BB73_1;
-; SM60-NEXT: // %bb.2: // %atomicrmw.end
-; SM60-NEXT: shr.u32 %r13, %r3, %r1;
-; SM60-NEXT: membar.cta;
-; SM60-NEXT: st.param.b16 [func_retval0], %r13;
-; SM60-NEXT: ret;
+; SM60-NOFTZ-LABEL: fsub_acq_rel_half_global_cta(
+; SM60-NOFTZ: {
+; SM60-NOFTZ-NEXT: .reg .pred %p<2>;
+; SM60-NOFTZ-NEXT: .reg .b16 %rs<4>;
+; SM60-NOFTZ-NEXT: .reg .b32 %r<15>;
+; SM60-NOFTZ-NEXT: .reg .b64 %rd<3>;
+; SM60-NOFTZ-EMPTY:
+; SM60-NOFTZ-NEXT: // %bb.0:
+; SM60-NOFTZ-NEXT: ld.param.b16 %rs1, [fsub_acq_rel_half_global_cta_param_1];
+; SM60-NOFTZ-NEXT: ld.param.b64 %rd2, [fsub_acq_rel_half_global_cta_param_0];
+; SM60-NOFTZ-NEXT: membar.cta;
+; SM60-NOFTZ-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NOFTZ-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-NOFTZ-NEXT: and.b32 %r5, %r4, 3;
+; SM60-NOFTZ-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-NOFTZ-NEXT: mov.b32 %r6, 65535;
+; SM60-NOFTZ-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-NOFTZ-NEXT: not.b32 %r2, %r7;
+; SM60-NOFTZ-NEXT: ld.volatile.global.b32 %r14, [%rd1];
+; SM60-NOFTZ-NEXT: $L__BB73_1: // %atomicrmw.start
+; SM60-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NOFTZ-NEXT: shr.u32 %r8, %r14, %r1;
+; SM60-NOFTZ-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM60-NOFTZ-NEXT: sub.rn.f16 %rs3, %rs2, %rs1;
+; SM60-NOFTZ-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM60-NOFTZ-NEXT: shl.b32 %r10, %r9, %r1;
+; SM60-NOFTZ-NEXT: and.b32 %r11, %r14, %r2;
+; SM60-NOFTZ-NEXT: or.b32 %r12, %r11, %r10;
+; SM60-NOFTZ-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM60-NOFTZ-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM60-NOFTZ-NEXT: mov.b32 %r14, %r3;
+; SM60-NOFTZ-NEXT: @%p1 bra $L__BB73_1;
+; SM60-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NOFTZ-NEXT: shr.u32 %r13, %r3, %r1;
+; SM60-NOFTZ-NEXT: membar.cta;
+; SM60-NOFTZ-NEXT: st.param.b16 [func_retval0], %r13;
+; SM60-NOFTZ-NEXT: ret;
+;
+; SM60-FTZ-LABEL: fsub_acq_rel_half_global_cta(
+; SM60-FTZ: {
+; SM60-FTZ-NEXT: .reg .pred %p<2>;
+; SM60-FTZ-NEXT: .reg .b16 %rs<4>;
+; SM60-FTZ-NEXT: .reg .b32 %r<15>;
+; SM60-FTZ-NEXT: .reg .b64 %rd<3>;
+; SM60-FTZ-EMPTY:
+; SM60-FTZ-NEXT: // %bb.0:
+; SM60-FTZ-NEXT: ld.param.b16 %rs1, [fsub_acq_rel_half_global_cta_param_1];
+; SM60-FTZ-NEXT: ld.param.b64 %rd2, [fsub_acq_rel_half_global_cta_param_0];
+; SM60-FTZ-NEXT: membar.cta;
+; SM60-FTZ-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-FTZ-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-FTZ-NEXT: and.b32 %r5, %r4, 3;
+; SM60-FTZ-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-FTZ-NEXT: mov.b32 %r6, 65535;
+; SM60-FTZ-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-FTZ-NEXT: not.b32 %r2, %r7;
+; SM60-FTZ-NEXT: ld.volatile.global.b32 %r14, [%rd1];
+; SM60-FTZ-NEXT: $L__BB73_1: // %atomicrmw.start
+; SM60-FTZ-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-FTZ-NEXT: shr.u32 %r8, %r14, %r1;
+; SM60-FTZ-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM60-FTZ-NEXT: sub.rn.ftz.f16 %rs3, %rs2, %rs1;
+; SM60-FTZ-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM60-FTZ-NEXT: shl.b32 %r10, %r9, %r1;
+; SM60-FTZ-NEXT: and.b32 %r11, %r14, %r2;
+; SM60-FTZ-NEXT: or.b32 %r12, %r11, %r10;
+; SM60-FTZ-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM60-FTZ-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM60-FTZ-NEXT: mov.b32 %r14, %r3;
+; SM60-FTZ-NEXT: @%p1 bra $L__BB73_1;
+; SM60-FTZ-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-FTZ-NEXT: shr.u32 %r13, %r3, %r1;
+; SM60-FTZ-NEXT: membar.cta;
+; SM60-FTZ-NEXT: st.param.b16 [func_retval0], %r13;
+; SM60-FTZ-NEXT: ret;
%retval = atomicrmw fsub ptr addrspace(1) %addr, half %val syncscope("block") acq_rel
ret half %retval
}
define half @fmin_acq_rel_half_global_cta(ptr addrspace(1) %addr, half %val) {
-; SM60-LABEL: fmin_acq_rel_half_global_cta(
-; SM60: {
-; SM60-NEXT: .reg .pred %p<2>;
-; SM60-NEXT: .reg .b16 %rs<4>;
-; SM60-NEXT: .reg .b32 %r<18>;
-; SM60-NEXT: .reg .b64 %rd<3>;
-; SM60-EMPTY:
-; SM60-NEXT: // %bb.0:
-; SM60-NEXT: ld.param.b16 %rs1, [fmin_acq_rel_half_global_cta_param_1];
-; SM60-NEXT: ld.param.b64 %rd2, [fmin_acq_rel_half_global_cta_param_0];
-; SM60-NEXT: membar.cta;
-; SM60-NEXT: and.b64 %rd1, %rd2, -4;
-; SM60-NEXT: cvt.u32.u64 %r4, %rd2;
-; SM60-NEXT: and.b32 %r5, %r4, 3;
-; SM60-NEXT: shl.b32 %r1, %r5, 3;
-; SM60-NEXT: mov.b32 %r6, 65535;
-; SM60-NEXT: shl.b32 %r7, %r6, %r1;
-; SM60-NEXT: not.b32 %r2, %r7;
-; SM60-NEXT: ld.volatile.global.b32 %r17, [%rd1];
-; SM60-NEXT: cvt.f32.f16 %r10, %rs1;
-; SM60-NEXT: $L__BB74_1: // %atomicrmw.start
-; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: shr.u32 %r8, %r17, %r1;
-; SM60-NEXT: cvt.u16.u32 %rs2, %r8;
-; SM60-NEXT: cvt.f32.f16 %r9, %rs2;
-; SM60-NEXT: min.f32 %r11, %r9, %r10;
-; SM60-NEXT: cvt.rn.f16.f32 %rs3, %r11;
-; SM60-NEXT: cvt.u32.u16 %r12, %rs3;
-; SM60-NEXT: shl.b32 %r13, %r12, %r1;
-; SM60-NEXT: and.b32 %r14, %r17, %r2;
-; SM60-NEXT: or.b32 %r15, %r14, %r13;
-; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r17, %r15;
-; SM60-NEXT: setp.ne.b32 %p1, %r3, %r17;
-; SM60-NEXT: mov.b32 %r17, %r3;
-; SM60-NEXT: @%p1 bra $L__BB74_1;
-; SM60-NEXT: // %bb.2: // %atomicrmw.end
-; SM60-NEXT: shr.u32 %r16, %r3, %r1;
-; SM60-NEXT: membar.cta;
-; SM60-NEXT: st.param.b16 [func_retval0], %r16;
-; SM60-NEXT: ret;
+; SM60-NOFTZ-LABEL: fmin_acq_rel_half_global_cta(
+; SM60-NOFTZ: {
+; SM60-NOFTZ-NEXT: .reg .pred %p<2>;
+; SM60-NOFTZ-NEXT: .reg .b16 %rs<4>;
+; SM60-NOFTZ-NEXT: .reg .b32 %r<18>;
+; SM60-NOFTZ-NEXT: .reg .b64 %rd<3>;
+; SM60-NOFTZ-EMPTY:
+; SM60-NOFTZ-NEXT: // %bb.0:
+; SM60-NOFTZ-NEXT: ld.param.b16 %rs1, [fmin_acq_rel_half_global_cta_param_1];
+; SM60-NOFTZ-NEXT: ld.param.b64 %rd2, [fmin_acq_rel_half_global_cta_param_0];
+; SM60-NOFTZ-NEXT: membar.cta;
+; SM60-NOFTZ-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NOFTZ-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-NOFTZ-NEXT: and.b32 %r5, %r4, 3;
+; SM60-NOFTZ-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-NOFTZ-NEXT: mov.b32 %r6, 65535;
+; SM60-NOFTZ-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-NOFTZ-NEXT: not.b32 %r2, %r7;
+; SM60-NOFTZ-NEXT: ld.volatile.global.b32 %r17, [%rd1];
+; SM60-NOFTZ-NEXT: cvt.f32.f16 %r10, %rs1;
+; SM60-NOFTZ-NEXT: $L__BB74_1: // %atomicrmw.start
+; SM60-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NOFTZ-NEXT: shr.u32 %r8, %r17, %r1;
+; SM60-NOFTZ-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM60-NOFTZ-NEXT: cvt.f32.f16 %r9, %rs2;
+; SM60-NOFTZ-NEXT: min.f32 %r11, %r9, %r10;
+; SM60-NOFTZ-NEXT: cvt.rn.f16.f32 %rs3, %r11;
+; SM60-NOFTZ-NEXT: cvt.u32.u16 %r12, %rs3;
+; SM60-NOFTZ-NEXT: shl.b32 %r13, %r12, %r1;
+; SM60-NOFTZ-NEXT: and.b32 %r14, %r17, %r2;
+; SM60-NOFTZ-NEXT: or.b32 %r15, %r14, %r13;
+; SM60-NOFTZ-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r17, %r15;
+; SM60-NOFTZ-NEXT: setp.ne.b32 %p1, %r3, %r17;
+; SM60-NOFTZ-NEXT: mov.b32 %r17, %r3;
+; SM60-NOFTZ-NEXT: @%p1 bra $L__BB74_1;
+; SM60-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NOFTZ-NEXT: shr.u32 %r16, %r3, %r1;
+; SM60-NOFTZ-NEXT: membar.cta;
+; SM60-NOFTZ-NEXT: st.param.b16 [func_retval0], %r16;
+; SM60-NOFTZ-NEXT: ret;
+;
+; SM60-FTZ-LABEL: fmin_acq_rel_half_global_cta(
+; SM60-FTZ: {
+; SM60-FTZ-NEXT: .reg .pred %p<2>;
+; SM60-FTZ-NEXT: .reg .b16 %rs<4>;
+; SM60-FTZ-NEXT: .reg .b32 %r<18>;
+; SM60-FTZ-NEXT: .reg .b64 %rd<3>;
+; SM60-FTZ-EMPTY:
+; SM60-FTZ-NEXT: // %bb.0:
+; SM60-FTZ-NEXT: ld.param.b16 %rs1, [fmin_acq_rel_half_global_cta_param_1];
+; SM60-FTZ-NEXT: ld.param.b64 %rd2, [fmin_acq_rel_half_global_cta_param_0];
+; SM60-FTZ-NEXT: membar.cta;
+; SM60-FTZ-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-FTZ-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-FTZ-NEXT: and.b32 %r5, %r4, 3;
+; SM60-FTZ-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-FTZ-NEXT: mov.b32 %r6, 65535;
+; SM60-FTZ-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-FTZ-NEXT: not.b32 %r2, %r7;
+; SM60-FTZ-NEXT: ld.volatile.global.b32 %r17, [%rd1];
+; SM60-FTZ-NEXT: cvt.ftz.f32.f16 %r10, %rs1;
+; SM60-FTZ-NEXT: $L__BB74_1: // %atomicrmw.start
+; SM60-FTZ-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-FTZ-NEXT: shr.u32 %r8, %r17, %r1;
+; SM60-FTZ-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM60-FTZ-NEXT: cvt.ftz.f32.f16 %r9, %rs2;
+; SM60-FTZ-NEXT: min.ftz.f32 %r11, %r9, %r10;
+; SM60-FTZ-NEXT: cvt.rn.f16.f32 %rs3, %r11;
+; SM60-FTZ-NEXT: cvt.u32.u16 %r12, %rs3;
+; SM60-FTZ-NEXT: shl.b32 %r13, %r12, %r1;
+; SM60-FTZ-NEXT: and.b32 %r14, %r17, %r2;
+; SM60-FTZ-NEXT: or.b32 %r15, %r14, %r13;
+; SM60-FTZ-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r17, %r15;
+; SM60-FTZ-NEXT: setp.ne.b32 %p1, %r3, %r17;
+; SM60-FTZ-NEXT: mov.b32 %r17, %r3;
+; SM60-FTZ-NEXT: @%p1 bra $L__BB74_1;
+; SM60-FTZ-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-FTZ-NEXT: shr.u32 %r16, %r3, %r1;
+; SM60-FTZ-NEXT: membar.cta;
+; SM60-FTZ-NEXT: st.param.b16 [func_retval0], %r16;
+; SM60-FTZ-NEXT: ret;
%retval = atomicrmw fmin ptr addrspace(1) %addr, half %val syncscope("block") acq_rel
ret half %retval
}
define half @fmax_acq_rel_half_global_cta(ptr addrspace(1) %addr, half %val) {
-; SM60-LABEL: fmax_acq_rel_half_global_cta(
-; SM60: {
-; SM60-NEXT: .reg .pred %p<2>;
-; SM60-NEXT: .reg .b16 %rs<4>;
-; SM60-NEXT: .reg .b32 %r<18>;
-; SM60-NEXT: .reg .b64 %rd<3>;
-; SM60-EMPTY:
-; SM60-NEXT: // %bb.0:
-; SM60-NEXT: ld.param.b16 %rs1, [fmax_acq_rel_half_global_cta_param_1];
-; SM60-NEXT: ld.param.b64 %rd2, [fmax_acq_rel_half_global_cta_param_0];
-; SM60-NEXT: membar.cta;
-; SM60-NEXT: and.b64 %rd1, %rd2, -4;
-; SM60-NEXT: cvt.u32.u64 %r4, %rd2;
-; SM60-NEXT: and.b32 %r5, %r4, 3;
-; SM60-NEXT: shl.b32 %r1, %r5, 3;
-; SM60-NEXT: mov.b32 %r6, 65535;
-; SM60-NEXT: shl.b32 %r7, %r6, %r1;
-; SM60-NEXT: not.b32 %r2, %r7;
-; SM60-NEXT: ld.volatile.global.b32 %r17, [%rd1];
-; SM60-NEXT: cvt.f32.f16 %r10, %rs1;
-; SM60-NEXT: $L__BB75_1: // %atomicrmw.start
-; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: shr.u32 %r8, %r17, %r1;
-; SM60-NEXT: cvt.u16.u32 %rs2, %r8;
-; SM60-NEXT: cvt.f32.f16 %r9, %rs2;
-; SM60-NEXT: max.f32 %r11, %r9, %r10;
-; SM60-NEXT: cvt.rn.f16.f32 %rs3, %r11;
-; SM60-NEXT: cvt.u32.u16 %r12, %rs3;
-; SM60-NEXT: shl.b32 %r13, %r12, %r1;
-; SM60-NEXT: and.b32 %r14, %r17, %r2;
-; SM60-NEXT: or.b32 %r15, %r14, %r13;
-; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r17, %r15;
-; SM60-NEXT: setp.ne.b32 %p1, %r3, %r17;
-; SM60-NEXT: mov.b32 %r17, %r3;
-; SM60-NEXT: @%p1 bra $L__BB75_1;
-; SM60-NEXT: // %bb.2: // %atomicrmw.end
-; SM60-NEXT: shr.u32 %r16, %r3, %r1;
-; SM60-NEXT: membar.cta;
-; SM60-NEXT: st.param.b16 [func_retval0], %r16;
-; SM60-NEXT: ret;
+; SM60-NOFTZ-LABEL: fmax_acq_rel_half_global_cta(
+; SM60-NOFTZ: {
+; SM60-NOFTZ-NEXT: .reg .pred %p<2>;
+; SM60-NOFTZ-NEXT: .reg .b16 %rs<4>;
+; SM60-NOFTZ-NEXT: .reg .b32 %r<18>;
+; SM60-NOFTZ-NEXT: .reg .b64 %rd<3>;
+; SM60-NOFTZ-EMPTY:
+; SM60-NOFTZ-NEXT: // %bb.0:
+; SM60-NOFTZ-NEXT: ld.param.b16 %rs1, [fmax_acq_rel_half_global_cta_param_1];
+; SM60-NOFTZ-NEXT: ld.param.b64 %rd2, [fmax_acq_rel_half_global_cta_param_0];
+; SM60-NOFTZ-NEXT: membar.cta;
+; SM60-NOFTZ-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NOFTZ-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-NOFTZ-NEXT: and.b32 %r5, %r4, 3;
+; SM60-NOFTZ-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-NOFTZ-NEXT: mov.b32 %r6, 65535;
+; SM60-NOFTZ-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-NOFTZ-NEXT: not.b32 %r2, %r7;
+; SM60-NOFTZ-NEXT: ld.volatile.global.b32 %r17, [%rd1];
+; SM60-NOFTZ-NEXT: cvt.f32.f16 %r10, %rs1;
+; SM60-NOFTZ-NEXT: $L__BB75_1: // %atomicrmw.start
+; SM60-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NOFTZ-NEXT: shr.u32 %r8, %r17, %r1;
+; SM60-NOFTZ-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM60-NOFTZ-NEXT: cvt.f32.f16 %r9, %rs2;
+; SM60-NOFTZ-NEXT: max.f32 %r11, %r9, %r10;
+; SM60-NOFTZ-NEXT: cvt.rn.f16.f32 %rs3, %r11;
+; SM60-NOFTZ-NEXT: cvt.u32.u16 %r12, %rs3;
+; SM60-NOFTZ-NEXT: shl.b32 %r13, %r12, %r1;
+; SM60-NOFTZ-NEXT: and.b32 %r14, %r17, %r2;
+; SM60-NOFTZ-NEXT: or.b32 %r15, %r14, %r13;
+; SM60-NOFTZ-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r17, %r15;
+; SM60-NOFTZ-NEXT: setp.ne.b32 %p1, %r3, %r17;
+; SM60-NOFTZ-NEXT: mov.b32 %r17, %r3;
+; SM60-NOFTZ-NEXT: @%p1 bra $L__BB75_1;
+; SM60-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NOFTZ-NEXT: shr.u32 %r16, %r3, %r1;
+; SM60-NOFTZ-NEXT: membar.cta;
+; SM60-NOFTZ-NEXT: st.param.b16 [func_retval0], %r16;
+; SM60-NOFTZ-NEXT: ret;
+;
+; SM60-FTZ-LABEL: fmax_acq_rel_half_global_cta(
+; SM60-FTZ: {
+; SM60-FTZ-NEXT: .reg .pred %p<2>;
+; SM60-FTZ-NEXT: .reg .b16 %rs<4>;
+; SM60-FTZ-NEXT: .reg .b32 %r<18>;
+; SM60-FTZ-NEXT: .reg .b64 %rd<3>;
+; SM60-FTZ-EMPTY:
+; SM60-FTZ-NEXT: // %bb.0:
+; SM60-FTZ-NEXT: ld.param.b16 %rs1, [fmax_acq_rel_half_global_cta_param_1];
+; SM60-FTZ-NEXT: ld.param.b64 %rd2, [fmax_acq_rel_half_global_cta_param_0];
+; SM60-FTZ-NEXT: membar.cta;
+; SM60-FTZ-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-FTZ-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-FTZ-NEXT: and.b32 %r5, %r4, 3;
+; SM60-FTZ-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-FTZ-NEXT: mov.b32 %r6, 65535;
+; SM60-FTZ-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-FTZ-NEXT: not.b32 %r2, %r7;
+; SM60-FTZ-NEXT: ld.volatile.global.b32 %r17, [%rd1];
+; SM60-FTZ-NEXT: cvt.ftz.f32.f16 %r10, %rs1;
+; SM60-FTZ-NEXT: $L__BB75_1: // %atomicrmw.start
+; SM60-FTZ-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-FTZ-NEXT: shr.u32 %r8, %r17, %r1;
+; SM60-FTZ-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM60-FTZ-NEXT: cvt.ftz.f32.f16 %r9, %rs2;
+; SM60-FTZ-NEXT: max.ftz.f32 %r11, %r9, %r10;
+; SM60-FTZ-NEXT: cvt.rn.f16.f32 %rs3, %r11;
+; SM60-FTZ-NEXT: cvt.u32.u16 %r12, %rs3;
+; SM60-FTZ-NEXT: shl.b32 %r13, %r12, %r1;
+; SM60-FTZ-NEXT: and.b32 %r14, %r17, %r2;
+; SM60-FTZ-NEXT: or.b32 %r15, %r14, %r13;
+; SM60-FTZ-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r17, %r15;
+; SM60-FTZ-NEXT: setp.ne.b32 %p1, %r3, %r17;
+; SM60-FTZ-NEXT: mov.b32 %r17, %r3;
+; SM60-FTZ-NEXT: @%p1 bra $L__BB75_1;
+; SM60-FTZ-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-FTZ-NEXT: shr.u32 %r16, %r3, %r1;
+; SM60-FTZ-NEXT: membar.cta;
+; SM60-FTZ-NEXT: st.param.b16 [func_retval0], %r16;
+; SM60-FTZ-NEXT: ret;
%retval = atomicrmw fmax ptr addrspace(1) %addr, half %val syncscope("block") acq_rel
ret half %retval
}
define half @fminimum_acq_rel_half_global_cta(ptr addrspace(1) %addr, half %val) {
-; SM60-LABEL: fminimum_acq_rel_half_global_cta(
-; SM60: {
-; SM60-NEXT: .reg .pred %p<7>;
-; SM60-NEXT: .reg .b16 %rs<9>;
-; SM60-NEXT: .reg .b32 %r<15>;
-; SM60-NEXT: .reg .b64 %rd<3>;
-; SM60-EMPTY:
-; SM60-NEXT: // %bb.0:
-; SM60-NEXT: ld.param.b16 %rs1, [fminimum_acq_rel_half_global_cta_param_1];
-; SM60-NEXT: ld.param.b64 %rd2, [fminimum_acq_rel_half_global_cta_param_0];
-; SM60-NEXT: membar.cta;
-; SM60-NEXT: and.b64 %rd1, %rd2, -4;
-; SM60-NEXT: cvt.u32.u64 %r4, %rd2;
-; SM60-NEXT: and.b32 %r5, %r4, 3;
-; SM60-NEXT: shl.b32 %r1, %r5, 3;
-; SM60-NEXT: mov.b32 %r6, 65535;
-; SM60-NEXT: shl.b32 %r7, %r6, %r1;
-; SM60-NEXT: not.b32 %r2, %r7;
-; SM60-NEXT: ld.volatile.global.b32 %r14, [%rd1];
-; SM60-NEXT: setp.eq.b16 %p4, %rs1, -32768;
-; SM60-NEXT: $L__BB76_1: // %atomicrmw.start
-; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: shr.u32 %r8, %r14, %r1;
-; SM60-NEXT: cvt.u16.u32 %rs2, %r8;
-; SM60-NEXT: setp.lt.f16 %p1, %rs2, %rs1;
-; SM60-NEXT: selp.b16 %rs3, %rs2, %rs1, %p1;
-; SM60-NEXT: setp.nan.f16 %p2, %rs2, %rs1;
-; SM60-NEXT: selp.b16 %rs4, 0x7E00, %rs3, %p2;
-; SM60-NEXT: setp.eq.b16 %p3, %rs2, -32768;
-; SM60-NEXT: selp.b16 %rs5, %rs2, %rs4, %p3;
-; SM60-NEXT: selp.b16 %rs6, %rs1, %rs5, %p4;
-; SM60-NEXT: mov.b16 %rs7, 0x0000;
-; SM60-NEXT: setp.eq.f16 %p5, %rs4, %rs7;
-; SM60-NEXT: selp.b16 %rs8, %rs6, %rs4, %p5;
-; SM60-NEXT: cvt.u32.u16 %r9, %rs8;
-; SM60-NEXT: shl.b32 %r10, %r9, %r1;
-; SM60-NEXT: and.b32 %r11, %r14, %r2;
-; SM60-NEXT: or.b32 %r12, %r11, %r10;
-; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
-; SM60-NEXT: setp.ne.b32 %p6, %r3, %r14;
-; SM60-NEXT: mov.b32 %r14, %r3;
-; SM60-NEXT: @%p6 bra $L__BB76_1;
-; SM60-NEXT: // %bb.2: // %atomicrmw.end
-; SM60-NEXT: shr.u32 %r13, %r3, %r1;
-; SM60-NEXT: membar.cta;
-; SM60-NEXT: st.param.b16 [func_retval0], %r13;
-; SM60-NEXT: ret;
+; SM60-NOFTZ-LABEL: fminimum_acq_rel_half_global_cta(
+; SM60-NOFTZ: {
+; SM60-NOFTZ-NEXT: .reg .pred %p<7>;
+; SM60-NOFTZ-NEXT: .reg .b16 %rs<9>;
+; SM60-NOFTZ-NEXT: .reg .b32 %r<15>;
+; SM60-NOFTZ-NEXT: .reg .b64 %rd<3>;
+; SM60-NOFTZ-EMPTY:
+; SM60-NOFTZ-NEXT: // %bb.0:
+; SM60-NOFTZ-NEXT: ld.param.b16 %rs1, [fminimum_acq_rel_half_global_cta_param_1];
+; SM60-NOFTZ-NEXT: ld.param.b64 %rd2, [fminimum_acq_rel_half_global_cta_param_0];
+; SM60-NOFTZ-NEXT: membar.cta;
+; SM60-NOFTZ-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NOFTZ-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-NOFTZ-NEXT: and.b32 %r5, %r4, 3;
+; SM60-NOFTZ-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-NOFTZ-NEXT: mov.b32 %r6, 65535;
+; SM60-NOFTZ-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-NOFTZ-NEXT: not.b32 %r2, %r7;
+; SM60-NOFTZ-NEXT: ld.volatile.global.b32 %r14, [%rd1];
+; SM60-NOFTZ-NEXT: setp.eq.b16 %p4, %rs1, -32768;
+; SM60-NOFTZ-NEXT: $L__BB76_1: // %atomicrmw.start
+; SM60-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NOFTZ-NEXT: shr.u32 %r8, %r14, %r1;
+; SM60-NOFTZ-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM60-NOFTZ-NEXT: setp.lt.f16 %p1, %rs2, %rs1;
+; SM60-NOFTZ-NEXT: selp.b16 %rs3, %rs2, %rs1, %p1;
+; SM60-NOFTZ-NEXT: setp.nan.f16 %p2, %rs2, %rs1;
+; SM60-NOFTZ-NEXT: selp.b16 %rs4, 0x7E00, %rs3, %p2;
+; SM60-NOFTZ-NEXT: setp.eq.b16 %p3, %rs2, -32768;
+; SM60-NOFTZ-NEXT: selp.b16 %rs5, %rs2, %rs4, %p3;
+; SM60-NOFTZ-NEXT: selp.b16 %rs6, %rs1, %rs5, %p4;
+; SM60-NOFTZ-NEXT: mov.b16 %rs7, 0x0000;
+; SM60-NOFTZ-NEXT: setp.eq.f16 %p5, %rs4, %rs7;
+; SM60-NOFTZ-NEXT: selp.b16 %rs8, %rs6, %rs4, %p5;
+; SM60-NOFTZ-NEXT: cvt.u32.u16 %r9, %rs8;
+; SM60-NOFTZ-NEXT: shl.b32 %r10, %r9, %r1;
+; SM60-NOFTZ-NEXT: and.b32 %r11, %r14, %r2;
+; SM60-NOFTZ-NEXT: or.b32 %r12, %r11, %r10;
+; SM60-NOFTZ-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM60-NOFTZ-NEXT: setp.ne.b32 %p6, %r3, %r14;
+; SM60-NOFTZ-NEXT: mov.b32 %r14, %r3;
+; SM60-NOFTZ-NEXT: @%p6 bra $L__BB76_1;
+; SM60-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NOFTZ-NEXT: shr.u32 %r13, %r3, %r1;
+; SM60-NOFTZ-NEXT: membar.cta;
+; SM60-NOFTZ-NEXT: st.param.b16 [func_retval0], %r13;
+; SM60-NOFTZ-NEXT: ret;
+;
+; SM60-FTZ-LABEL: fminimum_acq_rel_half_global_cta(
+; SM60-FTZ: {
+; SM60-FTZ-NEXT: .reg .pred %p<7>;
+; SM60-FTZ-NEXT: .reg .b16 %rs<9>;
+; SM60-FTZ-NEXT: .reg .b32 %r<15>;
+; SM60-FTZ-NEXT: .reg .b64 %rd<3>;
+; SM60-FTZ-EMPTY:
+; SM60-FTZ-NEXT: // %bb.0:
+; SM60-FTZ-NEXT: ld.param.b16 %rs1, [fminimum_acq_rel_half_global_cta_param_1];
+; SM60-FTZ-NEXT: ld.param.b64 %rd2, [fminimum_acq_rel_half_global_cta_param_0];
+; SM60-FTZ-NEXT: membar.cta;
+; SM60-FTZ-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-FTZ-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-FTZ-NEXT: and.b32 %r5, %r4, 3;
+; SM60-FTZ-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-FTZ-NEXT: mov.b32 %r6, 65535;
+; SM60-FTZ-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-FTZ-NEXT: not.b32 %r2, %r7;
+; SM60-FTZ-NEXT: ld.volatile.global.b32 %r14, [%rd1];
+; SM60-FTZ-NEXT: setp.eq.b16 %p4, %rs1, -32768;
+; SM60-FTZ-NEXT: $L__BB76_1: // %atomicrmw.start
+; SM60-FTZ-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-FTZ-NEXT: shr.u32 %r8, %r14, %r1;
+; SM60-FTZ-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM60-FTZ-NEXT: setp.lt.ftz.f16 %p1, %rs2, %rs1;
+; SM60-FTZ-NEXT: selp.b16 %rs3, %rs2, %rs1, %p1;
+; SM60-FTZ-NEXT: setp.nan.ftz.f16 %p2, %rs2, %rs1;
+; SM60-FTZ-NEXT: selp.b16 %rs4, 0x7E00, %rs3, %p2;
+; SM60-FTZ-NEXT: setp.eq.b16 %p3, %rs2, -32768;
+; SM60-FTZ-NEXT: selp.b16 %rs5, %rs2, %rs4, %p3;
+; SM60-FTZ-NEXT: selp.b16 %rs6, %rs1, %rs5, %p4;
+; SM60-FTZ-NEXT: mov.b16 %rs7, 0x0000;
+; SM60-FTZ-NEXT: setp.eq.ftz.f16 %p5, %rs4, %rs7;
+; SM60-FTZ-NEXT: selp.b16 %rs8, %rs6, %rs4, %p5;
+; SM60-FTZ-NEXT: cvt.u32.u16 %r9, %rs8;
+; SM60-FTZ-NEXT: shl.b32 %r10, %r9, %r1;
+; SM60-FTZ-NEXT: and.b32 %r11, %r14, %r2;
+; SM60-FTZ-NEXT: or.b32 %r12, %r11, %r10;
+; SM60-FTZ-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM60-FTZ-NEXT: setp.ne.b32 %p6, %r3, %r14;
+; SM60-FTZ-NEXT: mov.b32 %r14, %r3;
+; SM60-FTZ-NEXT: @%p6 bra $L__BB76_1;
+; SM60-FTZ-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-FTZ-NEXT: shr.u32 %r13, %r3, %r1;
+; SM60-FTZ-NEXT: membar.cta;
+; SM60-FTZ-NEXT: st.param.b16 [func_retval0], %r13;
+; SM60-FTZ-NEXT: ret;
%retval = atomicrmw fminimum ptr addrspace(1) %addr, half %val syncscope("block") acq_rel
ret half %retval
}
define half @fmaximum_acq_rel_half_global_cta(ptr addrspace(1) %addr, half %val) {
-; SM60-LABEL: fmaximum_acq_rel_half_global_cta(
-; SM60: {
-; SM60-NEXT: .reg .pred %p<7>;
-; SM60-NEXT: .reg .b16 %rs<9>;
-; SM60-NEXT: .reg .b32 %r<15>;
-; SM60-NEXT: .reg .b64 %rd<3>;
-; SM60-EMPTY:
-; SM60-NEXT: // %bb.0:
-; SM60-NEXT: ld.param.b16 %rs1, [fmaximum_acq_rel_half_global_cta_param_1];
-; SM60-NEXT: ld.param.b64 %rd2, [fmaximum_acq_rel_half_global_cta_param_0];
-; SM60-NEXT: membar.cta;
-; SM60-NEXT: and.b64 %rd1, %rd2, -4;
-; SM60-NEXT: cvt.u32.u64 %r4, %rd2;
-; SM60-NEXT: and.b32 %r5, %r4, 3;
-; SM60-NEXT: shl.b32 %r1, %r5, 3;
-; SM60-NEXT: mov.b32 %r6, 65535;
-; SM60-NEXT: shl.b32 %r7, %r6, %r1;
-; SM60-NEXT: not.b32 %r2, %r7;
-; SM60-NEXT: ld.volatile.global.b32 %r14, [%rd1];
-; SM60-NEXT: setp.eq.b16 %p4, %rs1, 0;
-; SM60-NEXT: $L__BB77_1: // %atomicrmw.start
-; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: shr.u32 %r8, %r14, %r1;
-; SM60-NEXT: cvt.u16.u32 %rs2, %r8;
-; SM60-NEXT: setp.gt.f16 %p1, %rs2, %rs1;
-; SM60-NEXT: selp.b16 %rs3, %rs2, %rs1, %p1;
-; SM60-NEXT: setp.nan.f16 %p2, %rs2, %rs1;
-; SM60-NEXT: selp.b16 %rs4, 0x7E00, %rs3, %p2;
-; SM60-NEXT: setp.eq.b16 %p3, %rs2, 0;
-; SM60-NEXT: selp.b16 %rs5, %rs2, %rs4, %p3;
-; SM60-NEXT: selp.b16 %rs6, %rs1, %rs5, %p4;
-; SM60-NEXT: mov.b16 %rs7, 0x0000;
-; SM60-NEXT: setp.eq.f16 %p5, %rs4, %rs7;
-; SM60-NEXT: selp.b16 %rs8, %rs6, %rs4, %p5;
-; SM60-NEXT: cvt.u32.u16 %r9, %rs8;
-; SM60-NEXT: shl.b32 %r10, %r9, %r1;
-; SM60-NEXT: and.b32 %r11, %r14, %r2;
-; SM60-NEXT: or.b32 %r12, %r11, %r10;
-; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
-; SM60-NEXT: setp.ne.b32 %p6, %r3, %r14;
-; SM60-NEXT: mov.b32 %r14, %r3;
-; SM60-NEXT: @%p6 bra $L__BB77_1;
-; SM60-NEXT: // %bb.2: // %atomicrmw.end
-; SM60-NEXT: shr.u32 %r13, %r3, %r1;
-; SM60-NEXT: membar.cta;
-; SM60-NEXT: st.param.b16 [func_retval0], %r13;
-; SM60-NEXT: ret;
+; SM60-NOFTZ-LABEL: fmaximum_acq_rel_half_global_cta(
+; SM60-NOFTZ: {
+; SM60-NOFTZ-NEXT: .reg .pred %p<7>;
+; SM60-NOFTZ-NEXT: .reg .b16 %rs<9>;
+; SM60-NOFTZ-NEXT: .reg .b32 %r<15>;
+; SM60-NOFTZ-NEXT: .reg .b64 %rd<3>;
+; SM60-NOFTZ-EMPTY:
+; SM60-NOFTZ-NEXT: // %bb.0:
+; SM60-NOFTZ-NEXT: ld.param.b16 %rs1, [fmaximum_acq_rel_half_global_cta_param_1];
+; SM60-NOFTZ-NEXT: ld.param.b64 %rd2, [fmaximum_acq_rel_half_global_cta_param_0];
+; SM60-NOFTZ-NEXT: membar.cta;
+; SM60-NOFTZ-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NOFTZ-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-NOFTZ-NEXT: and.b32 %r5, %r4, 3;
+; SM60-NOFTZ-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-NOFTZ-NEXT: mov.b32 %r6, 65535;
+; SM60-NOFTZ-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-NOFTZ-NEXT: not.b32 %r2, %r7;
+; SM60-NOFTZ-NEXT: ld.volatile.global.b32 %r14, [%rd1];
+; SM60-NOFTZ-NEXT: setp.eq.b16 %p4, %rs1, 0;
+; SM60-NOFTZ-NEXT: $L__BB77_1: // %atomicrmw.start
+; SM60-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NOFTZ-NEXT: shr.u32 %r8, %r14, %r1;
+; SM60-NOFTZ-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM60-NOFTZ-NEXT: setp.gt.f16 %p1, %rs2, %rs1;
+; SM60-NOFTZ-NEXT: selp.b16 %rs3, %rs2, %rs1, %p1;
+; SM60-NOFTZ-NEXT: setp.nan.f16 %p2, %rs2, %rs1;
+; SM60-NOFTZ-NEXT: selp.b16 %rs4, 0x7E00, %rs3, %p2;
+; SM60-NOFTZ-NEXT: setp.eq.b16 %p3, %rs2, 0;
+; SM60-NOFTZ-NEXT: selp.b16 %rs5, %rs2, %rs4, %p3;
+; SM60-NOFTZ-NEXT: selp.b16 %rs6, %rs1, %rs5, %p4;
+; SM60-NOFTZ-NEXT: mov.b16 %rs7, 0x0000;
+; SM60-NOFTZ-NEXT: setp.eq.f16 %p5, %rs4, %rs7;
+; SM60-NOFTZ-NEXT: selp.b16 %rs8, %rs6, %rs4, %p5;
+; SM60-NOFTZ-NEXT: cvt.u32.u16 %r9, %rs8;
+; SM60-NOFTZ-NEXT: shl.b32 %r10, %r9, %r1;
+; SM60-NOFTZ-NEXT: and.b32 %r11, %r14, %r2;
+; SM60-NOFTZ-NEXT: or.b32 %r12, %r11, %r10;
+; SM60-NOFTZ-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM60-NOFTZ-NEXT: setp.ne.b32 %p6, %r3, %r14;
+; SM60-NOFTZ-NEXT: mov.b32 %r14, %r3;
+; SM60-NOFTZ-NEXT: @%p6 bra $L__BB77_1;
+; SM60-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NOFTZ-NEXT: shr.u32 %r13, %r3, %r1;
+; SM60-NOFTZ-NEXT: membar.cta;
+; SM60-NOFTZ-NEXT: st.param.b16 [func_retval0], %r13;
+; SM60-NOFTZ-NEXT: ret;
+;
+; SM60-FTZ-LABEL: fmaximum_acq_rel_half_global_cta(
+; SM60-FTZ: {
+; SM60-FTZ-NEXT: .reg .pred %p<7>;
+; SM60-FTZ-NEXT: .reg .b16 %rs<9>;
+; SM60-FTZ-NEXT: .reg .b32 %r<15>;
+; SM60-FTZ-NEXT: .reg .b64 %rd<3>;
+; SM60-FTZ-EMPTY:
+; SM60-FTZ-NEXT: // %bb.0:
+; SM60-FTZ-NEXT: ld.param.b16 %rs1, [fmaximum_acq_rel_half_global_cta_param_1];
+; SM60-FTZ-NEXT: ld.param.b64 %rd2, [fmaximum_acq_rel_half_global_cta_param_0];
+; SM60-FTZ-NEXT: membar.cta;
+; SM60-FTZ-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-FTZ-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-FTZ-NEXT: and.b32 %r5, %r4, 3;
+; SM60-FTZ-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-FTZ-NEXT: mov.b32 %r6, 65535;
+; SM60-FTZ-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-FTZ-NEXT: not.b32 %r2, %r7;
+; SM60-FTZ-NEXT: ld.volatile.global.b32 %r14, [%rd1];
+; SM60-FTZ-NEXT: setp.eq.b16 %p4, %rs1, 0;
+; SM60-FTZ-NEXT: $L__BB77_1: // %atomicrmw.start
+; SM60-FTZ-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-FTZ-NEXT: shr.u32 %r8, %r14, %r1;
+; SM60-FTZ-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM60-FTZ-NEXT: setp.gt.ftz.f16 %p1, %rs2, %rs1;
+; SM60-FTZ-NEXT: selp.b16 %rs3, %rs2, %rs1, %p1;
+; SM60-FTZ-NEXT: setp.nan.ftz.f16 %p2, %rs2, %rs1;
+; SM60-FTZ-NEXT: selp.b16 %rs4, 0x7E00, %rs3, %p2;
+; SM60-FTZ-NEXT: setp.eq.b16 %p3, %rs2, 0;
+; SM60-FTZ-NEXT: selp.b16 %rs5, %rs2, %rs4, %p3;
+; SM60-FTZ-NEXT: selp.b16 %rs6, %rs1, %rs5, %p4;
+; SM60-FTZ-NEXT: mov.b16 %rs7, 0x0000;
+; SM60-FTZ-NEXT: setp.eq.ftz.f16 %p5, %rs4, %rs7;
+; SM60-FTZ-NEXT: selp.b16 %rs8, %rs6, %rs4, %p5;
+; SM60-FTZ-NEXT: cvt.u32.u16 %r9, %rs8;
+; SM60-FTZ-NEXT: shl.b32 %r10, %r9, %r1;
+; SM60-FTZ-NEXT: and.b32 %r11, %r14, %r2;
+; SM60-FTZ-NEXT: or.b32 %r12, %r11, %r10;
+; SM60-FTZ-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM60-FTZ-NEXT: setp.ne.b32 %p6, %r3, %r14;
+; SM60-FTZ-NEXT: mov.b32 %r14, %r3;
+; SM60-FTZ-NEXT: @%p6 bra $L__BB77_1;
+; SM60-FTZ-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-FTZ-NEXT: shr.u32 %r13, %r3, %r1;
+; SM60-FTZ-NEXT: membar.cta;
+; SM60-FTZ-NEXT: st.param.b16 [func_retval0], %r13;
+; SM60-FTZ-NEXT: ret;
%retval = atomicrmw fmaximum ptr addrspace(1) %addr, half %val syncscope("block") acq_rel
ret half %retval
}
define bfloat @fadd_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat %val) {
-; SM60-LABEL: fadd_acq_rel_bfloat_global_cta(
-; SM60: {
-; SM60-NEXT: .reg .pred %p<3>;
-; SM60-NEXT: .reg .b16 %rs<2>;
-; SM60-NEXT: .reg .b32 %r<24>;
-; SM60-NEXT: .reg .b64 %rd<3>;
-; SM60-EMPTY:
-; SM60-NEXT: // %bb.0:
-; SM60-NEXT: ld.param.b16 %rs1, [fadd_acq_rel_bfloat_global_cta_param_1];
-; SM60-NEXT: ld.param.b64 %rd2, [fadd_acq_rel_bfloat_global_cta_param_0];
-; SM60-NEXT: membar.cta;
-; SM60-NEXT: and.b64 %rd1, %rd2, -4;
-; SM60-NEXT: cvt.u32.u64 %r4, %rd2;
-; SM60-NEXT: and.b32 %r5, %r4, 3;
-; SM60-NEXT: shl.b32 %r1, %r5, 3;
-; SM60-NEXT: mov.b32 %r6, 65535;
-; SM60-NEXT: shl.b32 %r7, %r6, %r1;
-; SM60-NEXT: not.b32 %r2, %r7;
-; SM60-NEXT: ld.volatile.global.b32 %r23, [%rd1];
-; SM60-NEXT: cvt.u32.u16 %r10, %rs1;
-; SM60-NEXT: shl.b32 %r11, %r10, 16;
-; SM60-NEXT: $L__BB78_1: // %atomicrmw.start
-; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: shr.u32 %r8, %r23, %r1;
-; SM60-NEXT: shl.b32 %r9, %r8, 16;
-; SM60-NEXT: add.rn.f32 %r12, %r9, %r11;
-; SM60-NEXT: bfe.u32 %r13, %r12, 16, 1;
-; SM60-NEXT: add.s32 %r14, %r13, %r12;
-; SM60-NEXT: add.s32 %r15, %r14, 32767;
-; SM60-NEXT: setp.nan.f32 %p1, %r12, %r12;
-; SM60-NEXT: or.b32 %r16, %r12, 4194304;
-; SM60-NEXT: selp.b32 %r17, %r16, %r15, %p1;
-; SM60-NEXT: shr.u32 %r18, %r17, 16;
-; SM60-NEXT: shl.b32 %r19, %r18, %r1;
-; SM60-NEXT: and.b32 %r20, %r23, %r2;
-; SM60-NEXT: or.b32 %r21, %r20, %r19;
-; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r23, %r21;
-; SM60-NEXT: setp.ne.b32 %p2, %r3, %r23;
-; SM60-NEXT: mov.b32 %r23, %r3;
-; SM60-NEXT: @%p2 bra $L__BB78_1;
-; SM60-NEXT: // %bb.2: // %atomicrmw.end
-; SM60-NEXT: shr.u32 %r22, %r3, %r1;
-; SM60-NEXT: membar.cta;
-; SM60-NEXT: st.param.b16 [func_retval0], %r22;
-; SM60-NEXT: ret;
+; SM60-NOFTZ-LABEL: fadd_acq_rel_bfloat_global_cta(
+; SM60-NOFTZ: {
+; SM60-NOFTZ-NEXT: .reg .pred %p<3>;
+; SM60-NOFTZ-NEXT: .reg .b16 %rs<2>;
+; SM60-NOFTZ-NEXT: .reg .b32 %r<24>;
+; SM60-NOFTZ-NEXT: .reg .b64 %rd<3>;
+; SM60-NOFTZ-EMPTY:
+; SM60-NOFTZ-NEXT: // %bb.0:
+; SM60-NOFTZ-NEXT: ld.param.b16 %rs1, [fadd_acq_rel_bfloat_global_cta_param_1];
+; SM60-NOFTZ-NEXT: ld.param.b64 %rd2, [fadd_acq_rel_bfloat_global_cta_param_0];
+; SM60-NOFTZ-NEXT: membar.cta;
+; SM60-NOFTZ-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NOFTZ-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-NOFTZ-NEXT: and.b32 %r5, %r4, 3;
+; SM60-NOFTZ-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-NOFTZ-NEXT: mov.b32 %r6, 65535;
+; SM60-NOFTZ-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-NOFTZ-NEXT: not.b32 %r2, %r7;
+; SM60-NOFTZ-NEXT: ld.volatile.global.b32 %r23, [%rd1];
+; SM60-NOFTZ-NEXT: cvt.u32.u16 %r10, %rs1;
+; SM60-NOFTZ-NEXT: shl.b32 %r11, %r10, 16;
+; SM60-NOFTZ-NEXT: $L__BB78_1: // %atomicrmw.start
+; SM60-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NOFTZ-NEXT: shr.u32 %r8, %r23, %r1;
+; SM60-NOFTZ-NEXT: shl.b32 %r9, %r8, 16;
+; SM60-NOFTZ-NEXT: add.rn.f32 %r12, %r9, %r11;
+; SM60-NOFTZ-NEXT: bfe.u32 %r13, %r12, 16, 1;
+; SM60-NOFTZ-NEXT: add.s32 %r14, %r13, %r12;
+; SM60-NOFTZ-NEXT: add.s32 %r15, %r14, 32767;
+; SM60-NOFTZ-NEXT: setp.nan.f32 %p1, %r12, %r12;
+; SM60-NOFTZ-NEXT: or.b32 %r16, %r12, 4194304;
+; SM60-NOFTZ-NEXT: selp.b32 %r17, %r16, %r15, %p1;
+; SM60-NOFTZ-NEXT: shr.u32 %r18, %r17, 16;
+; SM60-NOFTZ-NEXT: shl.b32 %r19, %r18, %r1;
+; SM60-NOFTZ-NEXT: and.b32 %r20, %r23, %r2;
+; SM60-NOFTZ-NEXT: or.b32 %r21, %r20, %r19;
+; SM60-NOFTZ-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r23, %r21;
+; SM60-NOFTZ-NEXT: setp.ne.b32 %p2, %r3, %r23;
+; SM60-NOFTZ-NEXT: mov.b32 %r23, %r3;
+; SM60-NOFTZ-NEXT: @%p2 bra $L__BB78_1;
+; SM60-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NOFTZ-NEXT: shr.u32 %r22, %r3, %r1;
+; SM60-NOFTZ-NEXT: membar.cta;
+; SM60-NOFTZ-NEXT: st.param.b16 [func_retval0], %r22;
+; SM60-NOFTZ-NEXT: ret;
+;
+; SM60-FTZ-LABEL: fadd_acq_rel_bfloat_global_cta(
+; SM60-FTZ: {
+; SM60-FTZ-NEXT: .reg .pred %p<3>;
+; SM60-FTZ-NEXT: .reg .b16 %rs<2>;
+; SM60-FTZ-NEXT: .reg .b32 %r<24>;
+; SM60-FTZ-NEXT: .reg .b64 %rd<3>;
+; SM60-FTZ-EMPTY:
+; SM60-FTZ-NEXT: // %bb.0:
+; SM60-FTZ-NEXT: ld.param.b16 %rs1, [fadd_acq_rel_bfloat_global_cta_param_1];
+; SM60-FTZ-NEXT: ld.param.b64 %rd2, [fadd_acq_rel_bfloat_global_cta_param_0];
+; SM60-FTZ-NEXT: membar.cta;
+; SM60-FTZ-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-FTZ-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-FTZ-NEXT: and.b32 %r5, %r4, 3;
+; SM60-FTZ-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-FTZ-NEXT: mov.b32 %r6, 65535;
+; SM60-FTZ-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-FTZ-NEXT: not.b32 %r2, %r7;
+; SM60-FTZ-NEXT: ld.volatile.global.b32 %r23, [%rd1];
+; SM60-FTZ-NEXT: cvt.u32.u16 %r10, %rs1;
+; SM60-FTZ-NEXT: shl.b32 %r11, %r10, 16;
+; SM60-FTZ-NEXT: $L__BB78_1: // %atomicrmw.start
+; SM60-FTZ-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-FTZ-NEXT: shr.u32 %r8, %r23, %r1;
+; SM60-FTZ-NEXT: shl.b32 %r9, %r8, 16;
+; SM60-FTZ-NEXT: add.rn.ftz.f32 %r12, %r9, %r11;
+; SM60-FTZ-NEXT: bfe.u32 %r13, %r12, 16, 1;
+; SM60-FTZ-NEXT: add.s32 %r14, %r13, %r12;
+; SM60-FTZ-NEXT: add.s32 %r15, %r14, 32767;
+; SM60-FTZ-NEXT: setp.nan.ftz.f32 %p1, %r12, %r12;
+; SM60-FTZ-NEXT: or.b32 %r16, %r12, 4194304;
+; SM60-FTZ-NEXT: selp.b32 %r17, %r16, %r15, %p1;
+; SM60-FTZ-NEXT: shr.u32 %r18, %r17, 16;
+; SM60-FTZ-NEXT: shl.b32 %r19, %r18, %r1;
+; SM60-FTZ-NEXT: and.b32 %r20, %r23, %r2;
+; SM60-FTZ-NEXT: or.b32 %r21, %r20, %r19;
+; SM60-FTZ-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r23, %r21;
+; SM60-FTZ-NEXT: setp.ne.b32 %p2, %r3, %r23;
+; SM60-FTZ-NEXT: mov.b32 %r23, %r3;
+; SM60-FTZ-NEXT: @%p2 bra $L__BB78_1;
+; SM60-FTZ-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-FTZ-NEXT: shr.u32 %r22, %r3, %r1;
+; SM60-FTZ-NEXT: membar.cta;
+; SM60-FTZ-NEXT: st.param.b16 [func_retval0], %r22;
+; SM60-FTZ-NEXT: ret;
%retval = atomicrmw fadd ptr addrspace(1) %addr, bfloat %val syncscope("block") acq_rel
ret bfloat %retval
}
define bfloat @fsub_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat %val) {
-; SM60-LABEL: fsub_acq_rel_bfloat_global_cta(
-; SM60: {
-; SM60-NEXT: .reg .pred %p<3>;
-; SM60-NEXT: .reg .b16 %rs<2>;
-; SM60-NEXT: .reg .b32 %r<24>;
-; SM60-NEXT: .reg .b64 %rd<3>;
-; SM60-EMPTY:
-; SM60-NEXT: // %bb.0:
-; SM60-NEXT: ld.param.b16 %rs1, [fsub_acq_rel_bfloat_global_cta_param_1];
-; SM60-NEXT: ld.param.b64 %rd2, [fsub_acq_rel_bfloat_global_cta_param_0];
-; SM60-NEXT: membar.cta;
-; SM60-NEXT: and.b64 %rd1, %rd2, -4;
-; SM60-NEXT: cvt.u32.u64 %r4, %rd2;
-; SM60-NEXT: and.b32 %r5, %r4, 3;
-; SM60-NEXT: shl.b32 %r1, %r5, 3;
-; SM60-NEXT: mov.b32 %r6, 65535;
-; SM60-NEXT: shl.b32 %r7, %r6, %r1;
-; SM60-NEXT: not.b32 %r2, %r7;
-; SM60-NEXT: ld.volatile.global.b32 %r23, [%rd1];
-; SM60-NEXT: cvt.u32.u16 %r10, %rs1;
-; SM60-NEXT: shl.b32 %r11, %r10, 16;
-; SM60-NEXT: $L__BB79_1: // %atomicrmw.start
-; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: shr.u32 %r8, %r23, %r1;
-; SM60-NEXT: shl.b32 %r9, %r8, 16;
-; SM60-NEXT: sub.rn.f32 %r12, %r9, %r11;
-; SM60-NEXT: bfe.u32 %r13, %r12, 16, 1;
-; SM60-NEXT: add.s32 %r14, %r13, %r12;
-; SM60-NEXT: add.s32 %r15, %r14, 32767;
-; SM60-NEXT: setp.nan.f32 %p1, %r12, %r12;
-; SM60-NEXT: or.b32 %r16, %r12, 4194304;
-; SM60-NEXT: selp.b32 %r17, %r16, %r15, %p1;
-; SM60-NEXT: shr.u32 %r18, %r17, 16;
-; SM60-NEXT: shl.b32 %r19, %r18, %r1;
-; SM60-NEXT: and.b32 %r20, %r23, %r2;
-; SM60-NEXT: or.b32 %r21, %r20, %r19;
-; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r23, %r21;
-; SM60-NEXT: setp.ne.b32 %p2, %r3, %r23;
-; SM60-NEXT: mov.b32 %r23, %r3;
-; SM60-NEXT: @%p2 bra $L__BB79_1;
-; SM60-NEXT: // %bb.2: // %atomicrmw.end
-; SM60-NEXT: shr.u32 %r22, %r3, %r1;
-; SM60-NEXT: membar.cta;
-; SM60-NEXT: st.param.b16 [func_retval0], %r22;
-; SM60-NEXT: ret;
+; SM60-NOFTZ-LABEL: fsub_acq_rel_bfloat_global_cta(
+; SM60-NOFTZ: {
+; SM60-NOFTZ-NEXT: .reg .pred %p<3>;
+; SM60-NOFTZ-NEXT: .reg .b16 %rs<2>;
+; SM60-NOFTZ-NEXT: .reg .b32 %r<24>;
+; SM60-NOFTZ-NEXT: .reg .b64 %rd<3>;
+; SM60-NOFTZ-EMPTY:
+; SM60-NOFTZ-NEXT: // %bb.0:
+; SM60-NOFTZ-NEXT: ld.param.b16 %rs1, [fsub_acq_rel_bfloat_global_cta_param_1];
+; SM60-NOFTZ-NEXT: ld.param.b64 %rd2, [fsub_acq_rel_bfloat_global_cta_param_0];
+; SM60-NOFTZ-NEXT: membar.cta;
+; SM60-NOFTZ-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NOFTZ-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-NOFTZ-NEXT: and.b32 %r5, %r4, 3;
+; SM60-NOFTZ-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-NOFTZ-NEXT: mov.b32 %r6, 65535;
+; SM60-NOFTZ-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-NOFTZ-NEXT: not.b32 %r2, %r7;
+; SM60-NOFTZ-NEXT: ld.volatile.global.b32 %r23, [%rd1];
+; SM60-NOFTZ-NEXT: cvt.u32.u16 %r10, %rs1;
+; SM60-NOFTZ-NEXT: shl.b32 %r11, %r10, 16;
+; SM60-NOFTZ-NEXT: $L__BB79_1: // %atomicrmw.start
+; SM60-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NOFTZ-NEXT: shr.u32 %r8, %r23, %r1;
+; SM60-NOFTZ-NEXT: shl.b32 %r9, %r8, 16;
+; SM60-NOFTZ-NEXT: sub.rn.f32 %r12, %r9, %r11;
+; SM60-NOFTZ-NEXT: bfe.u32 %r13, %r12, 16, 1;
+; SM60-NOFTZ-NEXT: add.s32 %r14, %r13, %r12;
+; SM60-NOFTZ-NEXT: add.s32 %r15, %r14, 32767;
+; SM60-NOFTZ-NEXT: setp.nan.f32 %p1, %r12, %r12;
+; SM60-NOFTZ-NEXT: or.b32 %r16, %r12, 4194304;
+; SM60-NOFTZ-NEXT: selp.b32 %r17, %r16, %r15, %p1;
+; SM60-NOFTZ-NEXT: shr.u32 %r18, %r17, 16;
+; SM60-NOFTZ-NEXT: shl.b32 %r19, %r18, %r1;
+; SM60-NOFTZ-NEXT: and.b32 %r20, %r23, %r2;
+; SM60-NOFTZ-NEXT: or.b32 %r21, %r20, %r19;
+; SM60-NOFTZ-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r23, %r21;
+; SM60-NOFTZ-NEXT: setp.ne.b32 %p2, %r3, %r23;
+; SM60-NOFTZ-NEXT: mov.b32 %r23, %r3;
+; SM60-NOFTZ-NEXT: @%p2 bra $L__BB79_1;
+; SM60-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NOFTZ-NEXT: shr.u32 %r22, %r3, %r1;
+; SM60-NOFTZ-NEXT: membar.cta;
+; SM60-NOFTZ-NEXT: st.param.b16 [func_retval0], %r22;
+; SM60-NOFTZ-NEXT: ret;
+;
+; SM60-FTZ-LABEL: fsub_acq_rel_bfloat_global_cta(
+; SM60-FTZ: {
+; SM60-FTZ-NEXT: .reg .pred %p<3>;
+; SM60-FTZ-NEXT: .reg .b16 %rs<2>;
+; SM60-FTZ-NEXT: .reg .b32 %r<24>;
+; SM60-FTZ-NEXT: .reg .b64 %rd<3>;
+; SM60-FTZ-EMPTY:
+; SM60-FTZ-NEXT: // %bb.0:
+; SM60-FTZ-NEXT: ld.param.b16 %rs1, [fsub_acq_rel_bfloat_global_cta_param_1];
+; SM60-FTZ-NEXT: ld.param.b64 %rd2, [fsub_acq_rel_bfloat_global_cta_param_0];
+; SM60-FTZ-NEXT: membar.cta;
+; SM60-FTZ-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-FTZ-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-FTZ-NEXT: and.b32 %r5, %r4, 3;
+; SM60-FTZ-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-FTZ-NEXT: mov.b32 %r6, 65535;
+; SM60-FTZ-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-FTZ-NEXT: not.b32 %r2, %r7;
+; SM60-FTZ-NEXT: ld.volatile.global.b32 %r23, [%rd1];
+; SM60-FTZ-NEXT: cvt.u32.u16 %r10, %rs1;
+; SM60-FTZ-NEXT: shl.b32 %r11, %r10, 16;
+; SM60-FTZ-NEXT: $L__BB79_1: // %atomicrmw.start
+; SM60-FTZ-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-FTZ-NEXT: shr.u32 %r8, %r23, %r1;
+; SM60-FTZ-NEXT: shl.b32 %r9, %r8, 16;
+; SM60-FTZ-NEXT: sub.rn.ftz.f32 %r12, %r9, %r11;
+; SM60-FTZ-NEXT: bfe.u32 %r13, %r12, 16, 1;
+; SM60-FTZ-NEXT: add.s32 %r14, %r13, %r12;
+; SM60-FTZ-NEXT: add.s32 %r15, %r14, 32767;
+; SM60-FTZ-NEXT: setp.nan.ftz.f32 %p1, %r12, %r12;
+; SM60-FTZ-NEXT: or.b32 %r16, %r12, 4194304;
+; SM60-FTZ-NEXT: selp.b32 %r17, %r16, %r15, %p1;
+; SM60-FTZ-NEXT: shr.u32 %r18, %r17, 16;
+; SM60-FTZ-NEXT: shl.b32 %r19, %r18, %r1;
+; SM60-FTZ-NEXT: and.b32 %r20, %r23, %r2;
+; SM60-FTZ-NEXT: or.b32 %r21, %r20, %r19;
+; SM60-FTZ-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r23, %r21;
+; SM60-FTZ-NEXT: setp.ne.b32 %p2, %r3, %r23;
+; SM60-FTZ-NEXT: mov.b32 %r23, %r3;
+; SM60-FTZ-NEXT: @%p2 bra $L__BB79_1;
+; SM60-FTZ-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-FTZ-NEXT: shr.u32 %r22, %r3, %r1;
+; SM60-FTZ-NEXT: membar.cta;
+; SM60-FTZ-NEXT: st.param.b16 [func_retval0], %r22;
+; SM60-FTZ-NEXT: ret;
%retval = atomicrmw fsub ptr addrspace(1) %addr, bfloat %val syncscope("block") acq_rel
ret bfloat %retval
}
define bfloat @fmin_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat %val) {
-; SM60-LABEL: fmin_acq_rel_bfloat_global_cta(
-; SM60: {
-; SM60-NEXT: .reg .pred %p<3>;
-; SM60-NEXT: .reg .b16 %rs<2>;
-; SM60-NEXT: .reg .b32 %r<24>;
-; SM60-NEXT: .reg .b64 %rd<3>;
-; SM60-EMPTY:
-; SM60-NEXT: // %bb.0:
-; SM60-NEXT: ld.param.b16 %rs1, [fmin_acq_rel_bfloat_global_cta_param_1];
-; SM60-NEXT: ld.param.b64 %rd2, [fmin_acq_rel_bfloat_global_cta_param_0];
-; SM60-NEXT: membar.cta;
-; SM60-NEXT: and.b64 %rd1, %rd2, -4;
-; SM60-NEXT: cvt.u32.u64 %r4, %rd2;
-; SM60-NEXT: and.b32 %r5, %r4, 3;
-; SM60-NEXT: shl.b32 %r1, %r5, 3;
-; SM60-NEXT: mov.b32 %r6, 65535;
-; SM60-NEXT: shl.b32 %r7, %r6, %r1;
-; SM60-NEXT: not.b32 %r2, %r7;
-; SM60-NEXT: ld.volatile.global.b32 %r23, [%rd1];
-; SM60-NEXT: cvt.u32.u16 %r10, %rs1;
-; SM60-NEXT: shl.b32 %r11, %r10, 16;
-; SM60-NEXT: $L__BB80_1: // %atomicrmw.start
-; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: shr.u32 %r8, %r23, %r1;
-; SM60-NEXT: shl.b32 %r9, %r8, 16;
-; SM60-NEXT: min.f32 %r12, %r9, %r11;
-; SM60-NEXT: bfe.u32 %r13, %r12, 16, 1;
-; SM60-NEXT: add.s32 %r14, %r13, %r12;
-; SM60-NEXT: add.s32 %r15, %r14, 32767;
-; SM60-NEXT: setp.nan.f32 %p1, %r12, %r12;
-; SM60-NEXT: or.b32 %r16, %r12, 4194304;
-; SM60-NEXT: selp.b32 %r17, %r16, %r15, %p1;
-; SM60-NEXT: shr.u32 %r18, %r17, 16;
-; SM60-NEXT: shl.b32 %r19, %r18, %r1;
-; SM60-NEXT: and.b32 %r20, %r23, %r2;
-; SM60-NEXT: or.b32 %r21, %r20, %r19;
-; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r23, %r21;
-; SM60-NEXT: setp.ne.b32 %p2, %r3, %r23;
-; SM60-NEXT: mov.b32 %r23, %r3;
-; SM60-NEXT: @%p2 bra $L__BB80_1;
-; SM60-NEXT: // %bb.2: // %atomicrmw.end
-; SM60-NEXT: shr.u32 %r22, %r3, %r1;
-; SM60-NEXT: membar.cta;
-; SM60-NEXT: st.param.b16 [func_retval0], %r22;
-; SM60-NEXT: ret;
+; SM60-NOFTZ-LABEL: fmin_acq_rel_bfloat_global_cta(
+; SM60-NOFTZ: {
+; SM60-NOFTZ-NEXT: .reg .pred %p<3>;
+; SM60-NOFTZ-NEXT: .reg .b16 %rs<2>;
+; SM60-NOFTZ-NEXT: .reg .b32 %r<24>;
+; SM60-NOFTZ-NEXT: .reg .b64 %rd<3>;
+; SM60-NOFTZ-EMPTY:
+; SM60-NOFTZ-NEXT: // %bb.0:
+; SM60-NOFTZ-NEXT: ld.param.b16 %rs1, [fmin_acq_rel_bfloat_global_cta_param_1];
+; SM60-NOFTZ-NEXT: ld.param.b64 %rd2, [fmin_acq_rel_bfloat_global_cta_param_0];
+; SM60-NOFTZ-NEXT: membar.cta;
+; SM60-NOFTZ-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NOFTZ-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-NOFTZ-NEXT: and.b32 %r5, %r4, 3;
+; SM60-NOFTZ-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-NOFTZ-NEXT: mov.b32 %r6, 65535;
+; SM60-NOFTZ-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-NOFTZ-NEXT: not.b32 %r2, %r7;
+; SM60-NOFTZ-NEXT: ld.volatile.global.b32 %r23, [%rd1];
+; SM60-NOFTZ-NEXT: cvt.u32.u16 %r10, %rs1;
+; SM60-NOFTZ-NEXT: shl.b32 %r11, %r10, 16;
+; SM60-NOFTZ-NEXT: $L__BB80_1: // %atomicrmw.start
+; SM60-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NOFTZ-NEXT: shr.u32 %r8, %r23, %r1;
+; SM60-NOFTZ-NEXT: shl.b32 %r9, %r8, 16;
+; SM60-NOFTZ-NEXT: min.f32 %r12, %r9, %r11;
+; SM60-NOFTZ-NEXT: bfe.u32 %r13, %r12, 16, 1;
+; SM60-NOFTZ-NEXT: add.s32 %r14, %r13, %r12;
+; SM60-NOFTZ-NEXT: add.s32 %r15, %r14, 32767;
+; SM60-NOFTZ-NEXT: setp.nan.f32 %p1, %r12, %r12;
+; SM60-NOFTZ-NEXT: or.b32 %r16, %r12, 4194304;
+; SM60-NOFTZ-NEXT: selp.b32 %r17, %r16, %r15, %p1;
+; SM60-NOFTZ-NEXT: shr.u32 %r18, %r17, 16;
+; SM60-NOFTZ-NEXT: shl.b32 %r19, %r18, %r1;
+; SM60-NOFTZ-NEXT: and.b32 %r20, %r23, %r2;
+; SM60-NOFTZ-NEXT: or.b32 %r21, %r20, %r19;
+; SM60-NOFTZ-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r23, %r21;
+; SM60-NOFTZ-NEXT: setp.ne.b32 %p2, %r3, %r23;
+; SM60-NOFTZ-NEXT: mov.b32 %r23, %r3;
+; SM60-NOFTZ-NEXT: @%p2 bra $L__BB80_1;
+; SM60-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NOFTZ-NEXT: shr.u32 %r22, %r3, %r1;
+; SM60-NOFTZ-NEXT: membar.cta;
+; SM60-NOFTZ-NEXT: st.param.b16 [func_retval0], %r22;
+; SM60-NOFTZ-NEXT: ret;
+;
+; SM60-FTZ-LABEL: fmin_acq_rel_bfloat_global_cta(
+; SM60-FTZ: {
+; SM60-FTZ-NEXT: .reg .pred %p<3>;
+; SM60-FTZ-NEXT: .reg .b16 %rs<2>;
+; SM60-FTZ-NEXT: .reg .b32 %r<24>;
+; SM60-FTZ-NEXT: .reg .b64 %rd<3>;
+; SM60-FTZ-EMPTY:
+; SM60-FTZ-NEXT: // %bb.0:
+; SM60-FTZ-NEXT: ld.param.b16 %rs1, [fmin_acq_rel_bfloat_global_cta_param_1];
+; SM60-FTZ-NEXT: ld.param.b64 %rd2, [fmin_acq_rel_bfloat_global_cta_param_0];
+; SM60-FTZ-NEXT: membar.cta;
+; SM60-FTZ-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-FTZ-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-FTZ-NEXT: and.b32 %r5, %r4, 3;
+; SM60-FTZ-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-FTZ-NEXT: mov.b32 %r6, 65535;
+; SM60-FTZ-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-FTZ-NEXT: not.b32 %r2, %r7;
+; SM60-FTZ-NEXT: ld.volatile.global.b32 %r23, [%rd1];
+; SM60-FTZ-NEXT: cvt.u32.u16 %r10, %rs1;
+; SM60-FTZ-NEXT: shl.b32 %r11, %r10, 16;
+; SM60-FTZ-NEXT: $L__BB80_1: // %atomicrmw.start
+; SM60-FTZ-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-FTZ-NEXT: shr.u32 %r8, %r23, %r1;
+; SM60-FTZ-NEXT: shl.b32 %r9, %r8, 16;
+; SM60-FTZ-NEXT: min.ftz.f32 %r12, %r9, %r11;
+; SM60-FTZ-NEXT: bfe.u32 %r13, %r12, 16, 1;
+; SM60-FTZ-NEXT: add.s32 %r14, %r13, %r12;
+; SM60-FTZ-NEXT: add.s32 %r15, %r14, 32767;
+; SM60-FTZ-NEXT: setp.nan.ftz.f32 %p1, %r12, %r12;
+; SM60-FTZ-NEXT: or.b32 %r16, %r12, 4194304;
+; SM60-FTZ-NEXT: selp.b32 %r17, %r16, %r15, %p1;
+; SM60-FTZ-NEXT: shr.u32 %r18, %r17, 16;
+; SM60-FTZ-NEXT: shl.b32 %r19, %r18, %r1;
+; SM60-FTZ-NEXT: and.b32 %r20, %r23, %r2;
+; SM60-FTZ-NEXT: or.b32 %r21, %r20, %r19;
+; SM60-FTZ-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r23, %r21;
+; SM60-FTZ-NEXT: setp.ne.b32 %p2, %r3, %r23;
+; SM60-FTZ-NEXT: mov.b32 %r23, %r3;
+; SM60-FTZ-NEXT: @%p2 bra $L__BB80_1;
+; SM60-FTZ-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-FTZ-NEXT: shr.u32 %r22, %r3, %r1;
+; SM60-FTZ-NEXT: membar.cta;
+; SM60-FTZ-NEXT: st.param.b16 [func_retval0], %r22;
+; SM60-FTZ-NEXT: ret;
%retval = atomicrmw fmin ptr addrspace(1) %addr, bfloat %val syncscope("block") acq_rel
ret bfloat %retval
}
define bfloat @fmax_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat %val) {
-; SM60-LABEL: fmax_acq_rel_bfloat_global_cta(
-; SM60: {
-; SM60-NEXT: .reg .pred %p<3>;
-; SM60-NEXT: .reg .b16 %rs<2>;
-; SM60-NEXT: .reg .b32 %r<24>;
-; SM60-NEXT: .reg .b64 %rd<3>;
-; SM60-EMPTY:
-; SM60-NEXT: // %bb.0:
-; SM60-NEXT: ld.param.b16 %rs1, [fmax_acq_rel_bfloat_global_cta_param_1];
-; SM60-NEXT: ld.param.b64 %rd2, [fmax_acq_rel_bfloat_global_cta_param_0];
-; SM60-NEXT: membar.cta;
-; SM60-NEXT: and.b64 %rd1, %rd2, -4;
-; SM60-NEXT: cvt.u32.u64 %r4, %rd2;
-; SM60-NEXT: and.b32 %r5, %r4, 3;
-; SM60-NEXT: shl.b32 %r1, %r5, 3;
-; SM60-NEXT: mov.b32 %r6, 65535;
-; SM60-NEXT: shl.b32 %r7, %r6, %r1;
-; SM60-NEXT: not.b32 %r2, %r7;
-; SM60-NEXT: ld.volatile.global.b32 %r23, [%rd1];
-; SM60-NEXT: cvt.u32.u16 %r10, %rs1;
-; SM60-NEXT: shl.b32 %r11, %r10, 16;
-; SM60-NEXT: $L__BB81_1: // %atomicrmw.start
-; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: shr.u32 %r8, %r23, %r1;
-; SM60-NEXT: shl.b32 %r9, %r8, 16;
-; SM60-NEXT: max.f32 %r12, %r9, %r11;
-; SM60-NEXT: bfe.u32 %r13, %r12, 16, 1;
-; SM60-NEXT: add.s32 %r14, %r13, %r12;
-; SM60-NEXT: add.s32 %r15, %r14, 32767;
-; SM60-NEXT: setp.nan.f32 %p1, %r12, %r12;
-; SM60-NEXT: or.b32 %r16, %r12, 4194304;
-; SM60-NEXT: selp.b32 %r17, %r16, %r15, %p1;
-; SM60-NEXT: shr.u32 %r18, %r17, 16;
-; SM60-NEXT: shl.b32 %r19, %r18, %r1;
-; SM60-NEXT: and.b32 %r20, %r23, %r2;
-; SM60-NEXT: or.b32 %r21, %r20, %r19;
-; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r23, %r21;
-; SM60-NEXT: setp.ne.b32 %p2, %r3, %r23;
-; SM60-NEXT: mov.b32 %r23, %r3;
-; SM60-NEXT: @%p2 bra $L__BB81_1;
-; SM60-NEXT: // %bb.2: // %atomicrmw.end
-; SM60-NEXT: shr.u32 %r22, %r3, %r1;
-; SM60-NEXT: membar.cta;
-; SM60-NEXT: st.param.b16 [func_retval0], %r22;
-; SM60-NEXT: ret;
+; SM60-NOFTZ-LABEL: fmax_acq_rel_bfloat_global_cta(
+; SM60-NOFTZ: {
+; SM60-NOFTZ-NEXT: .reg .pred %p<3>;
+; SM60-NOFTZ-NEXT: .reg .b16 %rs<2>;
+; SM60-NOFTZ-NEXT: .reg .b32 %r<24>;
+; SM60-NOFTZ-NEXT: .reg .b64 %rd<3>;
+; SM60-NOFTZ-EMPTY:
+; SM60-NOFTZ-NEXT: // %bb.0:
+; SM60-NOFTZ-NEXT: ld.param.b16 %rs1, [fmax_acq_rel_bfloat_global_cta_param_1];
+; SM60-NOFTZ-NEXT: ld.param.b64 %rd2, [fmax_acq_rel_bfloat_global_cta_param_0];
+; SM60-NOFTZ-NEXT: membar.cta;
+; SM60-NOFTZ-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NOFTZ-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-NOFTZ-NEXT: and.b32 %r5, %r4, 3;
+; SM60-NOFTZ-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-NOFTZ-NEXT: mov.b32 %r6, 65535;
+; SM60-NOFTZ-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-NOFTZ-NEXT: not.b32 %r2, %r7;
+; SM60-NOFTZ-NEXT: ld.volatile.global.b32 %r23, [%rd1];
+; SM60-NOFTZ-NEXT: cvt.u32.u16 %r10, %rs1;
+; SM60-NOFTZ-NEXT: shl.b32 %r11, %r10, 16;
+; SM60-NOFTZ-NEXT: $L__BB81_1: // %atomicrmw.start
+; SM60-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NOFTZ-NEXT: shr.u32 %r8, %r23, %r1;
+; SM60-NOFTZ-NEXT: shl.b32 %r9, %r8, 16;
+; SM60-NOFTZ-NEXT: max.f32 %r12, %r9, %r11;
+; SM60-NOFTZ-NEXT: bfe.u32 %r13, %r12, 16, 1;
+; SM60-NOFTZ-NEXT: add.s32 %r14, %r13, %r12;
+; SM60-NOFTZ-NEXT: add.s32 %r15, %r14, 32767;
+; SM60-NOFTZ-NEXT: setp.nan.f32 %p1, %r12, %r12;
+; SM60-NOFTZ-NEXT: or.b32 %r16, %r12, 4194304;
+; SM60-NOFTZ-NEXT: selp.b32 %r17, %r16, %r15, %p1;
+; SM60-NOFTZ-NEXT: shr.u32 %r18, %r17, 16;
+; SM60-NOFTZ-NEXT: shl.b32 %r19, %r18, %r1;
+; SM60-NOFTZ-NEXT: and.b32 %r20, %r23, %r2;
+; SM60-NOFTZ-NEXT: or.b32 %r21, %r20, %r19;
+; SM60-NOFTZ-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r23, %r21;
+; SM60-NOFTZ-NEXT: setp.ne.b32 %p2, %r3, %r23;
+; SM60-NOFTZ-NEXT: mov.b32 %r23, %r3;
+; SM60-NOFTZ-NEXT: @%p2 bra $L__BB81_1;
+; SM60-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NOFTZ-NEXT: shr.u32 %r22, %r3, %r1;
+; SM60-NOFTZ-NEXT: membar.cta;
+; SM60-NOFTZ-NEXT: st.param.b16 [func_retval0], %r22;
+; SM60-NOFTZ-NEXT: ret;
+;
+; SM60-FTZ-LABEL: fmax_acq_rel_bfloat_global_cta(
+; SM60-FTZ: {
+; SM60-FTZ-NEXT: .reg .pred %p<3>;
+; SM60-FTZ-NEXT: .reg .b16 %rs<2>;
+; SM60-FTZ-NEXT: .reg .b32 %r<24>;
+; SM60-FTZ-NEXT: .reg .b64 %rd<3>;
+; SM60-FTZ-EMPTY:
+; SM60-FTZ-NEXT: // %bb.0:
+; SM60-FTZ-NEXT: ld.param.b16 %rs1, [fmax_acq_rel_bfloat_global_cta_param_1];
+; SM60-FTZ-NEXT: ld.param.b64 %rd2, [fmax_acq_rel_bfloat_global_cta_param_0];
+; SM60-FTZ-NEXT: membar.cta;
+; SM60-FTZ-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-FTZ-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-FTZ-NEXT: and.b32 %r5, %r4, 3;
+; SM60-FTZ-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-FTZ-NEXT: mov.b32 %r6, 65535;
+; SM60-FTZ-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-FTZ-NEXT: not.b32 %r2, %r7;
+; SM60-FTZ-NEXT: ld.volatile.global.b32 %r23, [%rd1];
+; SM60-FTZ-NEXT: cvt.u32.u16 %r10, %rs1;
+; SM60-FTZ-NEXT: shl.b32 %r11, %r10, 16;
+; SM60-FTZ-NEXT: $L__BB81_1: // %atomicrmw.start
+; SM60-FTZ-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-FTZ-NEXT: shr.u32 %r8, %r23, %r1;
+; SM60-FTZ-NEXT: shl.b32 %r9, %r8, 16;
+; SM60-FTZ-NEXT: max.ftz.f32 %r12, %r9, %r11;
+; SM60-FTZ-NEXT: bfe.u32 %r13, %r12, 16, 1;
+; SM60-FTZ-NEXT: add.s32 %r14, %r13, %r12;
+; SM60-FTZ-NEXT: add.s32 %r15, %r14, 32767;
+; SM60-FTZ-NEXT: setp.nan.ftz.f32 %p1, %r12, %r12;
+; SM60-FTZ-NEXT: or.b32 %r16, %r12, 4194304;
+; SM60-FTZ-NEXT: selp.b32 %r17, %r16, %r15, %p1;
+; SM60-FTZ-NEXT: shr.u32 %r18, %r17, 16;
+; SM60-FTZ-NEXT: shl.b32 %r19, %r18, %r1;
+; SM60-FTZ-NEXT: and.b32 %r20, %r23, %r2;
+; SM60-FTZ-NEXT: or.b32 %r21, %r20, %r19;
+; SM60-FTZ-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r23, %r21;
+; SM60-FTZ-NEXT: setp.ne.b32 %p2, %r3, %r23;
+; SM60-FTZ-NEXT: mov.b32 %r23, %r3;
+; SM60-FTZ-NEXT: @%p2 bra $L__BB81_1;
+; SM60-FTZ-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-FTZ-NEXT: shr.u32 %r22, %r3, %r1;
+; SM60-FTZ-NEXT: membar.cta;
+; SM60-FTZ-NEXT: st.param.b16 [func_retval0], %r22;
+; SM60-FTZ-NEXT: ret;
%retval = atomicrmw fmax ptr addrspace(1) %addr, bfloat %val syncscope("block") acq_rel
ret bfloat %retval
}
define bfloat @fminimum_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat %val) {
-; SM60-LABEL: fminimum_acq_rel_bfloat_global_cta(
-; SM60: {
-; SM60-NEXT: .reg .pred %p<7>;
-; SM60-NEXT: .reg .b16 %rs<8>;
-; SM60-NEXT: .reg .b32 %r<20>;
-; SM60-NEXT: .reg .b64 %rd<3>;
-; SM60-EMPTY:
-; SM60-NEXT: // %bb.0:
-; SM60-NEXT: ld.param.b16 %rs1, [fminimum_acq_rel_bfloat_global_cta_param_1];
-; SM60-NEXT: ld.param.b64 %rd2, [fminimum_acq_rel_bfloat_global_cta_param_0];
-; SM60-NEXT: membar.cta;
-; SM60-NEXT: and.b64 %rd1, %rd2, -4;
-; SM60-NEXT: cvt.u32.u64 %r4, %rd2;
-; SM60-NEXT: and.b32 %r5, %r4, 3;
-; SM60-NEXT: shl.b32 %r1, %r5, 3;
-; SM60-NEXT: mov.b32 %r6, 65535;
-; SM60-NEXT: shl.b32 %r7, %r6, %r1;
-; SM60-NEXT: not.b32 %r2, %r7;
-; SM60-NEXT: ld.volatile.global.b32 %r19, [%rd1];
-; SM60-NEXT: cvt.u32.u16 %r10, %rs1;
-; SM60-NEXT: shl.b32 %r11, %r10, 16;
-; SM60-NEXT: setp.eq.b16 %p4, %rs1, -32768;
-; SM60-NEXT: $L__BB82_1: // %atomicrmw.start
-; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: shr.u32 %r8, %r19, %r1;
-; SM60-NEXT: cvt.u16.u32 %rs2, %r8;
-; SM60-NEXT: shl.b32 %r9, %r8, 16;
-; SM60-NEXT: setp.lt.f32 %p1, %r9, %r11;
-; SM60-NEXT: selp.b16 %rs3, %rs2, %rs1, %p1;
-; SM60-NEXT: setp.nan.f32 %p2, %r9, %r11;
-; SM60-NEXT: selp.b16 %rs4, 0x7FC0, %rs3, %p2;
-; SM60-NEXT: setp.eq.b16 %p3, %rs2, -32768;
-; SM60-NEXT: selp.b16 %rs5, %rs2, %rs4, %p3;
-; SM60-NEXT: selp.b16 %rs6, %rs1, %rs5, %p4;
-; SM60-NEXT: cvt.u32.u16 %r12, %rs4;
-; SM60-NEXT: shl.b32 %r13, %r12, 16;
-; SM60-NEXT: setp.eq.f32 %p5, %r13, 0f00000000;
-; SM60-NEXT: selp.b16 %rs7, %rs6, %rs4, %p5;
-; SM60-NEXT: cvt.u32.u16 %r14, %rs7;
-; SM60-NEXT: shl.b32 %r15, %r14, %r1;
-; SM60-NEXT: and.b32 %r16, %r19, %r2;
-; SM60-NEXT: or.b32 %r17, %r16, %r15;
-; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r19, %r17;
-; SM60-NEXT: setp.ne.b32 %p6, %r3, %r19;
-; SM60-NEXT: mov.b32 %r19, %r3;
-; SM60-NEXT: @%p6 bra $L__BB82_1;
-; SM60-NEXT: // %bb.2: // %atomicrmw.end
-; SM60-NEXT: shr.u32 %r18, %r3, %r1;
-; SM60-NEXT: membar.cta;
-; SM60-NEXT: st.param.b16 [func_retval0], %r18;
-; SM60-NEXT: ret;
+; SM60-NOFTZ-LABEL: fminimum_acq_rel_bfloat_global_cta(
+; SM60-NOFTZ: {
+; SM60-NOFTZ-NEXT: .reg .pred %p<7>;
+; SM60-NOFTZ-NEXT: .reg .b16 %rs<8>;
+; SM60-NOFTZ-NEXT: .reg .b32 %r<20>;
+; SM60-NOFTZ-NEXT: .reg .b64 %rd<3>;
+; SM60-NOFTZ-EMPTY:
+; SM60-NOFTZ-NEXT: // %bb.0:
+; SM60-NOFTZ-NEXT: ld.param.b16 %rs1, [fminimum_acq_rel_bfloat_global_cta_param_1];
+; SM60-NOFTZ-NEXT: ld.param.b64 %rd2, [fminimum_acq_rel_bfloat_global_cta_param_0];
+; SM60-NOFTZ-NEXT: membar.cta;
+; SM60-NOFTZ-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NOFTZ-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-NOFTZ-NEXT: and.b32 %r5, %r4, 3;
+; SM60-NOFTZ-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-NOFTZ-NEXT: mov.b32 %r6, 65535;
+; SM60-NOFTZ-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-NOFTZ-NEXT: not.b32 %r2, %r7;
+; SM60-NOFTZ-NEXT: ld.volatile.global.b32 %r19, [%rd1];
+; SM60-NOFTZ-NEXT: cvt.u32.u16 %r10, %rs1;
+; SM60-NOFTZ-NEXT: shl.b32 %r11, %r10, 16;
+; SM60-NOFTZ-NEXT: setp.eq.b16 %p4, %rs1, -32768;
+; SM60-NOFTZ-NEXT: $L__BB82_1: // %atomicrmw.start
+; SM60-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NOFTZ-NEXT: shr.u32 %r8, %r19, %r1;
+; SM60-NOFTZ-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM60-NOFTZ-NEXT: shl.b32 %r9, %r8, 16;
+; SM60-NOFTZ-NEXT: setp.lt.f32 %p1, %r9, %r11;
+; SM60-NOFTZ-NEXT: selp.b16 %rs3, %rs2, %rs1, %p1;
+; SM60-NOFTZ-NEXT: setp.nan.f32 %p2, %r9, %r11;
+; SM60-NOFTZ-NEXT: selp.b16 %rs4, 0x7FC0, %rs3, %p2;
+; SM60-NOFTZ-NEXT: setp.eq.b16 %p3, %rs2, -32768;
+; SM60-NOFTZ-NEXT: selp.b16 %rs5, %rs2, %rs4, %p3;
+; SM60-NOFTZ-NEXT: selp.b16 %rs6, %rs1, %rs5, %p4;
+; SM60-NOFTZ-NEXT: cvt.u32.u16 %r12, %rs4;
+; SM60-NOFTZ-NEXT: shl.b32 %r13, %r12, 16;
+; SM60-NOFTZ-NEXT: setp.eq.f32 %p5, %r13, 0f00000000;
+; SM60-NOFTZ-NEXT: selp.b16 %rs7, %rs6, %rs4, %p5;
+; SM60-NOFTZ-NEXT: cvt.u32.u16 %r14, %rs7;
+; SM60-NOFTZ-NEXT: shl.b32 %r15, %r14, %r1;
+; SM60-NOFTZ-NEXT: and.b32 %r16, %r19, %r2;
+; SM60-NOFTZ-NEXT: or.b32 %r17, %r16, %r15;
+; SM60-NOFTZ-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r19, %r17;
+; SM60-NOFTZ-NEXT: setp.ne.b32 %p6, %r3, %r19;
+; SM60-NOFTZ-NEXT: mov.b32 %r19, %r3;
+; SM60-NOFTZ-NEXT: @%p6 bra $L__BB82_1;
+; SM60-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NOFTZ-NEXT: shr.u32 %r18, %r3, %r1;
+; SM60-NOFTZ-NEXT: membar.cta;
+; SM60-NOFTZ-NEXT: st.param.b16 [func_retval0], %r18;
+; SM60-NOFTZ-NEXT: ret;
+;
+; SM60-FTZ-LABEL: fminimum_acq_rel_bfloat_global_cta(
+; SM60-FTZ: {
+; SM60-FTZ-NEXT: .reg .pred %p<7>;
+; SM60-FTZ-NEXT: .reg .b16 %rs<8>;
+; SM60-FTZ-NEXT: .reg .b32 %r<20>;
+; SM60-FTZ-NEXT: .reg .b64 %rd<3>;
+; SM60-FTZ-EMPTY:
+; SM60-FTZ-NEXT: // %bb.0:
+; SM60-FTZ-NEXT: ld.param.b16 %rs1, [fminimum_acq_rel_bfloat_global_cta_param_1];
+; SM60-FTZ-NEXT: ld.param.b64 %rd2, [fminimum_acq_rel_bfloat_global_cta_param_0];
+; SM60-FTZ-NEXT: membar.cta;
+; SM60-FTZ-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-FTZ-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-FTZ-NEXT: and.b32 %r5, %r4, 3;
+; SM60-FTZ-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-FTZ-NEXT: mov.b32 %r6, 65535;
+; SM60-FTZ-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-FTZ-NEXT: not.b32 %r2, %r7;
+; SM60-FTZ-NEXT: ld.volatile.global.b32 %r19, [%rd1];
+; SM60-FTZ-NEXT: cvt.u32.u16 %r10, %rs1;
+; SM60-FTZ-NEXT: shl.b32 %r11, %r10, 16;
+; SM60-FTZ-NEXT: setp.eq.b16 %p4, %rs1, -32768;
+; SM60-FTZ-NEXT: $L__BB82_1: // %atomicrmw.start
+; SM60-FTZ-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-FTZ-NEXT: shr.u32 %r8, %r19, %r1;
+; SM60-FTZ-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM60-FTZ-NEXT: shl.b32 %r9, %r8, 16;
+; SM60-FTZ-NEXT: setp.lt.ftz.f32 %p1, %r9, %r11;
+; SM60-FTZ-NEXT: selp.b16 %rs3, %rs2, %rs1, %p1;
+; SM60-FTZ-NEXT: setp.nan.ftz.f32 %p2, %r9, %r11;
+; SM60-FTZ-NEXT: selp.b16 %rs4, 0x7FC0, %rs3, %p2;
+; SM60-FTZ-NEXT: setp.eq.b16 %p3, %rs2, -32768;
+; SM60-FTZ-NEXT: selp.b16 %rs5, %rs2, %rs4, %p3;
+; SM60-FTZ-NEXT: selp.b16 %rs6, %rs1, %rs5, %p4;
+; SM60-FTZ-NEXT: cvt.u32.u16 %r12, %rs4;
+; SM60-FTZ-NEXT: shl.b32 %r13, %r12, 16;
+; SM60-FTZ-NEXT: setp.eq.ftz.f32 %p5, %r13, 0f00000000;
+; SM60-FTZ-NEXT: selp.b16 %rs7, %rs6, %rs4, %p5;
+; SM60-FTZ-NEXT: cvt.u32.u16 %r14, %rs7;
+; SM60-FTZ-NEXT: shl.b32 %r15, %r14, %r1;
+; SM60-FTZ-NEXT: and.b32 %r16, %r19, %r2;
+; SM60-FTZ-NEXT: or.b32 %r17, %r16, %r15;
+; SM60-FTZ-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r19, %r17;
+; SM60-FTZ-NEXT: setp.ne.b32 %p6, %r3, %r19;
+; SM60-FTZ-NEXT: mov.b32 %r19, %r3;
+; SM60-FTZ-NEXT: @%p6 bra $L__BB82_1;
+; SM60-FTZ-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-FTZ-NEXT: shr.u32 %r18, %r3, %r1;
+; SM60-FTZ-NEXT: membar.cta;
+; SM60-FTZ-NEXT: st.param.b16 [func_retval0], %r18;
+; SM60-FTZ-NEXT: ret;
%retval = atomicrmw fminimum ptr addrspace(1) %addr, bfloat %val syncscope("block") acq_rel
ret bfloat %retval
}
define bfloat @fmaximum_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat %val) {
-; SM60-LABEL: fmaximum_acq_rel_bfloat_global_cta(
-; SM60: {
-; SM60-NEXT: .reg .pred %p<7>;
-; SM60-NEXT: .reg .b16 %rs<8>;
-; SM60-NEXT: .reg .b32 %r<20>;
-; SM60-NEXT: .reg .b64 %rd<3>;
-; SM60-EMPTY:
-; SM60-NEXT: // %bb.0:
-; SM60-NEXT: ld.param.b16 %rs1, [fmaximum_acq_rel_bfloat_global_cta_param_1];
-; SM60-NEXT: ld.param.b64 %rd2, [fmaximum_acq_rel_bfloat_global_cta_param_0];
-; SM60-NEXT: membar.cta;
-; SM60-NEXT: and.b64 %rd1, %rd2, -4;
-; SM60-NEXT: cvt.u32.u64 %r4, %rd2;
-; SM60-NEXT: and.b32 %r5, %r4, 3;
-; SM60-NEXT: shl.b32 %r1, %r5, 3;
-; SM60-NEXT: mov.b32 %r6, 65535;
-; SM60-NEXT: shl.b32 %r7, %r6, %r1;
-; SM60-NEXT: not.b32 %r2, %r7;
-; SM60-NEXT: ld.volatile.global.b32 %r19, [%rd1];
-; SM60-NEXT: cvt.u32.u16 %r10, %rs1;
-; SM60-NEXT: shl.b32 %r11, %r10, 16;
-; SM60-NEXT: setp.eq.b16 %p4, %rs1, 0;
-; SM60-NEXT: $L__BB83_1: // %atomicrmw.start
-; SM60-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NEXT: shr.u32 %r8, %r19, %r1;
-; SM60-NEXT: cvt.u16.u32 %rs2, %r8;
-; SM60-NEXT: shl.b32 %r9, %r8, 16;
-; SM60-NEXT: setp.gt.f32 %p1, %r9, %r11;
-; SM60-NEXT: selp.b16 %rs3, %rs2, %rs1, %p1;
-; SM60-NEXT: setp.nan.f32 %p2, %r9, %r11;
-; SM60-NEXT: selp.b16 %rs4, 0x7FC0, %rs3, %p2;
-; SM60-NEXT: setp.eq.b16 %p3, %rs2, 0;
-; SM60-NEXT: selp.b16 %rs5, %rs2, %rs4, %p3;
-; SM60-NEXT: selp.b16 %rs6, %rs1, %rs5, %p4;
-; SM60-NEXT: cvt.u32.u16 %r12, %rs4;
-; SM60-NEXT: shl.b32 %r13, %r12, 16;
-; SM60-NEXT: setp.eq.f32 %p5, %r13, 0f00000000;
-; SM60-NEXT: selp.b16 %rs7, %rs6, %rs4, %p5;
-; SM60-NEXT: cvt.u32.u16 %r14, %rs7;
-; SM60-NEXT: shl.b32 %r15, %r14, %r1;
-; SM60-NEXT: and.b32 %r16, %r19, %r2;
-; SM60-NEXT: or.b32 %r17, %r16, %r15;
-; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r19, %r17;
-; SM60-NEXT: setp.ne.b32 %p6, %r3, %r19;
-; SM60-NEXT: mov.b32 %r19, %r3;
-; SM60-NEXT: @%p6 bra $L__BB83_1;
-; SM60-NEXT: // %bb.2: // %atomicrmw.end
-; SM60-NEXT: shr.u32 %r18, %r3, %r1;
-; SM60-NEXT: membar.cta;
-; SM60-NEXT: st.param.b16 [func_retval0], %r18;
-; SM60-NEXT: ret;
+; SM60-NOFTZ-LABEL: fmaximum_acq_rel_bfloat_global_cta(
+; SM60-NOFTZ: {
+; SM60-NOFTZ-NEXT: .reg .pred %p<7>;
+; SM60-NOFTZ-NEXT: .reg .b16 %rs<8>;
+; SM60-NOFTZ-NEXT: .reg .b32 %r<20>;
+; SM60-NOFTZ-NEXT: .reg .b64 %rd<3>;
+; SM60-NOFTZ-EMPTY:
+; SM60-NOFTZ-NEXT: // %bb.0:
+; SM60-NOFTZ-NEXT: ld.param.b16 %rs1, [fmaximum_acq_rel_bfloat_global_cta_param_1];
+; SM60-NOFTZ-NEXT: ld.param.b64 %rd2, [fmaximum_acq_rel_bfloat_global_cta_param_0];
+; SM60-NOFTZ-NEXT: membar.cta;
+; SM60-NOFTZ-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NOFTZ-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-NOFTZ-NEXT: and.b32 %r5, %r4, 3;
+; SM60-NOFTZ-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-NOFTZ-NEXT: mov.b32 %r6, 65535;
+; SM60-NOFTZ-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-NOFTZ-NEXT: not.b32 %r2, %r7;
+; SM60-NOFTZ-NEXT: ld.volatile.global.b32 %r19, [%rd1];
+; SM60-NOFTZ-NEXT: cvt.u32.u16 %r10, %rs1;
+; SM60-NOFTZ-NEXT: shl.b32 %r11, %r10, 16;
+; SM60-NOFTZ-NEXT: setp.eq.b16 %p4, %rs1, 0;
+; SM60-NOFTZ-NEXT: $L__BB83_1: // %atomicrmw.start
+; SM60-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NOFTZ-NEXT: shr.u32 %r8, %r19, %r1;
+; SM60-NOFTZ-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM60-NOFTZ-NEXT: shl.b32 %r9, %r8, 16;
+; SM60-NOFTZ-NEXT: setp.gt.f32 %p1, %r9, %r11;
+; SM60-NOFTZ-NEXT: selp.b16 %rs3, %rs2, %rs1, %p1;
+; SM60-NOFTZ-NEXT: setp.nan.f32 %p2, %r9, %r11;
+; SM60-NOFTZ-NEXT: selp.b16 %rs4, 0x7FC0, %rs3, %p2;
+; SM60-NOFTZ-NEXT: setp.eq.b16 %p3, %rs2, 0;
+; SM60-NOFTZ-NEXT: selp.b16 %rs5, %rs2, %rs4, %p3;
+; SM60-NOFTZ-NEXT: selp.b16 %rs6, %rs1, %rs5, %p4;
+; SM60-NOFTZ-NEXT: cvt.u32.u16 %r12, %rs4;
+; SM60-NOFTZ-NEXT: shl.b32 %r13, %r12, 16;
+; SM60-NOFTZ-NEXT: setp.eq.f32 %p5, %r13, 0f00000000;
+; SM60-NOFTZ-NEXT: selp.b16 %rs7, %rs6, %rs4, %p5;
+; SM60-NOFTZ-NEXT: cvt.u32.u16 %r14, %rs7;
+; SM60-NOFTZ-NEXT: shl.b32 %r15, %r14, %r1;
+; SM60-NOFTZ-NEXT: and.b32 %r16, %r19, %r2;
+; SM60-NOFTZ-NEXT: or.b32 %r17, %r16, %r15;
+; SM60-NOFTZ-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r19, %r17;
+; SM60-NOFTZ-NEXT: setp.ne.b32 %p6, %r3, %r19;
+; SM60-NOFTZ-NEXT: mov.b32 %r19, %r3;
+; SM60-NOFTZ-NEXT: @%p6 bra $L__BB83_1;
+; SM60-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NOFTZ-NEXT: shr.u32 %r18, %r3, %r1;
+; SM60-NOFTZ-NEXT: membar.cta;
+; SM60-NOFTZ-NEXT: st.param.b16 [func_retval0], %r18;
+; SM60-NOFTZ-NEXT: ret;
+;
+; SM60-FTZ-LABEL: fmaximum_acq_rel_bfloat_global_cta(
+; SM60-FTZ: {
+; SM60-FTZ-NEXT: .reg .pred %p<7>;
+; SM60-FTZ-NEXT: .reg .b16 %rs<8>;
+; SM60-FTZ-NEXT: .reg .b32 %r<20>;
+; SM60-FTZ-NEXT: .reg .b64 %rd<3>;
+; SM60-FTZ-EMPTY:
+; SM60-FTZ-NEXT: // %bb.0:
+; SM60-FTZ-NEXT: ld.param.b16 %rs1, [fmaximum_acq_rel_bfloat_global_cta_param_1];
+; SM60-FTZ-NEXT: ld.param.b64 %rd2, [fmaximum_acq_rel_bfloat_global_cta_param_0];
+; SM60-FTZ-NEXT: membar.cta;
+; SM60-FTZ-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-FTZ-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-FTZ-NEXT: and.b32 %r5, %r4, 3;
+; SM60-FTZ-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-FTZ-NEXT: mov.b32 %r6, 65535;
+; SM60-FTZ-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-FTZ-NEXT: not.b32 %r2, %r7;
+; SM60-FTZ-NEXT: ld.volatile.global.b32 %r19, [%rd1];
+; SM60-FTZ-NEXT: cvt.u32.u16 %r10, %rs1;
+; SM60-FTZ-NEXT: shl.b32 %r11, %r10, 16;
+; SM60-FTZ-NEXT: setp.eq.b16 %p4, %rs1, 0;
+; SM60-FTZ-NEXT: $L__BB83_1: // %atomicrmw.start
+; SM60-FTZ-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-FTZ-NEXT: shr.u32 %r8, %r19, %r1;
+; SM60-FTZ-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM60-FTZ-NEXT: shl.b32 %r9, %r8, 16;
+; SM60-FTZ-NEXT: setp.gt.ftz.f32 %p1, %r9, %r11;
+; SM60-FTZ-NEXT: selp.b16 %rs3, %rs2, %rs1, %p1;
+; SM60-FTZ-NEXT: setp.nan.ftz.f32 %p2, %r9, %r11;
+; SM60-FTZ-NEXT: selp.b16 %rs4, 0x7FC0, %rs3, %p2;
+; SM60-FTZ-NEXT: setp.eq.b16 %p3, %rs2, 0;
+; SM60-FTZ-NEXT: selp.b16 %rs5, %rs2, %rs4, %p3;
+; SM60-FTZ-NEXT: selp.b16 %rs6, %rs1, %rs5, %p4;
+; SM60-FTZ-NEXT: cvt.u32.u16 %r12, %rs4;
+; SM60-FTZ-NEXT: shl.b32 %r13, %r12, 16;
+; SM60-FTZ-NEXT: setp.eq.ftz.f32 %p5, %r13, 0f00000000;
+; SM60-FTZ-NEXT: selp.b16 %rs7, %rs6, %rs4, %p5;
+; SM60-FTZ-NEXT: cvt.u32.u16 %r14, %rs7;
+; SM60-FTZ-NEXT: shl.b32 %r15, %r14, %r1;
+; SM60-FTZ-NEXT: and.b32 %r16, %r19, %r2;
+; SM60-FTZ-NEXT: or.b32 %r17, %r16, %r15;
+; SM60-FTZ-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r19, %r17;
+; SM60-FTZ-NEXT: setp.ne.b32 %p6, %r3, %r19;
+; SM60-FTZ-NEXT: mov.b32 %r19, %r3;
+; SM60-FTZ-NEXT: @%p6 bra $L__BB83_1;
+; SM60-FTZ-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-FTZ-NEXT: shr.u32 %r18, %r3, %r1;
+; SM60-FTZ-NEXT: membar.cta;
+; SM60-FTZ-NEXT: st.param.b16 [func_retval0], %r18;
+; SM60-FTZ-NEXT: ret;
%retval = atomicrmw fmaximum ptr addrspace(1) %addr, bfloat %val syncscope("block") acq_rel
ret bfloat %retval
}
@@ -3254,17 +3952,40 @@ define i64 @nand_acq_rel_i64_global_sys(ptr addrspace(1) %addr, i64 %val) {
}
define float @fadd_acq_rel_float_global_sys(ptr addrspace(1) %addr, float %val) {
-; SM60-LABEL: fadd_acq_rel_float_global_sys(
-; SM60: {
-; SM60-NEXT: .reg .b32 %r<3>;
-; SM60-NEXT: .reg .b64 %rd<2>;
-; SM60-EMPTY:
-; SM60-NEXT: // %bb.0:
-; SM60-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_sys_param_0];
-; SM60-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_global_sys_param_1];
-; SM60-NEXT: atom.sys.global.add.f32 %r2, [%rd1], %r1;
-; SM60-NEXT: st.param.b32 [func_retval0], %r2;
-; SM60-NEXT: ret;
+; SM60-NOFTZ-LABEL: fadd_acq_rel_float_global_sys(
+; SM60-NOFTZ: {
+; SM60-NOFTZ-NEXT: .reg .pred %p<2>;
+; SM60-NOFTZ-NEXT: .reg .b32 %r<5>;
+; SM60-NOFTZ-NEXT: .reg .b64 %rd<2>;
+; SM60-NOFTZ-EMPTY:
+; SM60-NOFTZ-NEXT: // %bb.0:
+; SM60-NOFTZ-NEXT: ld.param.b32 %r2, [fadd_acq_rel_float_global_sys_param_1];
+; SM60-NOFTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_sys_param_0];
+; SM60-NOFTZ-NEXT: membar.sys;
+; SM60-NOFTZ-NEXT: ld.volatile.global.b32 %r4, [%rd1];
+; SM60-NOFTZ-NEXT: $L__BB106_1: // %atomicrmw.start
+; SM60-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NOFTZ-NEXT: add.rn.f32 %r3, %r4, %r2;
+; SM60-NOFTZ-NEXT: atom.sys.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM60-NOFTZ-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM60-NOFTZ-NEXT: mov.b32 %r4, %r1;
+; SM60-NOFTZ-NEXT: @%p1 bra $L__BB106_1;
+; SM60-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NOFTZ-NEXT: membar.sys;
+; SM60-NOFTZ-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NOFTZ-NEXT: ret;
+;
+; SM60-FTZ-LABEL: fadd_acq_rel_float_global_sys(
+; SM60-FTZ: {
+; SM60-FTZ-NEXT: .reg .b32 %r<3>;
+; SM60-FTZ-NEXT: .reg .b64 %rd<2>;
+; SM60-FTZ-EMPTY:
+; SM60-FTZ-NEXT: // %bb.0:
+; SM60-FTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_sys_param_0];
+; SM60-FTZ-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_global_sys_param_1];
+; SM60-FTZ-NEXT: atom.sys.global.add.f32 %r2, [%rd1], %r1;
+; SM60-FTZ-NEXT: st.param.b32 [func_retval0], %r2;
+; SM60-FTZ-NEXT: ret;
%retval = atomicrmw fadd ptr addrspace(1) %addr, float %val syncscope("") acq_rel
ret float %retval
}
@@ -3403,17 +4124,40 @@ define i64 @nand_acq_rel_i64_global_cluster(ptr addrspace(1) %addr, i64 %val) {
}
define float @fadd_acq_rel_float_global_cluster(ptr addrspace(1) %addr, float %val) {
-; SM60-LABEL: fadd_acq_rel_float_global_cluster(
-; SM60: {
-; SM60-NEXT: .reg .b32 %r<3>;
-; SM60-NEXT: .reg .b64 %rd<2>;
-; SM60-EMPTY:
-; SM60-NEXT: // %bb.0:
-; SM60-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_cluster_param_0];
-; SM60-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_global_cluster_param_1];
-; SM60-NEXT: atom.cta.global.add.f32 %r2, [%rd1], %r1;
-; SM60-NEXT: st.param.b32 [func_retval0], %r2;
-; SM60-NEXT: ret;
+; SM60-NOFTZ-LABEL: fadd_acq_rel_float_global_cluster(
+; SM60-NOFTZ: {
+; SM60-NOFTZ-NEXT: .reg .pred %p<2>;
+; SM60-NOFTZ-NEXT: .reg .b32 %r<5>;
+; SM60-NOFTZ-NEXT: .reg .b64 %rd<2>;
+; SM60-NOFTZ-EMPTY:
+; SM60-NOFTZ-NEXT: // %bb.0:
+; SM60-NOFTZ-NEXT: ld.param.b32 %r2, [fadd_acq_rel_float_global_cluster_param_1];
+; SM60-NOFTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_cluster_param_0];
+; SM60-NOFTZ-NEXT: membar.cta;
+; SM60-NOFTZ-NEXT: ld.volatile.global.b32 %r4, [%rd1];
+; SM60-NOFTZ-NEXT: $L__BB114_1: // %atomicrmw.start
+; SM60-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NOFTZ-NEXT: add.rn.f32 %r3, %r4, %r2;
+; SM60-NOFTZ-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM60-NOFTZ-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM60-NOFTZ-NEXT: mov.b32 %r4, %r1;
+; SM60-NOFTZ-NEXT: @%p1 bra $L__BB114_1;
+; SM60-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NOFTZ-NEXT: membar.cta;
+; SM60-NOFTZ-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NOFTZ-NEXT: ret;
+;
+; SM60-FTZ-LABEL: fadd_acq_rel_float_global_cluster(
+; SM60-FTZ: {
+; SM60-FTZ-NEXT: .reg .b32 %r<3>;
+; SM60-FTZ-NEXT: .reg .b64 %rd<2>;
+; SM60-FTZ-EMPTY:
+; SM60-FTZ-NEXT: // %bb.0:
+; SM60-FTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_cluster_param_0];
+; SM60-FTZ-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_global_cluster_param_1];
+; SM60-FTZ-NEXT: atom.cta.global.add.f32 %r2, [%rd1], %r1;
+; SM60-FTZ-NEXT: st.param.b32 [func_retval0], %r2;
+; SM60-FTZ-NEXT: ret;
%retval = atomicrmw fadd ptr addrspace(1) %addr, float %val syncscope("cluster") acq_rel
ret float %retval
}
@@ -3552,17 +4296,40 @@ define i64 @nand_acq_rel_i64_global_gpu(ptr addrspace(1) %addr, i64 %val) {
}
define float @fadd_acq_rel_float_global_gpu(ptr addrspace(1) %addr, float %val) {
-; SM60-LABEL: fadd_acq_rel_float_global_gpu(
-; SM60: {
-; SM60-NEXT: .reg .b32 %r<3>;
-; SM60-NEXT: .reg .b64 %rd<2>;
-; SM60-EMPTY:
-; SM60-NEXT: // %bb.0:
-; SM60-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_gpu_param_0];
-; SM60-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_global_gpu_param_1];
-; SM60-NEXT: atom.gpu.global.add.f32 %r2, [%rd1], %r1;
-; SM60-NEXT: st.param.b32 [func_retval0], %r2;
-; SM60-NEXT: ret;
+; SM60-NOFTZ-LABEL: fadd_acq_rel_float_global_gpu(
+; SM60-NOFTZ: {
+; SM60-NOFTZ-NEXT: .reg .pred %p<2>;
+; SM60-NOFTZ-NEXT: .reg .b32 %r<5>;
+; SM60-NOFTZ-NEXT: .reg .b64 %rd<2>;
+; SM60-NOFTZ-EMPTY:
+; SM60-NOFTZ-NEXT: // %bb.0:
+; SM60-NOFTZ-NEXT: ld.param.b32 %r2, [fadd_acq_rel_float_global_gpu_param_1];
+; SM60-NOFTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_gpu_param_0];
+; SM60-NOFTZ-NEXT: membar.gl;
+; SM60-NOFTZ-NEXT: ld.volatile.global.b32 %r4, [%rd1];
+; SM60-NOFTZ-NEXT: $L__BB122_1: // %atomicrmw.start
+; SM60-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NOFTZ-NEXT: add.rn.f32 %r3, %r4, %r2;
+; SM60-NOFTZ-NEXT: atom.gpu.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM60-NOFTZ-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM60-NOFTZ-NEXT: mov.b32 %r4, %r1;
+; SM60-NOFTZ-NEXT: @%p1 bra $L__BB122_1;
+; SM60-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NOFTZ-NEXT: membar.gl;
+; SM60-NOFTZ-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NOFTZ-NEXT: ret;
+;
+; SM60-FTZ-LABEL: fadd_acq_rel_float_global_gpu(
+; SM60-FTZ: {
+; SM60-FTZ-NEXT: .reg .b32 %r<3>;
+; SM60-FTZ-NEXT: .reg .b64 %rd<2>;
+; SM60-FTZ-EMPTY:
+; SM60-FTZ-NEXT: // %bb.0:
+; SM60-FTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_gpu_param_0];
+; SM60-FTZ-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_global_gpu_param_1];
+; SM60-FTZ-NEXT: atom.gpu.global.add.f32 %r2, [%rd1], %r1;
+; SM60-FTZ-NEXT: st.param.b32 [func_retval0], %r2;
+; SM60-FTZ-NEXT: ret;
%retval = atomicrmw fadd ptr addrspace(1) %addr, float %val syncscope("device") acq_rel
ret float %retval
}
@@ -3701,17 +4468,51 @@ define i64 @nand_acq_rel_i64_generic_cta(ptr %addr, i64 %val) {
}
define float @fadd_acq_rel_float_generic_cta(ptr %addr, float %val) {
-; SM60-LABEL: fadd_acq_rel_float_generic_cta(
-; SM60: {
-; SM60-NEXT: .reg .b32 %r<3>;
-; SM60-NEXT: .reg .b64 %rd<2>;
-; SM60-EMPTY:
-; SM60-NEXT: // %bb.0:
-; SM60-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_generic_cta_param_0];
-; SM60-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_generic_cta_param_1];
-; SM60-NEXT: atom.cta.add.f32 %r2, [%rd1], %r1;
-; SM60-NEXT: st.param.b32 [func_retval0], %r2;
-; SM60-NEXT: ret;
+; SM60-NOFTZ-LABEL: fadd_acq_rel_float_generic_cta(
+; SM60-NOFTZ: {
+; SM60-NOFTZ-NEXT: .reg .pred %p<2>;
+; SM60-NOFTZ-NEXT: .reg .b32 %r<5>;
+; SM60-NOFTZ-NEXT: .reg .b64 %rd<2>;
+; SM60-NOFTZ-EMPTY:
+; SM60-NOFTZ-NEXT: // %bb.0:
+; SM60-NOFTZ-NEXT: ld.param.b32 %r2, [fadd_acq_rel_float_generic_cta_param_1];
+; SM60-NOFTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_generic_cta_param_0];
+; SM60-NOFTZ-NEXT: membar.cta;
+; SM60-NOFTZ-NEXT: ld.volatile.b32 %r4, [%rd1];
+; SM60-NOFTZ-NEXT: $L__BB130_1: // %atomicrmw.start
+; SM60-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NOFTZ-NEXT: add.rn.f32 %r3, %r4, %r2;
+; SM60-NOFTZ-NEXT: atom.cta.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM60-NOFTZ-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM60-NOFTZ-NEXT: mov.b32 %r4, %r1;
+; SM60-NOFTZ-NEXT: @%p1 bra $L__BB130_1;
+; SM60-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NOFTZ-NEXT: membar.cta;
+; SM60-NOFTZ-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NOFTZ-NEXT: ret;
+;
+; SM60-FTZ-LABEL: fadd_acq_rel_float_generic_cta(
+; SM60-FTZ: {
+; SM60-FTZ-NEXT: .reg .pred %p<2>;
+; SM60-FTZ-NEXT: .reg .b32 %r<5>;
+; SM60-FTZ-NEXT: .reg .b64 %rd<2>;
+; SM60-FTZ-EMPTY:
+; SM60-FTZ-NEXT: // %bb.0:
+; SM60-FTZ-NEXT: ld.param.b32 %r2, [fadd_acq_rel_float_generic_cta_param_1];
+; SM60-FTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_generic_cta_param_0];
+; SM60-FTZ-NEXT: membar.cta;
+; SM60-FTZ-NEXT: ld.volatile.b32 %r4, [%rd1];
+; SM60-FTZ-NEXT: $L__BB130_1: // %atomicrmw.start
+; SM60-FTZ-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-FTZ-NEXT: add.rn.ftz.f32 %r3, %r4, %r2;
+; SM60-FTZ-NEXT: atom.cta.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM60-FTZ-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM60-FTZ-NEXT: mov.b32 %r4, %r1;
+; SM60-FTZ-NEXT: @%p1 bra $L__BB130_1;
+; SM60-FTZ-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-FTZ-NEXT: membar.cta;
+; SM60-FTZ-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-FTZ-NEXT: ret;
%retval = atomicrmw fadd ptr %addr, float %val syncscope("block") acq_rel
ret float %retval
}
@@ -3850,17 +4651,40 @@ define i64 @nand_acq_rel_i64_shared_cta(ptr addrspace(3) %addr, i64 %val) {
}
define float @fadd_acq_rel_float_shared_cta(ptr addrspace(3) %addr, float %val) {
-; SM60-LABEL: fadd_acq_rel_float_shared_cta(
-; SM60: {
-; SM60-NEXT: .reg .b32 %r<3>;
-; SM60-NEXT: .reg .b64 %rd<2>;
-; SM60-EMPTY:
-; SM60-NEXT: // %bb.0:
-; SM60-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_shared_cta_param_0];
-; SM60-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_shared_cta_param_1];
-; SM60-NEXT: atom.cta.shared.add.f32 %r2, [%rd1], %r1;
-; SM60-NEXT: st.param.b32 [func_retval0], %r2;
-; SM60-NEXT: ret;
+; SM60-NOFTZ-LABEL: fadd_acq_rel_float_shared_cta(
+; SM60-NOFTZ: {
+; SM60-NOFTZ-NEXT: .reg .b32 %r<3>;
+; SM60-NOFTZ-NEXT: .reg .b64 %rd<2>;
+; SM60-NOFTZ-EMPTY:
+; SM60-NOFTZ-NEXT: // %bb.0:
+; SM60-NOFTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_shared_cta_param_0];
+; SM60-NOFTZ-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_shared_cta_param_1];
+; SM60-NOFTZ-NEXT: atom.cta.shared.add.f32 %r2, [%rd1], %r1;
+; SM60-NOFTZ-NEXT: st.param.b32 [func_retval0], %r2;
+; SM60-NOFTZ-NEXT: ret;
+;
+; SM60-FTZ-LABEL: fadd_acq_rel_float_shared_cta(
+; SM60-FTZ: {
+; SM60-FTZ-NEXT: .reg .pred %p<2>;
+; SM60-FTZ-NEXT: .reg .b32 %r<5>;
+; SM60-FTZ-NEXT: .reg .b64 %rd<2>;
+; SM60-FTZ-EMPTY:
+; SM60-FTZ-NEXT: // %bb.0:
+; SM60-FTZ-NEXT: ld.param.b32 %r2, [fadd_acq_rel_float_shared_cta_param_1];
+; SM60-FTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_shared_cta_param_0];
+; SM60-FTZ-NEXT: membar.cta;
+; SM60-FTZ-NEXT: ld.volatile.shared.b32 %r4, [%rd1];
+; SM60-FTZ-NEXT: $L__BB138_1: // %atomicrmw.start
+; SM60-FTZ-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-FTZ-NEXT: add.rn.ftz.f32 %r3, %r4, %r2;
+; SM60-FTZ-NEXT: atom.cta.shared.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM60-FTZ-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM60-FTZ-NEXT: mov.b32 %r4, %r1;
+; SM60-FTZ-NEXT: @%p1 bra $L__BB138_1;
+; SM60-FTZ-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-FTZ-NEXT: membar.cta;
+; SM60-FTZ-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-FTZ-NEXT: ret;
%retval = atomicrmw fadd ptr addrspace(3) %addr, float %val syncscope("block") acq_rel
ret float %retval
}
diff --git a/llvm/test/CodeGen/NVPTX/atomicrmw-sm70.ll b/llvm/test/CodeGen/NVPTX/atomicrmw-sm70.ll
index b0fd2e6758d55..10744421115b4 100644
--- a/llvm/test/CodeGen/NVPTX/atomicrmw-sm70.ll
+++ b/llvm/test/CodeGen/NVPTX/atomicrmw-sm70.ll
@@ -1,6 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
-; RUN: llc < %s -march=nvptx64 -mcpu=sm_70 -mattr=+ptx63 | FileCheck %s --check-prefix=SM70
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_70 -mattr=+ptx63 | FileCheck %s --check-prefixes=SM70,SM70-NOFTZ
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_70 -mattr=+ptx63 -denormal-fp-math-f32=preserve-sign | FileCheck %s --check-prefixes=SM70,SM70-FTZ
; RUN: %if ptxas-sm_70 && ptxas-isa-6.3 %{ llc < %s -march=nvptx64 -mcpu=sm_70 -mattr=+ptx63 | %ptxas-verify -arch=sm_70 %}
+; RUN: %if ptxas-sm_70 && ptxas-isa-6.3 %{ llc < %s -march=nvptx64 -mcpu=sm_70 -mattr=+ptx63 -denormal-fp-math-f32=preserve-sign | %ptxas-verify -arch=sm_70 %}
define i8 @xchg_acq_rel_i8_global_cta(ptr addrspace(1) %addr, i8 %val) {
; SM70-LABEL: xchg_acq_rel_i8_global_cta(
@@ -1737,168 +1739,322 @@ define i64 @usub_sat_acq_rel_i64_global_cta(ptr addrspace(1) %addr, i64 %val) {
}
define float @fadd_acq_rel_float_global_cta(ptr addrspace(1) %addr, float %val) {
-; SM70-LABEL: fadd_acq_rel_float_global_cta(
-; SM70: {
-; SM70-NEXT: .reg .b32 %r<3>;
-; SM70-NEXT: .reg .b64 %rd<2>;
-; SM70-EMPTY:
-; SM70-NEXT: // %bb.0:
-; SM70-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_cta_param_0];
-; SM70-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_global_cta_param_1];
-; SM70-NEXT: atom.acq_rel.cta.global.add.f32 %r2, [%rd1], %r1;
-; SM70-NEXT: st.param.b32 [func_retval0], %r2;
-; SM70-NEXT: ret;
+; SM70-NOFTZ-LABEL: fadd_acq_rel_float_global_cta(
+; SM70-NOFTZ: {
+; SM70-NOFTZ-NEXT: .reg .pred %p<2>;
+; SM70-NOFTZ-NEXT: .reg .b32 %r<5>;
+; SM70-NOFTZ-NEXT: .reg .b64 %rd<2>;
+; SM70-NOFTZ-EMPTY:
+; SM70-NOFTZ-NEXT: // %bb.0:
+; SM70-NOFTZ-NEXT: ld.param.b32 %r2, [fadd_acq_rel_float_global_cta_param_1];
+; SM70-NOFTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_cta_param_0];
+; SM70-NOFTZ-NEXT: fence.acq_rel.cta;
+; SM70-NOFTZ-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM70-NOFTZ-NEXT: $L__BB60_1: // %atomicrmw.start
+; SM70-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NOFTZ-NEXT: add.rn.f32 %r3, %r4, %r2;
+; SM70-NOFTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM70-NOFTZ-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM70-NOFTZ-NEXT: mov.b32 %r4, %r1;
+; SM70-NOFTZ-NEXT: @%p1 bra $L__BB60_1;
+; SM70-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NOFTZ-NEXT: fence.acq_rel.cta;
+; SM70-NOFTZ-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NOFTZ-NEXT: ret;
+;
+; SM70-FTZ-LABEL: fadd_acq_rel_float_global_cta(
+; SM70-FTZ: {
+; SM70-FTZ-NEXT: .reg .b32 %r<3>;
+; SM70-FTZ-NEXT: .reg .b64 %rd<2>;
+; SM70-FTZ-EMPTY:
+; SM70-FTZ-NEXT: // %bb.0:
+; SM70-FTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_cta_param_0];
+; SM70-FTZ-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_global_cta_param_1];
+; SM70-FTZ-NEXT: atom.acq_rel.cta.global.add.f32 %r2, [%rd1], %r1;
+; SM70-FTZ-NEXT: st.param.b32 [func_retval0], %r2;
+; SM70-FTZ-NEXT: ret;
%retval = atomicrmw fadd ptr addrspace(1) %addr, float %val syncscope("block") acq_rel
ret float %retval
}
define float @fsub_acq_rel_float_global_cta(ptr addrspace(1) %addr, float %val) {
-; SM70-LABEL: fsub_acq_rel_float_global_cta(
-; SM70: {
-; SM70-NEXT: .reg .pred %p<2>;
-; SM70-NEXT: .reg .b32 %r<5>;
-; SM70-NEXT: .reg .b64 %rd<2>;
-; SM70-EMPTY:
-; SM70-NEXT: // %bb.0:
-; SM70-NEXT: ld.param.b32 %r2, [fsub_acq_rel_float_global_cta_param_1];
-; SM70-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_float_global_cta_param_0];
-; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
-; SM70-NEXT: $L__BB61_1: // %atomicrmw.start
-; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: sub.rn.f32 %r3, %r4, %r2;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
-; SM70-NEXT: setp.ne.b32 %p1, %r1, %r4;
-; SM70-NEXT: mov.b32 %r4, %r1;
-; SM70-NEXT: @%p1 bra $L__BB61_1;
-; SM70-NEXT: // %bb.2: // %atomicrmw.end
-; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: st.param.b32 [func_retval0], %r1;
-; SM70-NEXT: ret;
+; SM70-NOFTZ-LABEL: fsub_acq_rel_float_global_cta(
+; SM70-NOFTZ: {
+; SM70-NOFTZ-NEXT: .reg .pred %p<2>;
+; SM70-NOFTZ-NEXT: .reg .b32 %r<5>;
+; SM70-NOFTZ-NEXT: .reg .b64 %rd<2>;
+; SM70-NOFTZ-EMPTY:
+; SM70-NOFTZ-NEXT: // %bb.0:
+; SM70-NOFTZ-NEXT: ld.param.b32 %r2, [fsub_acq_rel_float_global_cta_param_1];
+; SM70-NOFTZ-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_float_global_cta_param_0];
+; SM70-NOFTZ-NEXT: fence.acq_rel.cta;
+; SM70-NOFTZ-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM70-NOFTZ-NEXT: $L__BB61_1: // %atomicrmw.start
+; SM70-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NOFTZ-NEXT: sub.rn.f32 %r3, %r4, %r2;
+; SM70-NOFTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM70-NOFTZ-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM70-NOFTZ-NEXT: mov.b32 %r4, %r1;
+; SM70-NOFTZ-NEXT: @%p1 bra $L__BB61_1;
+; SM70-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NOFTZ-NEXT: fence.acq_rel.cta;
+; SM70-NOFTZ-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NOFTZ-NEXT: ret;
+;
+; SM70-FTZ-LABEL: fsub_acq_rel_float_global_cta(
+; SM70-FTZ: {
+; SM70-FTZ-NEXT: .reg .pred %p<2>;
+; SM70-FTZ-NEXT: .reg .b32 %r<5>;
+; SM70-FTZ-NEXT: .reg .b64 %rd<2>;
+; SM70-FTZ-EMPTY:
+; SM70-FTZ-NEXT: // %bb.0:
+; SM70-FTZ-NEXT: ld.param.b32 %r2, [fsub_acq_rel_float_global_cta_param_1];
+; SM70-FTZ-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_float_global_cta_param_0];
+; SM70-FTZ-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM70-FTZ-NEXT: $L__BB61_1: // %atomicrmw.start
+; SM70-FTZ-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-FTZ-NEXT: sub.rn.ftz.f32 %r3, %r4, %r2;
+; SM70-FTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM70-FTZ-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM70-FTZ-NEXT: mov.b32 %r4, %r1;
+; SM70-FTZ-NEXT: @%p1 bra $L__BB61_1;
+; SM70-FTZ-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-FTZ-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-FTZ-NEXT: ret;
%retval = atomicrmw fsub ptr addrspace(1) %addr, float %val syncscope("block") acq_rel
ret float %retval
}
define float @fmin_acq_rel_float_global_cta(ptr addrspace(1) %addr, float %val) {
-; SM70-LABEL: fmin_acq_rel_float_global_cta(
-; SM70: {
-; SM70-NEXT: .reg .pred %p<2>;
-; SM70-NEXT: .reg .b32 %r<5>;
-; SM70-NEXT: .reg .b64 %rd<2>;
-; SM70-EMPTY:
-; SM70-NEXT: // %bb.0:
-; SM70-NEXT: ld.param.b32 %r2, [fmin_acq_rel_float_global_cta_param_1];
-; SM70-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_float_global_cta_param_0];
-; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
-; SM70-NEXT: $L__BB62_1: // %atomicrmw.start
-; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: min.f32 %r3, %r4, %r2;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
-; SM70-NEXT: setp.ne.b32 %p1, %r1, %r4;
-; SM70-NEXT: mov.b32 %r4, %r1;
-; SM70-NEXT: @%p1 bra $L__BB62_1;
-; SM70-NEXT: // %bb.2: // %atomicrmw.end
-; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: st.param.b32 [func_retval0], %r1;
-; SM70-NEXT: ret;
+; SM70-NOFTZ-LABEL: fmin_acq_rel_float_global_cta(
+; SM70-NOFTZ: {
+; SM70-NOFTZ-NEXT: .reg .pred %p<2>;
+; SM70-NOFTZ-NEXT: .reg .b32 %r<5>;
+; SM70-NOFTZ-NEXT: .reg .b64 %rd<2>;
+; SM70-NOFTZ-EMPTY:
+; SM70-NOFTZ-NEXT: // %bb.0:
+; SM70-NOFTZ-NEXT: ld.param.b32 %r2, [fmin_acq_rel_float_global_cta_param_1];
+; SM70-NOFTZ-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_float_global_cta_param_0];
+; SM70-NOFTZ-NEXT: fence.acq_rel.cta;
+; SM70-NOFTZ-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM70-NOFTZ-NEXT: $L__BB62_1: // %atomicrmw.start
+; SM70-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NOFTZ-NEXT: min.f32 %r3, %r4, %r2;
+; SM70-NOFTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM70-NOFTZ-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM70-NOFTZ-NEXT: mov.b32 %r4, %r1;
+; SM70-NOFTZ-NEXT: @%p1 bra $L__BB62_1;
+; SM70-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NOFTZ-NEXT: fence.acq_rel.cta;
+; SM70-NOFTZ-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NOFTZ-NEXT: ret;
+;
+; SM70-FTZ-LABEL: fmin_acq_rel_float_global_cta(
+; SM70-FTZ: {
+; SM70-FTZ-NEXT: .reg .pred %p<2>;
+; SM70-FTZ-NEXT: .reg .b32 %r<5>;
+; SM70-FTZ-NEXT: .reg .b64 %rd<2>;
+; SM70-FTZ-EMPTY:
+; SM70-FTZ-NEXT: // %bb.0:
+; SM70-FTZ-NEXT: ld.param.b32 %r2, [fmin_acq_rel_float_global_cta_param_1];
+; SM70-FTZ-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_float_global_cta_param_0];
+; SM70-FTZ-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM70-FTZ-NEXT: $L__BB62_1: // %atomicrmw.start
+; SM70-FTZ-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-FTZ-NEXT: min.ftz.f32 %r3, %r4, %r2;
+; SM70-FTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM70-FTZ-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM70-FTZ-NEXT: mov.b32 %r4, %r1;
+; SM70-FTZ-NEXT: @%p1 bra $L__BB62_1;
+; SM70-FTZ-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-FTZ-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-FTZ-NEXT: ret;
%retval = atomicrmw fmin ptr addrspace(1) %addr, float %val syncscope("block") acq_rel
ret float %retval
}
define float @fmax_acq_rel_float_global_cta(ptr addrspace(1) %addr, float %val) {
-; SM70-LABEL: fmax_acq_rel_float_global_cta(
-; SM70: {
-; SM70-NEXT: .reg .pred %p<2>;
-; SM70-NEXT: .reg .b32 %r<5>;
-; SM70-NEXT: .reg .b64 %rd<2>;
-; SM70-EMPTY:
-; SM70-NEXT: // %bb.0:
-; SM70-NEXT: ld.param.b32 %r2, [fmax_acq_rel_float_global_cta_param_1];
-; SM70-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_float_global_cta_param_0];
-; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
-; SM70-NEXT: $L__BB63_1: // %atomicrmw.start
-; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: max.f32 %r3, %r4, %r2;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
-; SM70-NEXT: setp.ne.b32 %p1, %r1, %r4;
-; SM70-NEXT: mov.b32 %r4, %r1;
-; SM70-NEXT: @%p1 bra $L__BB63_1;
-; SM70-NEXT: // %bb.2: // %atomicrmw.end
-; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: st.param.b32 [func_retval0], %r1;
-; SM70-NEXT: ret;
+; SM70-NOFTZ-LABEL: fmax_acq_rel_float_global_cta(
+; SM70-NOFTZ: {
+; SM70-NOFTZ-NEXT: .reg .pred %p<2>;
+; SM70-NOFTZ-NEXT: .reg .b32 %r<5>;
+; SM70-NOFTZ-NEXT: .reg .b64 %rd<2>;
+; SM70-NOFTZ-EMPTY:
+; SM70-NOFTZ-NEXT: // %bb.0:
+; SM70-NOFTZ-NEXT: ld.param.b32 %r2, [fmax_acq_rel_float_global_cta_param_1];
+; SM70-NOFTZ-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_float_global_cta_param_0];
+; SM70-NOFTZ-NEXT: fence.acq_rel.cta;
+; SM70-NOFTZ-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM70-NOFTZ-NEXT: $L__BB63_1: // %atomicrmw.start
+; SM70-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NOFTZ-NEXT: max.f32 %r3, %r4, %r2;
+; SM70-NOFTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM70-NOFTZ-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM70-NOFTZ-NEXT: mov.b32 %r4, %r1;
+; SM70-NOFTZ-NEXT: @%p1 bra $L__BB63_1;
+; SM70-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NOFTZ-NEXT: fence.acq_rel.cta;
+; SM70-NOFTZ-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NOFTZ-NEXT: ret;
+;
+; SM70-FTZ-LABEL: fmax_acq_rel_float_global_cta(
+; SM70-FTZ: {
+; SM70-FTZ-NEXT: .reg .pred %p<2>;
+; SM70-FTZ-NEXT: .reg .b32 %r<5>;
+; SM70-FTZ-NEXT: .reg .b64 %rd<2>;
+; SM70-FTZ-EMPTY:
+; SM70-FTZ-NEXT: // %bb.0:
+; SM70-FTZ-NEXT: ld.param.b32 %r2, [fmax_acq_rel_float_global_cta_param_1];
+; SM70-FTZ-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_float_global_cta_param_0];
+; SM70-FTZ-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM70-FTZ-NEXT: $L__BB63_1: // %atomicrmw.start
+; SM70-FTZ-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-FTZ-NEXT: max.ftz.f32 %r3, %r4, %r2;
+; SM70-FTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM70-FTZ-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM70-FTZ-NEXT: mov.b32 %r4, %r1;
+; SM70-FTZ-NEXT: @%p1 bra $L__BB63_1;
+; SM70-FTZ-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-FTZ-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-FTZ-NEXT: ret;
%retval = atomicrmw fmax ptr addrspace(1) %addr, float %val syncscope("block") acq_rel
ret float %retval
}
define float @fminimum_acq_rel_float_global_cta(ptr addrspace(1) %addr, float %val) {
-; SM70-LABEL: fminimum_acq_rel_float_global_cta(
-; SM70: {
-; SM70-NEXT: .reg .pred %p<6>;
-; SM70-NEXT: .reg .b32 %r<9>;
-; SM70-NEXT: .reg .b64 %rd<2>;
-; SM70-EMPTY:
-; SM70-NEXT: // %bb.0:
-; SM70-NEXT: ld.param.b32 %r2, [fminimum_acq_rel_float_global_cta_param_1];
-; SM70-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_float_global_cta_param_0];
-; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: ld.relaxed.cta.global.b32 %r8, [%rd1];
-; SM70-NEXT: setp.eq.b32 %p3, %r2, -2147483648;
-; SM70-NEXT: $L__BB64_1: // %atomicrmw.start
-; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: setp.nan.f32 %p1, %r8, %r2;
-; SM70-NEXT: min.f32 %r3, %r8, %r2;
-; SM70-NEXT: selp.f32 %r4, 0f7FC00000, %r3, %p1;
-; SM70-NEXT: setp.eq.b32 %p2, %r8, -2147483648;
-; SM70-NEXT: selp.f32 %r5, %r8, %r4, %p2;
-; SM70-NEXT: selp.f32 %r6, %r2, %r5, %p3;
-; SM70-NEXT: setp.eq.f32 %p4, %r4, 0f00000000;
-; SM70-NEXT: selp.f32 %r7, %r6, %r4, %p4;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r8, %r7;
-; SM70-NEXT: setp.ne.b32 %p5, %r1, %r8;
-; SM70-NEXT: mov.b32 %r8, %r1;
-; SM70-NEXT: @%p5 bra $L__BB64_1;
-; SM70-NEXT: // %bb.2: // %atomicrmw.end
-; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: st.param.b32 [func_retval0], %r1;
-; SM70-NEXT: ret;
+; SM70-NOFTZ-LABEL: fminimum_acq_rel_float_global_cta(
+; SM70-NOFTZ: {
+; SM70-NOFTZ-NEXT: .reg .pred %p<6>;
+; SM70-NOFTZ-NEXT: .reg .b32 %r<9>;
+; SM70-NOFTZ-NEXT: .reg .b64 %rd<2>;
+; SM70-NOFTZ-EMPTY:
+; SM70-NOFTZ-NEXT: // %bb.0:
+; SM70-NOFTZ-NEXT: ld.param.b32 %r2, [fminimum_acq_rel_float_global_cta_param_1];
+; SM70-NOFTZ-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_float_global_cta_param_0];
+; SM70-NOFTZ-NEXT: fence.acq_rel.cta;
+; SM70-NOFTZ-NEXT: ld.relaxed.cta.global.b32 %r8, [%rd1];
+; SM70-NOFTZ-NEXT: setp.eq.b32 %p3, %r2, -2147483648;
+; SM70-NOFTZ-NEXT: $L__BB64_1: // %atomicrmw.start
+; SM70-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NOFTZ-NEXT: setp.nan.f32 %p1, %r8, %r2;
+; SM70-NOFTZ-NEXT: min.f32 %r3, %r8, %r2;
+; SM70-NOFTZ-NEXT: selp.f32 %r4, 0f7FC00000, %r3, %p1;
+; SM70-NOFTZ-NEXT: setp.eq.b32 %p2, %r8, -2147483648;
+; SM70-NOFTZ-NEXT: selp.f32 %r5, %r8, %r4, %p2;
+; SM70-NOFTZ-NEXT: selp.f32 %r6, %r2, %r5, %p3;
+; SM70-NOFTZ-NEXT: setp.eq.f32 %p4, %r4, 0f00000000;
+; SM70-NOFTZ-NEXT: selp.f32 %r7, %r6, %r4, %p4;
+; SM70-NOFTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r8, %r7;
+; SM70-NOFTZ-NEXT: setp.ne.b32 %p5, %r1, %r8;
+; SM70-NOFTZ-NEXT: mov.b32 %r8, %r1;
+; SM70-NOFTZ-NEXT: @%p5 bra $L__BB64_1;
+; SM70-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NOFTZ-NEXT: fence.acq_rel.cta;
+; SM70-NOFTZ-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NOFTZ-NEXT: ret;
+;
+; SM70-FTZ-LABEL: fminimum_acq_rel_float_global_cta(
+; SM70-FTZ: {
+; SM70-FTZ-NEXT: .reg .pred %p<6>;
+; SM70-FTZ-NEXT: .reg .b32 %r<9>;
+; SM70-FTZ-NEXT: .reg .b64 %rd<2>;
+; SM70-FTZ-EMPTY:
+; SM70-FTZ-NEXT: // %bb.0:
+; SM70-FTZ-NEXT: ld.param.b32 %r2, [fminimum_acq_rel_float_global_cta_param_1];
+; SM70-FTZ-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_float_global_cta_param_0];
+; SM70-FTZ-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-NEXT: ld.relaxed.cta.global.b32 %r8, [%rd1];
+; SM70-FTZ-NEXT: setp.eq.b32 %p3, %r2, -2147483648;
+; SM70-FTZ-NEXT: $L__BB64_1: // %atomicrmw.start
+; SM70-FTZ-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-FTZ-NEXT: setp.nan.ftz.f32 %p1, %r8, %r2;
+; SM70-FTZ-NEXT: min.ftz.f32 %r3, %r8, %r2;
+; SM70-FTZ-NEXT: selp.f32 %r4, 0f7FC00000, %r3, %p1;
+; SM70-FTZ-NEXT: setp.eq.b32 %p2, %r8, -2147483648;
+; SM70-FTZ-NEXT: selp.f32 %r5, %r8, %r4, %p2;
+; SM70-FTZ-NEXT: selp.f32 %r6, %r2, %r5, %p3;
+; SM70-FTZ-NEXT: setp.eq.ftz.f32 %p4, %r4, 0f00000000;
+; SM70-FTZ-NEXT: selp.f32 %r7, %r6, %r4, %p4;
+; SM70-FTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r8, %r7;
+; SM70-FTZ-NEXT: setp.ne.b32 %p5, %r1, %r8;
+; SM70-FTZ-NEXT: mov.b32 %r8, %r1;
+; SM70-FTZ-NEXT: @%p5 bra $L__BB64_1;
+; SM70-FTZ-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-FTZ-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-FTZ-NEXT: ret;
%retval = atomicrmw fminimum ptr addrspace(1) %addr, float %val syncscope("block") acq_rel
ret float %retval
}
define float @fmaximum_acq_rel_float_global_cta(ptr addrspace(1) %addr, float %val) {
-; SM70-LABEL: fmaximum_acq_rel_float_global_cta(
-; SM70: {
-; SM70-NEXT: .reg .pred %p<6>;
-; SM70-NEXT: .reg .b32 %r<9>;
-; SM70-NEXT: .reg .b64 %rd<2>;
-; SM70-EMPTY:
-; SM70-NEXT: // %bb.0:
-; SM70-NEXT: ld.param.b32 %r2, [fmaximum_acq_rel_float_global_cta_param_1];
-; SM70-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_float_global_cta_param_0];
-; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: ld.relaxed.cta.global.b32 %r8, [%rd1];
-; SM70-NEXT: setp.eq.b32 %p3, %r2, 0;
-; SM70-NEXT: $L__BB65_1: // %atomicrmw.start
-; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: setp.nan.f32 %p1, %r8, %r2;
-; SM70-NEXT: max.f32 %r3, %r8, %r2;
-; SM70-NEXT: selp.f32 %r4, 0f7FC00000, %r3, %p1;
-; SM70-NEXT: setp.eq.b32 %p2, %r8, 0;
-; SM70-NEXT: selp.f32 %r5, %r8, %r4, %p2;
-; SM70-NEXT: selp.f32 %r6, %r2, %r5, %p3;
-; SM70-NEXT: setp.eq.f32 %p4, %r4, 0f00000000;
-; SM70-NEXT: selp.f32 %r7, %r6, %r4, %p4;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r8, %r7;
-; SM70-NEXT: setp.ne.b32 %p5, %r1, %r8;
-; SM70-NEXT: mov.b32 %r8, %r1;
-; SM70-NEXT: @%p5 bra $L__BB65_1;
-; SM70-NEXT: // %bb.2: // %atomicrmw.end
-; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: st.param.b32 [func_retval0], %r1;
-; SM70-NEXT: ret;
+; SM70-NOFTZ-LABEL: fmaximum_acq_rel_float_global_cta(
+; SM70-NOFTZ: {
+; SM70-NOFTZ-NEXT: .reg .pred %p<6>;
+; SM70-NOFTZ-NEXT: .reg .b32 %r<9>;
+; SM70-NOFTZ-NEXT: .reg .b64 %rd<2>;
+; SM70-NOFTZ-EMPTY:
+; SM70-NOFTZ-NEXT: // %bb.0:
+; SM70-NOFTZ-NEXT: ld.param.b32 %r2, [fmaximum_acq_rel_float_global_cta_param_1];
+; SM70-NOFTZ-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_float_global_cta_param_0];
+; SM70-NOFTZ-NEXT: fence.acq_rel.cta;
+; SM70-NOFTZ-NEXT: ld.relaxed.cta.global.b32 %r8, [%rd1];
+; SM70-NOFTZ-NEXT: setp.eq.b32 %p3, %r2, 0;
+; SM70-NOFTZ-NEXT: $L__BB65_1: // %atomicrmw.start
+; SM70-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NOFTZ-NEXT: setp.nan.f32 %p1, %r8, %r2;
+; SM70-NOFTZ-NEXT: max.f32 %r3, %r8, %r2;
+; SM70-NOFTZ-NEXT: selp.f32 %r4, 0f7FC00000, %r3, %p1;
+; SM70-NOFTZ-NEXT: setp.eq.b32 %p2, %r8, 0;
+; SM70-NOFTZ-NEXT: selp.f32 %r5, %r8, %r4, %p2;
+; SM70-NOFTZ-NEXT: selp.f32 %r6, %r2, %r5, %p3;
+; SM70-NOFTZ-NEXT: setp.eq.f32 %p4, %r4, 0f00000000;
+; SM70-NOFTZ-NEXT: selp.f32 %r7, %r6, %r4, %p4;
+; SM70-NOFTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r8, %r7;
+; SM70-NOFTZ-NEXT: setp.ne.b32 %p5, %r1, %r8;
+; SM70-NOFTZ-NEXT: mov.b32 %r8, %r1;
+; SM70-NOFTZ-NEXT: @%p5 bra $L__BB65_1;
+; SM70-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NOFTZ-NEXT: fence.acq_rel.cta;
+; SM70-NOFTZ-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NOFTZ-NEXT: ret;
+;
+; SM70-FTZ-LABEL: fmaximum_acq_rel_float_global_cta(
+; SM70-FTZ: {
+; SM70-FTZ-NEXT: .reg .pred %p<6>;
+; SM70-FTZ-NEXT: .reg .b32 %r<9>;
+; SM70-FTZ-NEXT: .reg .b64 %rd<2>;
+; SM70-FTZ-EMPTY:
+; SM70-FTZ-NEXT: // %bb.0:
+; SM70-FTZ-NEXT: ld.param.b32 %r2, [fmaximum_acq_rel_float_global_cta_param_1];
+; SM70-FTZ-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_float_global_cta_param_0];
+; SM70-FTZ-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-NEXT: ld.relaxed.cta.global.b32 %r8, [%rd1];
+; SM70-FTZ-NEXT: setp.eq.b32 %p3, %r2, 0;
+; SM70-FTZ-NEXT: $L__BB65_1: // %atomicrmw.start
+; SM70-FTZ-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-FTZ-NEXT: setp.nan.ftz.f32 %p1, %r8, %r2;
+; SM70-FTZ-NEXT: max.ftz.f32 %r3, %r8, %r2;
+; SM70-FTZ-NEXT: selp.f32 %r4, 0f7FC00000, %r3, %p1;
+; SM70-FTZ-NEXT: setp.eq.b32 %p2, %r8, 0;
+; SM70-FTZ-NEXT: selp.f32 %r5, %r8, %r4, %p2;
+; SM70-FTZ-NEXT: selp.f32 %r6, %r2, %r5, %p3;
+; SM70-FTZ-NEXT: setp.eq.ftz.f32 %p4, %r4, 0f00000000;
+; SM70-FTZ-NEXT: selp.f32 %r7, %r6, %r4, %p4;
+; SM70-FTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r8, %r7;
+; SM70-FTZ-NEXT: setp.ne.b32 %p5, %r1, %r8;
+; SM70-FTZ-NEXT: mov.b32 %r8, %r1;
+; SM70-FTZ-NEXT: @%p5 bra $L__BB65_1;
+; SM70-FTZ-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-FTZ-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-FTZ-NEXT: ret;
%retval = atomicrmw fmaximum ptr addrspace(1) %addr, float %val syncscope("block") acq_rel
ret float %retval
}
@@ -2065,565 +2221,1107 @@ define double @fmaximum_acq_rel_double_global_cta(ptr addrspace(1) %addr, double
}
define half @fadd_acq_rel_half_global_cta(ptr addrspace(1) %addr, half %val) {
-; SM70-LABEL: fadd_acq_rel_half_global_cta(
-; SM70: {
-; SM70-NEXT: .reg .b16 %rs<3>;
-; SM70-NEXT: .reg .b64 %rd<2>;
-; SM70-EMPTY:
-; SM70-NEXT: // %bb.0:
-; SM70-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_half_global_cta_param_0];
-; SM70-NEXT: ld.param.b16 %rs1, [fadd_acq_rel_half_global_cta_param_1];
-; SM70-NEXT: atom.acq_rel.cta.global.add.noftz.f16 %rs2, [%rd1], %rs1;
-; SM70-NEXT: st.param.b16 [func_retval0], %rs2;
-; SM70-NEXT: ret;
+; SM70-NOFTZ-LABEL: fadd_acq_rel_half_global_cta(
+; SM70-NOFTZ: {
+; SM70-NOFTZ-NEXT: .reg .b16 %rs<3>;
+; SM70-NOFTZ-NEXT: .reg .b64 %rd<2>;
+; SM70-NOFTZ-EMPTY:
+; SM70-NOFTZ-NEXT: // %bb.0:
+; SM70-NOFTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_half_global_cta_param_0];
+; SM70-NOFTZ-NEXT: ld.param.b16 %rs1, [fadd_acq_rel_half_global_cta_param_1];
+; SM70-NOFTZ-NEXT: atom.acq_rel.cta.global.add.noftz.f16 %rs2, [%rd1], %rs1;
+; SM70-NOFTZ-NEXT: st.param.b16 [func_retval0], %rs2;
+; SM70-NOFTZ-NEXT: ret;
+;
+; SM70-FTZ-LABEL: fadd_acq_rel_half_global_cta(
+; SM70-FTZ: {
+; SM70-FTZ-NEXT: .reg .pred %p<2>;
+; SM70-FTZ-NEXT: .reg .b16 %rs<4>;
+; SM70-FTZ-NEXT: .reg .b32 %r<15>;
+; SM70-FTZ-NEXT: .reg .b64 %rd<3>;
+; SM70-FTZ-EMPTY:
+; SM70-FTZ-NEXT: // %bb.0:
+; SM70-FTZ-NEXT: ld.param.b16 %rs1, [fadd_acq_rel_half_global_cta_param_1];
+; SM70-FTZ-NEXT: ld.param.b64 %rd2, [fadd_acq_rel_half_global_cta_param_0];
+; SM70-FTZ-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-FTZ-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-FTZ-NEXT: and.b32 %r5, %r4, 3;
+; SM70-FTZ-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-FTZ-NEXT: mov.b32 %r6, 65535;
+; SM70-FTZ-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-FTZ-NEXT: not.b32 %r2, %r7;
+; SM70-FTZ-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM70-FTZ-NEXT: $L__BB72_1: // %atomicrmw.start
+; SM70-FTZ-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-FTZ-NEXT: shr.u32 %r8, %r14, %r1;
+; SM70-FTZ-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM70-FTZ-NEXT: add.rn.ftz.f16 %rs3, %rs2, %rs1;
+; SM70-FTZ-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM70-FTZ-NEXT: shl.b32 %r10, %r9, %r1;
+; SM70-FTZ-NEXT: and.b32 %r11, %r14, %r2;
+; SM70-FTZ-NEXT: or.b32 %r12, %r11, %r10;
+; SM70-FTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM70-FTZ-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM70-FTZ-NEXT: mov.b32 %r14, %r3;
+; SM70-FTZ-NEXT: @%p1 bra $L__BB72_1;
+; SM70-FTZ-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-FTZ-NEXT: shr.u32 %r13, %r3, %r1;
+; SM70-FTZ-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-NEXT: st.param.b16 [func_retval0], %r13;
+; SM70-FTZ-NEXT: ret;
%retval = atomicrmw fadd ptr addrspace(1) %addr, half %val syncscope("block") acq_rel
ret half %retval
}
define half @fsub_acq_rel_half_global_cta(ptr addrspace(1) %addr, half %val) {
-; SM70-LABEL: fsub_acq_rel_half_global_cta(
-; SM70: {
-; SM70-NEXT: .reg .pred %p<2>;
-; SM70-NEXT: .reg .b16 %rs<4>;
-; SM70-NEXT: .reg .b32 %r<15>;
-; SM70-NEXT: .reg .b64 %rd<3>;
-; SM70-EMPTY:
-; SM70-NEXT: // %bb.0:
-; SM70-NEXT: ld.param.b16 %rs1, [fsub_acq_rel_half_global_cta_param_1];
-; SM70-NEXT: ld.param.b64 %rd2, [fsub_acq_rel_half_global_cta_param_0];
-; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: and.b64 %rd1, %rd2, -4;
-; SM70-NEXT: cvt.u32.u64 %r4, %rd2;
-; SM70-NEXT: and.b32 %r5, %r4, 3;
-; SM70-NEXT: shl.b32 %r1, %r5, 3;
-; SM70-NEXT: mov.b32 %r6, 65535;
-; SM70-NEXT: shl.b32 %r7, %r6, %r1;
-; SM70-NEXT: not.b32 %r2, %r7;
-; SM70-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
-; SM70-NEXT: $L__BB73_1: // %atomicrmw.start
-; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: shr.u32 %r8, %r14, %r1;
-; SM70-NEXT: cvt.u16.u32 %rs2, %r8;
-; SM70-NEXT: sub.rn.f16 %rs3, %rs2, %rs1;
-; SM70-NEXT: cvt.u32.u16 %r9, %rs3;
-; SM70-NEXT: shl.b32 %r10, %r9, %r1;
-; SM70-NEXT: and.b32 %r11, %r14, %r2;
-; SM70-NEXT: or.b32 %r12, %r11, %r10;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
-; SM70-NEXT: setp.ne.b32 %p1, %r3, %r14;
-; SM70-NEXT: mov.b32 %r14, %r3;
-; SM70-NEXT: @%p1 bra $L__BB73_1;
-; SM70-NEXT: // %bb.2: // %atomicrmw.end
-; SM70-NEXT: shr.u32 %r13, %r3, %r1;
-; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: st.param.b16 [func_retval0], %r13;
-; SM70-NEXT: ret;
+; SM70-NOFTZ-LABEL: fsub_acq_rel_half_global_cta(
+; SM70-NOFTZ: {
+; SM70-NOFTZ-NEXT: .reg .pred %p<2>;
+; SM70-NOFTZ-NEXT: .reg .b16 %rs<4>;
+; SM70-NOFTZ-NEXT: .reg .b32 %r<15>;
+; SM70-NOFTZ-NEXT: .reg .b64 %rd<3>;
+; SM70-NOFTZ-EMPTY:
+; SM70-NOFTZ-NEXT: // %bb.0:
+; SM70-NOFTZ-NEXT: ld.param.b16 %rs1, [fsub_acq_rel_half_global_cta_param_1];
+; SM70-NOFTZ-NEXT: ld.param.b64 %rd2, [fsub_acq_rel_half_global_cta_param_0];
+; SM70-NOFTZ-NEXT: fence.acq_rel.cta;
+; SM70-NOFTZ-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NOFTZ-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-NOFTZ-NEXT: and.b32 %r5, %r4, 3;
+; SM70-NOFTZ-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-NOFTZ-NEXT: mov.b32 %r6, 65535;
+; SM70-NOFTZ-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-NOFTZ-NEXT: not.b32 %r2, %r7;
+; SM70-NOFTZ-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM70-NOFTZ-NEXT: $L__BB73_1: // %atomicrmw.start
+; SM70-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NOFTZ-NEXT: shr.u32 %r8, %r14, %r1;
+; SM70-NOFTZ-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM70-NOFTZ-NEXT: sub.rn.f16 %rs3, %rs2, %rs1;
+; SM70-NOFTZ-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM70-NOFTZ-NEXT: shl.b32 %r10, %r9, %r1;
+; SM70-NOFTZ-NEXT: and.b32 %r11, %r14, %r2;
+; SM70-NOFTZ-NEXT: or.b32 %r12, %r11, %r10;
+; SM70-NOFTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM70-NOFTZ-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM70-NOFTZ-NEXT: mov.b32 %r14, %r3;
+; SM70-NOFTZ-NEXT: @%p1 bra $L__BB73_1;
+; SM70-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NOFTZ-NEXT: shr.u32 %r13, %r3, %r1;
+; SM70-NOFTZ-NEXT: fence.acq_rel.cta;
+; SM70-NOFTZ-NEXT: st.param.b16 [func_retval0], %r13;
+; SM70-NOFTZ-NEXT: ret;
+;
+; SM70-FTZ-LABEL: fsub_acq_rel_half_global_cta(
+; SM70-FTZ: {
+; SM70-FTZ-NEXT: .reg .pred %p<2>;
+; SM70-FTZ-NEXT: .reg .b16 %rs<4>;
+; SM70-FTZ-NEXT: .reg .b32 %r<15>;
+; SM70-FTZ-NEXT: .reg .b64 %rd<3>;
+; SM70-FTZ-EMPTY:
+; SM70-FTZ-NEXT: // %bb.0:
+; SM70-FTZ-NEXT: ld.param.b16 %rs1, [fsub_acq_rel_half_global_cta_param_1];
+; SM70-FTZ-NEXT: ld.param.b64 %rd2, [fsub_acq_rel_half_global_cta_param_0];
+; SM70-FTZ-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-FTZ-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-FTZ-NEXT: and.b32 %r5, %r4, 3;
+; SM70-FTZ-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-FTZ-NEXT: mov.b32 %r6, 65535;
+; SM70-FTZ-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-FTZ-NEXT: not.b32 %r2, %r7;
+; SM70-FTZ-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM70-FTZ-NEXT: $L__BB73_1: // %atomicrmw.start
+; SM70-FTZ-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-FTZ-NEXT: shr.u32 %r8, %r14, %r1;
+; SM70-FTZ-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM70-FTZ-NEXT: sub.rn.ftz.f16 %rs3, %rs2, %rs1;
+; SM70-FTZ-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM70-FTZ-NEXT: shl.b32 %r10, %r9, %r1;
+; SM70-FTZ-NEXT: and.b32 %r11, %r14, %r2;
+; SM70-FTZ-NEXT: or.b32 %r12, %r11, %r10;
+; SM70-FTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM70-FTZ-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM70-FTZ-NEXT: mov.b32 %r14, %r3;
+; SM70-FTZ-NEXT: @%p1 bra $L__BB73_1;
+; SM70-FTZ-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-FTZ-NEXT: shr.u32 %r13, %r3, %r1;
+; SM70-FTZ-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-NEXT: st.param.b16 [func_retval0], %r13;
+; SM70-FTZ-NEXT: ret;
%retval = atomicrmw fsub ptr addrspace(1) %addr, half %val syncscope("block") acq_rel
ret half %retval
}
define half @fmin_acq_rel_half_global_cta(ptr addrspace(1) %addr, half %val) {
-; SM70-LABEL: fmin_acq_rel_half_global_cta(
-; SM70: {
-; SM70-NEXT: .reg .pred %p<2>;
-; SM70-NEXT: .reg .b16 %rs<4>;
-; SM70-NEXT: .reg .b32 %r<18>;
-; SM70-NEXT: .reg .b64 %rd<3>;
-; SM70-EMPTY:
-; SM70-NEXT: // %bb.0:
-; SM70-NEXT: ld.param.b16 %rs1, [fmin_acq_rel_half_global_cta_param_1];
-; SM70-NEXT: ld.param.b64 %rd2, [fmin_acq_rel_half_global_cta_param_0];
-; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: and.b64 %rd1, %rd2, -4;
-; SM70-NEXT: cvt.u32.u64 %r4, %rd2;
-; SM70-NEXT: and.b32 %r5, %r4, 3;
-; SM70-NEXT: shl.b32 %r1, %r5, 3;
-; SM70-NEXT: mov.b32 %r6, 65535;
-; SM70-NEXT: shl.b32 %r7, %r6, %r1;
-; SM70-NEXT: not.b32 %r2, %r7;
-; SM70-NEXT: ld.relaxed.cta.global.b32 %r17, [%rd1];
-; SM70-NEXT: cvt.f32.f16 %r10, %rs1;
-; SM70-NEXT: $L__BB74_1: // %atomicrmw.start
-; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: shr.u32 %r8, %r17, %r1;
-; SM70-NEXT: cvt.u16.u32 %rs2, %r8;
-; SM70-NEXT: cvt.f32.f16 %r9, %rs2;
-; SM70-NEXT: min.f32 %r11, %r9, %r10;
-; SM70-NEXT: cvt.rn.f16.f32 %rs3, %r11;
-; SM70-NEXT: cvt.u32.u16 %r12, %rs3;
-; SM70-NEXT: shl.b32 %r13, %r12, %r1;
-; SM70-NEXT: and.b32 %r14, %r17, %r2;
-; SM70-NEXT: or.b32 %r15, %r14, %r13;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r17, %r15;
-; SM70-NEXT: setp.ne.b32 %p1, %r3, %r17;
-; SM70-NEXT: mov.b32 %r17, %r3;
-; SM70-NEXT: @%p1 bra $L__BB74_1;
-; SM70-NEXT: // %bb.2: // %atomicrmw.end
-; SM70-NEXT: shr.u32 %r16, %r3, %r1;
-; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: st.param.b16 [func_retval0], %r16;
-; SM70-NEXT: ret;
+; SM70-NOFTZ-LABEL: fmin_acq_rel_half_global_cta(
+; SM70-NOFTZ: {
+; SM70-NOFTZ-NEXT: .reg .pred %p<2>;
+; SM70-NOFTZ-NEXT: .reg .b16 %rs<4>;
+; SM70-NOFTZ-NEXT: .reg .b32 %r<18>;
+; SM70-NOFTZ-NEXT: .reg .b64 %rd<3>;
+; SM70-NOFTZ-EMPTY:
+; SM70-NOFTZ-NEXT: // %bb.0:
+; SM70-NOFTZ-NEXT: ld.param.b16 %rs1, [fmin_acq_rel_half_global_cta_param_1];
+; SM70-NOFTZ-NEXT: ld.param.b64 %rd2, [fmin_acq_rel_half_global_cta_param_0];
+; SM70-NOFTZ-NEXT: fence.acq_rel.cta;
+; SM70-NOFTZ-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NOFTZ-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-NOFTZ-NEXT: and.b32 %r5, %r4, 3;
+; SM70-NOFTZ-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-NOFTZ-NEXT: mov.b32 %r6, 65535;
+; SM70-NOFTZ-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-NOFTZ-NEXT: not.b32 %r2, %r7;
+; SM70-NOFTZ-NEXT: ld.relaxed.cta.global.b32 %r17, [%rd1];
+; SM70-NOFTZ-NEXT: cvt.f32.f16 %r10, %rs1;
+; SM70-NOFTZ-NEXT: $L__BB74_1: // %atomicrmw.start
+; SM70-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NOFTZ-NEXT: shr.u32 %r8, %r17, %r1;
+; SM70-NOFTZ-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM70-NOFTZ-NEXT: cvt.f32.f16 %r9, %rs2;
+; SM70-NOFTZ-NEXT: min.f32 %r11, %r9, %r10;
+; SM70-NOFTZ-NEXT: cvt.rn.f16.f32 %rs3, %r11;
+; SM70-NOFTZ-NEXT: cvt.u32.u16 %r12, %rs3;
+; SM70-NOFTZ-NEXT: shl.b32 %r13, %r12, %r1;
+; SM70-NOFTZ-NEXT: and.b32 %r14, %r17, %r2;
+; SM70-NOFTZ-NEXT: or.b32 %r15, %r14, %r13;
+; SM70-NOFTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r17, %r15;
+; SM70-NOFTZ-NEXT: setp.ne.b32 %p1, %r3, %r17;
+; SM70-NOFTZ-NEXT: mov.b32 %r17, %r3;
+; SM70-NOFTZ-NEXT: @%p1 bra $L__BB74_1;
+; SM70-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NOFTZ-NEXT: shr.u32 %r16, %r3, %r1;
+; SM70-NOFTZ-NEXT: fence.acq_rel.cta;
+; SM70-NOFTZ-NEXT: st.param.b16 [func_retval0], %r16;
+; SM70-NOFTZ-NEXT: ret;
+;
+; SM70-FTZ-LABEL: fmin_acq_rel_half_global_cta(
+; SM70-FTZ: {
+; SM70-FTZ-NEXT: .reg .pred %p<2>;
+; SM70-FTZ-NEXT: .reg .b16 %rs<4>;
+; SM70-FTZ-NEXT: .reg .b32 %r<18>;
+; SM70-FTZ-NEXT: .reg .b64 %rd<3>;
+; SM70-FTZ-EMPTY:
+; SM70-FTZ-NEXT: // %bb.0:
+; SM70-FTZ-NEXT: ld.param.b16 %rs1, [fmin_acq_rel_half_global_cta_param_1];
+; SM70-FTZ-NEXT: ld.param.b64 %rd2, [fmin_acq_rel_half_global_cta_param_0];
+; SM70-FTZ-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-FTZ-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-FTZ-NEXT: and.b32 %r5, %r4, 3;
+; SM70-FTZ-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-FTZ-NEXT: mov.b32 %r6, 65535;
+; SM70-FTZ-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-FTZ-NEXT: not.b32 %r2, %r7;
+; SM70-FTZ-NEXT: ld.relaxed.cta.global.b32 %r17, [%rd1];
+; SM70-FTZ-NEXT: cvt.ftz.f32.f16 %r10, %rs1;
+; SM70-FTZ-NEXT: $L__BB74_1: // %atomicrmw.start
+; SM70-FTZ-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-FTZ-NEXT: shr.u32 %r8, %r17, %r1;
+; SM70-FTZ-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM70-FTZ-NEXT: cvt.ftz.f32.f16 %r9, %rs2;
+; SM70-FTZ-NEXT: min.ftz.f32 %r11, %r9, %r10;
+; SM70-FTZ-NEXT: cvt.rn.f16.f32 %rs3, %r11;
+; SM70-FTZ-NEXT: cvt.u32.u16 %r12, %rs3;
+; SM70-FTZ-NEXT: shl.b32 %r13, %r12, %r1;
+; SM70-FTZ-NEXT: and.b32 %r14, %r17, %r2;
+; SM70-FTZ-NEXT: or.b32 %r15, %r14, %r13;
+; SM70-FTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r17, %r15;
+; SM70-FTZ-NEXT: setp.ne.b32 %p1, %r3, %r17;
+; SM70-FTZ-NEXT: mov.b32 %r17, %r3;
+; SM70-FTZ-NEXT: @%p1 bra $L__BB74_1;
+; SM70-FTZ-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-FTZ-NEXT: shr.u32 %r16, %r3, %r1;
+; SM70-FTZ-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-NEXT: st.param.b16 [func_retval0], %r16;
+; SM70-FTZ-NEXT: ret;
%retval = atomicrmw fmin ptr addrspace(1) %addr, half %val syncscope("block") acq_rel
ret half %retval
}
define half @fmax_acq_rel_half_global_cta(ptr addrspace(1) %addr, half %val) {
-; SM70-LABEL: fmax_acq_rel_half_global_cta(
-; SM70: {
-; SM70-NEXT: .reg .pred %p<2>;
-; SM70-NEXT: .reg .b16 %rs<4>;
-; SM70-NEXT: .reg .b32 %r<18>;
-; SM70-NEXT: .reg .b64 %rd<3>;
-; SM70-EMPTY:
-; SM70-NEXT: // %bb.0:
-; SM70-NEXT: ld.param.b16 %rs1, [fmax_acq_rel_half_global_cta_param_1];
-; SM70-NEXT: ld.param.b64 %rd2, [fmax_acq_rel_half_global_cta_param_0];
-; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: and.b64 %rd1, %rd2, -4;
-; SM70-NEXT: cvt.u32.u64 %r4, %rd2;
-; SM70-NEXT: and.b32 %r5, %r4, 3;
-; SM70-NEXT: shl.b32 %r1, %r5, 3;
-; SM70-NEXT: mov.b32 %r6, 65535;
-; SM70-NEXT: shl.b32 %r7, %r6, %r1;
-; SM70-NEXT: not.b32 %r2, %r7;
-; SM70-NEXT: ld.relaxed.cta.global.b32 %r17, [%rd1];
-; SM70-NEXT: cvt.f32.f16 %r10, %rs1;
-; SM70-NEXT: $L__BB75_1: // %atomicrmw.start
-; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: shr.u32 %r8, %r17, %r1;
-; SM70-NEXT: cvt.u16.u32 %rs2, %r8;
-; SM70-NEXT: cvt.f32.f16 %r9, %rs2;
-; SM70-NEXT: max.f32 %r11, %r9, %r10;
-; SM70-NEXT: cvt.rn.f16.f32 %rs3, %r11;
-; SM70-NEXT: cvt.u32.u16 %r12, %rs3;
-; SM70-NEXT: shl.b32 %r13, %r12, %r1;
-; SM70-NEXT: and.b32 %r14, %r17, %r2;
-; SM70-NEXT: or.b32 %r15, %r14, %r13;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r17, %r15;
-; SM70-NEXT: setp.ne.b32 %p1, %r3, %r17;
-; SM70-NEXT: mov.b32 %r17, %r3;
-; SM70-NEXT: @%p1 bra $L__BB75_1;
-; SM70-NEXT: // %bb.2: // %atomicrmw.end
-; SM70-NEXT: shr.u32 %r16, %r3, %r1;
-; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: st.param.b16 [func_retval0], %r16;
-; SM70-NEXT: ret;
+; SM70-NOFTZ-LABEL: fmax_acq_rel_half_global_cta(
+; SM70-NOFTZ: {
+; SM70-NOFTZ-NEXT: .reg .pred %p<2>;
+; SM70-NOFTZ-NEXT: .reg .b16 %rs<4>;
+; SM70-NOFTZ-NEXT: .reg .b32 %r<18>;
+; SM70-NOFTZ-NEXT: .reg .b64 %rd<3>;
+; SM70-NOFTZ-EMPTY:
+; SM70-NOFTZ-NEXT: // %bb.0:
+; SM70-NOFTZ-NEXT: ld.param.b16 %rs1, [fmax_acq_rel_half_global_cta_param_1];
+; SM70-NOFTZ-NEXT: ld.param.b64 %rd2, [fmax_acq_rel_half_global_cta_param_0];
+; SM70-NOFTZ-NEXT: fence.acq_rel.cta;
+; SM70-NOFTZ-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NOFTZ-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-NOFTZ-NEXT: and.b32 %r5, %r4, 3;
+; SM70-NOFTZ-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-NOFTZ-NEXT: mov.b32 %r6, 65535;
+; SM70-NOFTZ-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-NOFTZ-NEXT: not.b32 %r2, %r7;
+; SM70-NOFTZ-NEXT: ld.relaxed.cta.global.b32 %r17, [%rd1];
+; SM70-NOFTZ-NEXT: cvt.f32.f16 %r10, %rs1;
+; SM70-NOFTZ-NEXT: $L__BB75_1: // %atomicrmw.start
+; SM70-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NOFTZ-NEXT: shr.u32 %r8, %r17, %r1;
+; SM70-NOFTZ-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM70-NOFTZ-NEXT: cvt.f32.f16 %r9, %rs2;
+; SM70-NOFTZ-NEXT: max.f32 %r11, %r9, %r10;
+; SM70-NOFTZ-NEXT: cvt.rn.f16.f32 %rs3, %r11;
+; SM70-NOFTZ-NEXT: cvt.u32.u16 %r12, %rs3;
+; SM70-NOFTZ-NEXT: shl.b32 %r13, %r12, %r1;
+; SM70-NOFTZ-NEXT: and.b32 %r14, %r17, %r2;
+; SM70-NOFTZ-NEXT: or.b32 %r15, %r14, %r13;
+; SM70-NOFTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r17, %r15;
+; SM70-NOFTZ-NEXT: setp.ne.b32 %p1, %r3, %r17;
+; SM70-NOFTZ-NEXT: mov.b32 %r17, %r3;
+; SM70-NOFTZ-NEXT: @%p1 bra $L__BB75_1;
+; SM70-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NOFTZ-NEXT: shr.u32 %r16, %r3, %r1;
+; SM70-NOFTZ-NEXT: fence.acq_rel.cta;
+; SM70-NOFTZ-NEXT: st.param.b16 [func_retval0], %r16;
+; SM70-NOFTZ-NEXT: ret;
+;
+; SM70-FTZ-LABEL: fmax_acq_rel_half_global_cta(
+; SM70-FTZ: {
+; SM70-FTZ-NEXT: .reg .pred %p<2>;
+; SM70-FTZ-NEXT: .reg .b16 %rs<4>;
+; SM70-FTZ-NEXT: .reg .b32 %r<18>;
+; SM70-FTZ-NEXT: .reg .b64 %rd<3>;
+; SM70-FTZ-EMPTY:
+; SM70-FTZ-NEXT: // %bb.0:
+; SM70-FTZ-NEXT: ld.param.b16 %rs1, [fmax_acq_rel_half_global_cta_param_1];
+; SM70-FTZ-NEXT: ld.param.b64 %rd2, [fmax_acq_rel_half_global_cta_param_0];
+; SM70-FTZ-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-FTZ-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-FTZ-NEXT: and.b32 %r5, %r4, 3;
+; SM70-FTZ-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-FTZ-NEXT: mov.b32 %r6, 65535;
+; SM70-FTZ-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-FTZ-NEXT: not.b32 %r2, %r7;
+; SM70-FTZ-NEXT: ld.relaxed.cta.global.b32 %r17, [%rd1];
+; SM70-FTZ-NEXT: cvt.ftz.f32.f16 %r10, %rs1;
+; SM70-FTZ-NEXT: $L__BB75_1: // %atomicrmw.start
+; SM70-FTZ-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-FTZ-NEXT: shr.u32 %r8, %r17, %r1;
+; SM70-FTZ-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM70-FTZ-NEXT: cvt.ftz.f32.f16 %r9, %rs2;
+; SM70-FTZ-NEXT: max.ftz.f32 %r11, %r9, %r10;
+; SM70-FTZ-NEXT: cvt.rn.f16.f32 %rs3, %r11;
+; SM70-FTZ-NEXT: cvt.u32.u16 %r12, %rs3;
+; SM70-FTZ-NEXT: shl.b32 %r13, %r12, %r1;
+; SM70-FTZ-NEXT: and.b32 %r14, %r17, %r2;
+; SM70-FTZ-NEXT: or.b32 %r15, %r14, %r13;
+; SM70-FTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r17, %r15;
+; SM70-FTZ-NEXT: setp.ne.b32 %p1, %r3, %r17;
+; SM70-FTZ-NEXT: mov.b32 %r17, %r3;
+; SM70-FTZ-NEXT: @%p1 bra $L__BB75_1;
+; SM70-FTZ-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-FTZ-NEXT: shr.u32 %r16, %r3, %r1;
+; SM70-FTZ-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-NEXT: st.param.b16 [func_retval0], %r16;
+; SM70-FTZ-NEXT: ret;
%retval = atomicrmw fmax ptr addrspace(1) %addr, half %val syncscope("block") acq_rel
ret half %retval
}
define half @fminimum_acq_rel_half_global_cta(ptr addrspace(1) %addr, half %val) {
-; SM70-LABEL: fminimum_acq_rel_half_global_cta(
-; SM70: {
-; SM70-NEXT: .reg .pred %p<7>;
-; SM70-NEXT: .reg .b16 %rs<9>;
-; SM70-NEXT: .reg .b32 %r<15>;
-; SM70-NEXT: .reg .b64 %rd<3>;
-; SM70-EMPTY:
-; SM70-NEXT: // %bb.0:
-; SM70-NEXT: ld.param.b16 %rs1, [fminimum_acq_rel_half_global_cta_param_1];
-; SM70-NEXT: ld.param.b64 %rd2, [fminimum_acq_rel_half_global_cta_param_0];
-; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: and.b64 %rd1, %rd2, -4;
-; SM70-NEXT: cvt.u32.u64 %r4, %rd2;
-; SM70-NEXT: and.b32 %r5, %r4, 3;
-; SM70-NEXT: shl.b32 %r1, %r5, 3;
-; SM70-NEXT: mov.b32 %r6, 65535;
-; SM70-NEXT: shl.b32 %r7, %r6, %r1;
-; SM70-NEXT: not.b32 %r2, %r7;
-; SM70-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
-; SM70-NEXT: setp.eq.b16 %p4, %rs1, -32768;
-; SM70-NEXT: $L__BB76_1: // %atomicrmw.start
-; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: shr.u32 %r8, %r14, %r1;
-; SM70-NEXT: cvt.u16.u32 %rs2, %r8;
-; SM70-NEXT: setp.lt.f16 %p1, %rs2, %rs1;
-; SM70-NEXT: selp.b16 %rs3, %rs2, %rs1, %p1;
-; SM70-NEXT: setp.nan.f16 %p2, %rs2, %rs1;
-; SM70-NEXT: selp.b16 %rs4, 0x7E00, %rs3, %p2;
-; SM70-NEXT: setp.eq.b16 %p3, %rs2, -32768;
-; SM70-NEXT: selp.b16 %rs5, %rs2, %rs4, %p3;
-; SM70-NEXT: selp.b16 %rs6, %rs1, %rs5, %p4;
-; SM70-NEXT: mov.b16 %rs7, 0x0000;
-; SM70-NEXT: setp.eq.f16 %p5, %rs4, %rs7;
-; SM70-NEXT: selp.b16 %rs8, %rs6, %rs4, %p5;
-; SM70-NEXT: cvt.u32.u16 %r9, %rs8;
-; SM70-NEXT: shl.b32 %r10, %r9, %r1;
-; SM70-NEXT: and.b32 %r11, %r14, %r2;
-; SM70-NEXT: or.b32 %r12, %r11, %r10;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
-; SM70-NEXT: setp.ne.b32 %p6, %r3, %r14;
-; SM70-NEXT: mov.b32 %r14, %r3;
-; SM70-NEXT: @%p6 bra $L__BB76_1;
-; SM70-NEXT: // %bb.2: // %atomicrmw.end
-; SM70-NEXT: shr.u32 %r13, %r3, %r1;
-; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: st.param.b16 [func_retval0], %r13;
-; SM70-NEXT: ret;
+; SM70-NOFTZ-LABEL: fminimum_acq_rel_half_global_cta(
+; SM70-NOFTZ: {
+; SM70-NOFTZ-NEXT: .reg .pred %p<7>;
+; SM70-NOFTZ-NEXT: .reg .b16 %rs<9>;
+; SM70-NOFTZ-NEXT: .reg .b32 %r<15>;
+; SM70-NOFTZ-NEXT: .reg .b64 %rd<3>;
+; SM70-NOFTZ-EMPTY:
+; SM70-NOFTZ-NEXT: // %bb.0:
+; SM70-NOFTZ-NEXT: ld.param.b16 %rs1, [fminimum_acq_rel_half_global_cta_param_1];
+; SM70-NOFTZ-NEXT: ld.param.b64 %rd2, [fminimum_acq_rel_half_global_cta_param_0];
+; SM70-NOFTZ-NEXT: fence.acq_rel.cta;
+; SM70-NOFTZ-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NOFTZ-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-NOFTZ-NEXT: and.b32 %r5, %r4, 3;
+; SM70-NOFTZ-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-NOFTZ-NEXT: mov.b32 %r6, 65535;
+; SM70-NOFTZ-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-NOFTZ-NEXT: not.b32 %r2, %r7;
+; SM70-NOFTZ-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM70-NOFTZ-NEXT: setp.eq.b16 %p4, %rs1, -32768;
+; SM70-NOFTZ-NEXT: $L__BB76_1: // %atomicrmw.start
+; SM70-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NOFTZ-NEXT: shr.u32 %r8, %r14, %r1;
+; SM70-NOFTZ-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM70-NOFTZ-NEXT: setp.lt.f16 %p1, %rs2, %rs1;
+; SM70-NOFTZ-NEXT: selp.b16 %rs3, %rs2, %rs1, %p1;
+; SM70-NOFTZ-NEXT: setp.nan.f16 %p2, %rs2, %rs1;
+; SM70-NOFTZ-NEXT: selp.b16 %rs4, 0x7E00, %rs3, %p2;
+; SM70-NOFTZ-NEXT: setp.eq.b16 %p3, %rs2, -32768;
+; SM70-NOFTZ-NEXT: selp.b16 %rs5, %rs2, %rs4, %p3;
+; SM70-NOFTZ-NEXT: selp.b16 %rs6, %rs1, %rs5, %p4;
+; SM70-NOFTZ-NEXT: mov.b16 %rs7, 0x0000;
+; SM70-NOFTZ-NEXT: setp.eq.f16 %p5, %rs4, %rs7;
+; SM70-NOFTZ-NEXT: selp.b16 %rs8, %rs6, %rs4, %p5;
+; SM70-NOFTZ-NEXT: cvt.u32.u16 %r9, %rs8;
+; SM70-NOFTZ-NEXT: shl.b32 %r10, %r9, %r1;
+; SM70-NOFTZ-NEXT: and.b32 %r11, %r14, %r2;
+; SM70-NOFTZ-NEXT: or.b32 %r12, %r11, %r10;
+; SM70-NOFTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM70-NOFTZ-NEXT: setp.ne.b32 %p6, %r3, %r14;
+; SM70-NOFTZ-NEXT: mov.b32 %r14, %r3;
+; SM70-NOFTZ-NEXT: @%p6 bra $L__BB76_1;
+; SM70-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NOFTZ-NEXT: shr.u32 %r13, %r3, %r1;
+; SM70-NOFTZ-NEXT: fence.acq_rel.cta;
+; SM70-NOFTZ-NEXT: st.param.b16 [func_retval0], %r13;
+; SM70-NOFTZ-NEXT: ret;
+;
+; SM70-FTZ-LABEL: fminimum_acq_rel_half_global_cta(
+; SM70-FTZ: {
+; SM70-FTZ-NEXT: .reg .pred %p<7>;
+; SM70-FTZ-NEXT: .reg .b16 %rs<9>;
+; SM70-FTZ-NEXT: .reg .b32 %r<15>;
+; SM70-FTZ-NEXT: .reg .b64 %rd<3>;
+; SM70-FTZ-EMPTY:
+; SM70-FTZ-NEXT: // %bb.0:
+; SM70-FTZ-NEXT: ld.param.b16 %rs1, [fminimum_acq_rel_half_global_cta_param_1];
+; SM70-FTZ-NEXT: ld.param.b64 %rd2, [fminimum_acq_rel_half_global_cta_param_0];
+; SM70-FTZ-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-FTZ-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-FTZ-NEXT: and.b32 %r5, %r4, 3;
+; SM70-FTZ-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-FTZ-NEXT: mov.b32 %r6, 65535;
+; SM70-FTZ-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-FTZ-NEXT: not.b32 %r2, %r7;
+; SM70-FTZ-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM70-FTZ-NEXT: setp.eq.b16 %p4, %rs1, -32768;
+; SM70-FTZ-NEXT: $L__BB76_1: // %atomicrmw.start
+; SM70-FTZ-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-FTZ-NEXT: shr.u32 %r8, %r14, %r1;
+; SM70-FTZ-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM70-FTZ-NEXT: setp.lt.ftz.f16 %p1, %rs2, %rs1;
+; SM70-FTZ-NEXT: selp.b16 %rs3, %rs2, %rs1, %p1;
+; SM70-FTZ-NEXT: setp.nan.ftz.f16 %p2, %rs2, %rs1;
+; SM70-FTZ-NEXT: selp.b16 %rs4, 0x7E00, %rs3, %p2;
+; SM70-FTZ-NEXT: setp.eq.b16 %p3, %rs2, -32768;
+; SM70-FTZ-NEXT: selp.b16 %rs5, %rs2, %rs4, %p3;
+; SM70-FTZ-NEXT: selp.b16 %rs6, %rs1, %rs5, %p4;
+; SM70-FTZ-NEXT: mov.b16 %rs7, 0x0000;
+; SM70-FTZ-NEXT: setp.eq.ftz.f16 %p5, %rs4, %rs7;
+; SM70-FTZ-NEXT: selp.b16 %rs8, %rs6, %rs4, %p5;
+; SM70-FTZ-NEXT: cvt.u32.u16 %r9, %rs8;
+; SM70-FTZ-NEXT: shl.b32 %r10, %r9, %r1;
+; SM70-FTZ-NEXT: and.b32 %r11, %r14, %r2;
+; SM70-FTZ-NEXT: or.b32 %r12, %r11, %r10;
+; SM70-FTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM70-FTZ-NEXT: setp.ne.b32 %p6, %r3, %r14;
+; SM70-FTZ-NEXT: mov.b32 %r14, %r3;
+; SM70-FTZ-NEXT: @%p6 bra $L__BB76_1;
+; SM70-FTZ-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-FTZ-NEXT: shr.u32 %r13, %r3, %r1;
+; SM70-FTZ-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-NEXT: st.param.b16 [func_retval0], %r13;
+; SM70-FTZ-NEXT: ret;
%retval = atomicrmw fminimum ptr addrspace(1) %addr, half %val syncscope("block") acq_rel
ret half %retval
}
define half @fmaximum_acq_rel_half_global_cta(ptr addrspace(1) %addr, half %val) {
-; SM70-LABEL: fmaximum_acq_rel_half_global_cta(
-; SM70: {
-; SM70-NEXT: .reg .pred %p<7>;
-; SM70-NEXT: .reg .b16 %rs<9>;
-; SM70-NEXT: .reg .b32 %r<15>;
-; SM70-NEXT: .reg .b64 %rd<3>;
-; SM70-EMPTY:
-; SM70-NEXT: // %bb.0:
-; SM70-NEXT: ld.param.b16 %rs1, [fmaximum_acq_rel_half_global_cta_param_1];
-; SM70-NEXT: ld.param.b64 %rd2, [fmaximum_acq_rel_half_global_cta_param_0];
-; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: and.b64 %rd1, %rd2, -4;
-; SM70-NEXT: cvt.u32.u64 %r4, %rd2;
-; SM70-NEXT: and.b32 %r5, %r4, 3;
-; SM70-NEXT: shl.b32 %r1, %r5, 3;
-; SM70-NEXT: mov.b32 %r6, 65535;
-; SM70-NEXT: shl.b32 %r7, %r6, %r1;
-; SM70-NEXT: not.b32 %r2, %r7;
-; SM70-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
-; SM70-NEXT: setp.eq.b16 %p4, %rs1, 0;
-; SM70-NEXT: $L__BB77_1: // %atomicrmw.start
-; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: shr.u32 %r8, %r14, %r1;
-; SM70-NEXT: cvt.u16.u32 %rs2, %r8;
-; SM70-NEXT: setp.gt.f16 %p1, %rs2, %rs1;
-; SM70-NEXT: selp.b16 %rs3, %rs2, %rs1, %p1;
-; SM70-NEXT: setp.nan.f16 %p2, %rs2, %rs1;
-; SM70-NEXT: selp.b16 %rs4, 0x7E00, %rs3, %p2;
-; SM70-NEXT: setp.eq.b16 %p3, %rs2, 0;
-; SM70-NEXT: selp.b16 %rs5, %rs2, %rs4, %p3;
-; SM70-NEXT: selp.b16 %rs6, %rs1, %rs5, %p4;
-; SM70-NEXT: mov.b16 %rs7, 0x0000;
-; SM70-NEXT: setp.eq.f16 %p5, %rs4, %rs7;
-; SM70-NEXT: selp.b16 %rs8, %rs6, %rs4, %p5;
-; SM70-NEXT: cvt.u32.u16 %r9, %rs8;
-; SM70-NEXT: shl.b32 %r10, %r9, %r1;
-; SM70-NEXT: and.b32 %r11, %r14, %r2;
-; SM70-NEXT: or.b32 %r12, %r11, %r10;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
-; SM70-NEXT: setp.ne.b32 %p6, %r3, %r14;
-; SM70-NEXT: mov.b32 %r14, %r3;
-; SM70-NEXT: @%p6 bra $L__BB77_1;
-; SM70-NEXT: // %bb.2: // %atomicrmw.end
-; SM70-NEXT: shr.u32 %r13, %r3, %r1;
-; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: st.param.b16 [func_retval0], %r13;
-; SM70-NEXT: ret;
+; SM70-NOFTZ-LABEL: fmaximum_acq_rel_half_global_cta(
+; SM70-NOFTZ: {
+; SM70-NOFTZ-NEXT: .reg .pred %p<7>;
+; SM70-NOFTZ-NEXT: .reg .b16 %rs<9>;
+; SM70-NOFTZ-NEXT: .reg .b32 %r<15>;
+; SM70-NOFTZ-NEXT: .reg .b64 %rd<3>;
+; SM70-NOFTZ-EMPTY:
+; SM70-NOFTZ-NEXT: // %bb.0:
+; SM70-NOFTZ-NEXT: ld.param.b16 %rs1, [fmaximum_acq_rel_half_global_cta_param_1];
+; SM70-NOFTZ-NEXT: ld.param.b64 %rd2, [fmaximum_acq_rel_half_global_cta_param_0];
+; SM70-NOFTZ-NEXT: fence.acq_rel.cta;
+; SM70-NOFTZ-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NOFTZ-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-NOFTZ-NEXT: and.b32 %r5, %r4, 3;
+; SM70-NOFTZ-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-NOFTZ-NEXT: mov.b32 %r6, 65535;
+; SM70-NOFTZ-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-NOFTZ-NEXT: not.b32 %r2, %r7;
+; SM70-NOFTZ-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM70-NOFTZ-NEXT: setp.eq.b16 %p4, %rs1, 0;
+; SM70-NOFTZ-NEXT: $L__BB77_1: // %atomicrmw.start
+; SM70-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NOFTZ-NEXT: shr.u32 %r8, %r14, %r1;
+; SM70-NOFTZ-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM70-NOFTZ-NEXT: setp.gt.f16 %p1, %rs2, %rs1;
+; SM70-NOFTZ-NEXT: selp.b16 %rs3, %rs2, %rs1, %p1;
+; SM70-NOFTZ-NEXT: setp.nan.f16 %p2, %rs2, %rs1;
+; SM70-NOFTZ-NEXT: selp.b16 %rs4, 0x7E00, %rs3, %p2;
+; SM70-NOFTZ-NEXT: setp.eq.b16 %p3, %rs2, 0;
+; SM70-NOFTZ-NEXT: selp.b16 %rs5, %rs2, %rs4, %p3;
+; SM70-NOFTZ-NEXT: selp.b16 %rs6, %rs1, %rs5, %p4;
+; SM70-NOFTZ-NEXT: mov.b16 %rs7, 0x0000;
+; SM70-NOFTZ-NEXT: setp.eq.f16 %p5, %rs4, %rs7;
+; SM70-NOFTZ-NEXT: selp.b16 %rs8, %rs6, %rs4, %p5;
+; SM70-NOFTZ-NEXT: cvt.u32.u16 %r9, %rs8;
+; SM70-NOFTZ-NEXT: shl.b32 %r10, %r9, %r1;
+; SM70-NOFTZ-NEXT: and.b32 %r11, %r14, %r2;
+; SM70-NOFTZ-NEXT: or.b32 %r12, %r11, %r10;
+; SM70-NOFTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM70-NOFTZ-NEXT: setp.ne.b32 %p6, %r3, %r14;
+; SM70-NOFTZ-NEXT: mov.b32 %r14, %r3;
+; SM70-NOFTZ-NEXT: @%p6 bra $L__BB77_1;
+; SM70-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NOFTZ-NEXT: shr.u32 %r13, %r3, %r1;
+; SM70-NOFTZ-NEXT: fence.acq_rel.cta;
+; SM70-NOFTZ-NEXT: st.param.b16 [func_retval0], %r13;
+; SM70-NOFTZ-NEXT: ret;
+;
+; SM70-FTZ-LABEL: fmaximum_acq_rel_half_global_cta(
+; SM70-FTZ: {
+; SM70-FTZ-NEXT: .reg .pred %p<7>;
+; SM70-FTZ-NEXT: .reg .b16 %rs<9>;
+; SM70-FTZ-NEXT: .reg .b32 %r<15>;
+; SM70-FTZ-NEXT: .reg .b64 %rd<3>;
+; SM70-FTZ-EMPTY:
+; SM70-FTZ-NEXT: // %bb.0:
+; SM70-FTZ-NEXT: ld.param.b16 %rs1, [fmaximum_acq_rel_half_global_cta_param_1];
+; SM70-FTZ-NEXT: ld.param.b64 %rd2, [fmaximum_acq_rel_half_global_cta_param_0];
+; SM70-FTZ-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-FTZ-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-FTZ-NEXT: and.b32 %r5, %r4, 3;
+; SM70-FTZ-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-FTZ-NEXT: mov.b32 %r6, 65535;
+; SM70-FTZ-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-FTZ-NEXT: not.b32 %r2, %r7;
+; SM70-FTZ-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM70-FTZ-NEXT: setp.eq.b16 %p4, %rs1, 0;
+; SM70-FTZ-NEXT: $L__BB77_1: // %atomicrmw.start
+; SM70-FTZ-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-FTZ-NEXT: shr.u32 %r8, %r14, %r1;
+; SM70-FTZ-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM70-FTZ-NEXT: setp.gt.ftz.f16 %p1, %rs2, %rs1;
+; SM70-FTZ-NEXT: selp.b16 %rs3, %rs2, %rs1, %p1;
+; SM70-FTZ-NEXT: setp.nan.ftz.f16 %p2, %rs2, %rs1;
+; SM70-FTZ-NEXT: selp.b16 %rs4, 0x7E00, %rs3, %p2;
+; SM70-FTZ-NEXT: setp.eq.b16 %p3, %rs2, 0;
+; SM70-FTZ-NEXT: selp.b16 %rs5, %rs2, %rs4, %p3;
+; SM70-FTZ-NEXT: selp.b16 %rs6, %rs1, %rs5, %p4;
+; SM70-FTZ-NEXT: mov.b16 %rs7, 0x0000;
+; SM70-FTZ-NEXT: setp.eq.ftz.f16 %p5, %rs4, %rs7;
+; SM70-FTZ-NEXT: selp.b16 %rs8, %rs6, %rs4, %p5;
+; SM70-FTZ-NEXT: cvt.u32.u16 %r9, %rs8;
+; SM70-FTZ-NEXT: shl.b32 %r10, %r9, %r1;
+; SM70-FTZ-NEXT: and.b32 %r11, %r14, %r2;
+; SM70-FTZ-NEXT: or.b32 %r12, %r11, %r10;
+; SM70-FTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM70-FTZ-NEXT: setp.ne.b32 %p6, %r3, %r14;
+; SM70-FTZ-NEXT: mov.b32 %r14, %r3;
+; SM70-FTZ-NEXT: @%p6 bra $L__BB77_1;
+; SM70-FTZ-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-FTZ-NEXT: shr.u32 %r13, %r3, %r1;
+; SM70-FTZ-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-NEXT: st.param.b16 [func_retval0], %r13;
+; SM70-FTZ-NEXT: ret;
%retval = atomicrmw fmaximum ptr addrspace(1) %addr, half %val syncscope("block") acq_rel
ret half %retval
}
define bfloat @fadd_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat %val) {
-; SM70-LABEL: fadd_acq_rel_bfloat_global_cta(
-; SM70: {
-; SM70-NEXT: .reg .pred %p<3>;
-; SM70-NEXT: .reg .b16 %rs<2>;
-; SM70-NEXT: .reg .b32 %r<24>;
-; SM70-NEXT: .reg .b64 %rd<3>;
-; SM70-EMPTY:
-; SM70-NEXT: // %bb.0:
-; SM70-NEXT: ld.param.b16 %rs1, [fadd_acq_rel_bfloat_global_cta_param_1];
-; SM70-NEXT: ld.param.b64 %rd2, [fadd_acq_rel_bfloat_global_cta_param_0];
-; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: and.b64 %rd1, %rd2, -4;
-; SM70-NEXT: cvt.u32.u64 %r4, %rd2;
-; SM70-NEXT: and.b32 %r5, %r4, 3;
-; SM70-NEXT: shl.b32 %r1, %r5, 3;
-; SM70-NEXT: mov.b32 %r6, 65535;
-; SM70-NEXT: shl.b32 %r7, %r6, %r1;
-; SM70-NEXT: not.b32 %r2, %r7;
-; SM70-NEXT: ld.relaxed.cta.global.b32 %r23, [%rd1];
-; SM70-NEXT: cvt.u32.u16 %r10, %rs1;
-; SM70-NEXT: shl.b32 %r11, %r10, 16;
-; SM70-NEXT: $L__BB78_1: // %atomicrmw.start
-; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: shr.u32 %r8, %r23, %r1;
-; SM70-NEXT: shl.b32 %r9, %r8, 16;
-; SM70-NEXT: add.rn.f32 %r12, %r9, %r11;
-; SM70-NEXT: bfe.u32 %r13, %r12, 16, 1;
-; SM70-NEXT: add.s32 %r14, %r13, %r12;
-; SM70-NEXT: add.s32 %r15, %r14, 32767;
-; SM70-NEXT: setp.nan.f32 %p1, %r12, %r12;
-; SM70-NEXT: or.b32 %r16, %r12, 4194304;
-; SM70-NEXT: selp.b32 %r17, %r16, %r15, %p1;
-; SM70-NEXT: shr.u32 %r18, %r17, 16;
-; SM70-NEXT: shl.b32 %r19, %r18, %r1;
-; SM70-NEXT: and.b32 %r20, %r23, %r2;
-; SM70-NEXT: or.b32 %r21, %r20, %r19;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r23, %r21;
-; SM70-NEXT: setp.ne.b32 %p2, %r3, %r23;
-; SM70-NEXT: mov.b32 %r23, %r3;
-; SM70-NEXT: @%p2 bra $L__BB78_1;
-; SM70-NEXT: // %bb.2: // %atomicrmw.end
-; SM70-NEXT: shr.u32 %r22, %r3, %r1;
-; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: st.param.b16 [func_retval0], %r22;
-; SM70-NEXT: ret;
+; SM70-NOFTZ-LABEL: fadd_acq_rel_bfloat_global_cta(
+; SM70-NOFTZ: {
+; SM70-NOFTZ-NEXT: .reg .pred %p<3>;
+; SM70-NOFTZ-NEXT: .reg .b16 %rs<2>;
+; SM70-NOFTZ-NEXT: .reg .b32 %r<24>;
+; SM70-NOFTZ-NEXT: .reg .b64 %rd<3>;
+; SM70-NOFTZ-EMPTY:
+; SM70-NOFTZ-NEXT: // %bb.0:
+; SM70-NOFTZ-NEXT: ld.param.b16 %rs1, [fadd_acq_rel_bfloat_global_cta_param_1];
+; SM70-NOFTZ-NEXT: ld.param.b64 %rd2, [fadd_acq_rel_bfloat_global_cta_param_0];
+; SM70-NOFTZ-NEXT: fence.acq_rel.cta;
+; SM70-NOFTZ-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NOFTZ-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-NOFTZ-NEXT: and.b32 %r5, %r4, 3;
+; SM70-NOFTZ-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-NOFTZ-NEXT: mov.b32 %r6, 65535;
+; SM70-NOFTZ-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-NOFTZ-NEXT: not.b32 %r2, %r7;
+; SM70-NOFTZ-NEXT: ld.relaxed.cta.global.b32 %r23, [%rd1];
+; SM70-NOFTZ-NEXT: cvt.u32.u16 %r10, %rs1;
+; SM70-NOFTZ-NEXT: shl.b32 %r11, %r10, 16;
+; SM70-NOFTZ-NEXT: $L__BB78_1: // %atomicrmw.start
+; SM70-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NOFTZ-NEXT: shr.u32 %r8, %r23, %r1;
+; SM70-NOFTZ-NEXT: shl.b32 %r9, %r8, 16;
+; SM70-NOFTZ-NEXT: add.rn.f32 %r12, %r9, %r11;
+; SM70-NOFTZ-NEXT: bfe.u32 %r13, %r12, 16, 1;
+; SM70-NOFTZ-NEXT: add.s32 %r14, %r13, %r12;
+; SM70-NOFTZ-NEXT: add.s32 %r15, %r14, 32767;
+; SM70-NOFTZ-NEXT: setp.nan.f32 %p1, %r12, %r12;
+; SM70-NOFTZ-NEXT: or.b32 %r16, %r12, 4194304;
+; SM70-NOFTZ-NEXT: selp.b32 %r17, %r16, %r15, %p1;
+; SM70-NOFTZ-NEXT: shr.u32 %r18, %r17, 16;
+; SM70-NOFTZ-NEXT: shl.b32 %r19, %r18, %r1;
+; SM70-NOFTZ-NEXT: and.b32 %r20, %r23, %r2;
+; SM70-NOFTZ-NEXT: or.b32 %r21, %r20, %r19;
+; SM70-NOFTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r23, %r21;
+; SM70-NOFTZ-NEXT: setp.ne.b32 %p2, %r3, %r23;
+; SM70-NOFTZ-NEXT: mov.b32 %r23, %r3;
+; SM70-NOFTZ-NEXT: @%p2 bra $L__BB78_1;
+; SM70-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NOFTZ-NEXT: shr.u32 %r22, %r3, %r1;
+; SM70-NOFTZ-NEXT: fence.acq_rel.cta;
+; SM70-NOFTZ-NEXT: st.param.b16 [func_retval0], %r22;
+; SM70-NOFTZ-NEXT: ret;
+;
+; SM70-FTZ-LABEL: fadd_acq_rel_bfloat_global_cta(
+; SM70-FTZ: {
+; SM70-FTZ-NEXT: .reg .pred %p<3>;
+; SM70-FTZ-NEXT: .reg .b16 %rs<2>;
+; SM70-FTZ-NEXT: .reg .b32 %r<24>;
+; SM70-FTZ-NEXT: .reg .b64 %rd<3>;
+; SM70-FTZ-EMPTY:
+; SM70-FTZ-NEXT: // %bb.0:
+; SM70-FTZ-NEXT: ld.param.b16 %rs1, [fadd_acq_rel_bfloat_global_cta_param_1];
+; SM70-FTZ-NEXT: ld.param.b64 %rd2, [fadd_acq_rel_bfloat_global_cta_param_0];
+; SM70-FTZ-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-FTZ-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-FTZ-NEXT: and.b32 %r5, %r4, 3;
+; SM70-FTZ-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-FTZ-NEXT: mov.b32 %r6, 65535;
+; SM70-FTZ-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-FTZ-NEXT: not.b32 %r2, %r7;
+; SM70-FTZ-NEXT: ld.relaxed.cta.global.b32 %r23, [%rd1];
+; SM70-FTZ-NEXT: cvt.u32.u16 %r10, %rs1;
+; SM70-FTZ-NEXT: shl.b32 %r11, %r10, 16;
+; SM70-FTZ-NEXT: $L__BB78_1: // %atomicrmw.start
+; SM70-FTZ-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-FTZ-NEXT: shr.u32 %r8, %r23, %r1;
+; SM70-FTZ-NEXT: shl.b32 %r9, %r8, 16;
+; SM70-FTZ-NEXT: add.rn.ftz.f32 %r12, %r9, %r11;
+; SM70-FTZ-NEXT: bfe.u32 %r13, %r12, 16, 1;
+; SM70-FTZ-NEXT: add.s32 %r14, %r13, %r12;
+; SM70-FTZ-NEXT: add.s32 %r15, %r14, 32767;
+; SM70-FTZ-NEXT: setp.nan.ftz.f32 %p1, %r12, %r12;
+; SM70-FTZ-NEXT: or.b32 %r16, %r12, 4194304;
+; SM70-FTZ-NEXT: selp.b32 %r17, %r16, %r15, %p1;
+; SM70-FTZ-NEXT: shr.u32 %r18, %r17, 16;
+; SM70-FTZ-NEXT: shl.b32 %r19, %r18, %r1;
+; SM70-FTZ-NEXT: and.b32 %r20, %r23, %r2;
+; SM70-FTZ-NEXT: or.b32 %r21, %r20, %r19;
+; SM70-FTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r23, %r21;
+; SM70-FTZ-NEXT: setp.ne.b32 %p2, %r3, %r23;
+; SM70-FTZ-NEXT: mov.b32 %r23, %r3;
+; SM70-FTZ-NEXT: @%p2 bra $L__BB78_1;
+; SM70-FTZ-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-FTZ-NEXT: shr.u32 %r22, %r3, %r1;
+; SM70-FTZ-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-NEXT: st.param.b16 [func_retval0], %r22;
+; SM70-FTZ-NEXT: ret;
%retval = atomicrmw fadd ptr addrspace(1) %addr, bfloat %val syncscope("block") acq_rel
ret bfloat %retval
}
define bfloat @fsub_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat %val) {
-; SM70-LABEL: fsub_acq_rel_bfloat_global_cta(
-; SM70: {
-; SM70-NEXT: .reg .pred %p<3>;
-; SM70-NEXT: .reg .b16 %rs<2>;
-; SM70-NEXT: .reg .b32 %r<24>;
-; SM70-NEXT: .reg .b64 %rd<3>;
-; SM70-EMPTY:
-; SM70-NEXT: // %bb.0:
-; SM70-NEXT: ld.param.b16 %rs1, [fsub_acq_rel_bfloat_global_cta_param_1];
-; SM70-NEXT: ld.param.b64 %rd2, [fsub_acq_rel_bfloat_global_cta_param_0];
-; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: and.b64 %rd1, %rd2, -4;
-; SM70-NEXT: cvt.u32.u64 %r4, %rd2;
-; SM70-NEXT: and.b32 %r5, %r4, 3;
-; SM70-NEXT: shl.b32 %r1, %r5, 3;
-; SM70-NEXT: mov.b32 %r6, 65535;
-; SM70-NEXT: shl.b32 %r7, %r6, %r1;
-; SM70-NEXT: not.b32 %r2, %r7;
-; SM70-NEXT: ld.relaxed.cta.global.b32 %r23, [%rd1];
-; SM70-NEXT: cvt.u32.u16 %r10, %rs1;
-; SM70-NEXT: shl.b32 %r11, %r10, 16;
-; SM70-NEXT: $L__BB79_1: // %atomicrmw.start
-; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: shr.u32 %r8, %r23, %r1;
-; SM70-NEXT: shl.b32 %r9, %r8, 16;
-; SM70-NEXT: sub.rn.f32 %r12, %r9, %r11;
-; SM70-NEXT: bfe.u32 %r13, %r12, 16, 1;
-; SM70-NEXT: add.s32 %r14, %r13, %r12;
-; SM70-NEXT: add.s32 %r15, %r14, 32767;
-; SM70-NEXT: setp.nan.f32 %p1, %r12, %r12;
-; SM70-NEXT: or.b32 %r16, %r12, 4194304;
-; SM70-NEXT: selp.b32 %r17, %r16, %r15, %p1;
-; SM70-NEXT: shr.u32 %r18, %r17, 16;
-; SM70-NEXT: shl.b32 %r19, %r18, %r1;
-; SM70-NEXT: and.b32 %r20, %r23, %r2;
-; SM70-NEXT: or.b32 %r21, %r20, %r19;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r23, %r21;
-; SM70-NEXT: setp.ne.b32 %p2, %r3, %r23;
-; SM70-NEXT: mov.b32 %r23, %r3;
-; SM70-NEXT: @%p2 bra $L__BB79_1;
-; SM70-NEXT: // %bb.2: // %atomicrmw.end
-; SM70-NEXT: shr.u32 %r22, %r3, %r1;
-; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: st.param.b16 [func_retval0], %r22;
-; SM70-NEXT: ret;
+; SM70-NOFTZ-LABEL: fsub_acq_rel_bfloat_global_cta(
+; SM70-NOFTZ: {
+; SM70-NOFTZ-NEXT: .reg .pred %p<3>;
+; SM70-NOFTZ-NEXT: .reg .b16 %rs<2>;
+; SM70-NOFTZ-NEXT: .reg .b32 %r<24>;
+; SM70-NOFTZ-NEXT: .reg .b64 %rd<3>;
+; SM70-NOFTZ-EMPTY:
+; SM70-NOFTZ-NEXT: // %bb.0:
+; SM70-NOFTZ-NEXT: ld.param.b16 %rs1, [fsub_acq_rel_bfloat_global_cta_param_1];
+; SM70-NOFTZ-NEXT: ld.param.b64 %rd2, [fsub_acq_rel_bfloat_global_cta_param_0];
+; SM70-NOFTZ-NEXT: fence.acq_rel.cta;
+; SM70-NOFTZ-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NOFTZ-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-NOFTZ-NEXT: and.b32 %r5, %r4, 3;
+; SM70-NOFTZ-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-NOFTZ-NEXT: mov.b32 %r6, 65535;
+; SM70-NOFTZ-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-NOFTZ-NEXT: not.b32 %r2, %r7;
+; SM70-NOFTZ-NEXT: ld.relaxed.cta.global.b32 %r23, [%rd1];
+; SM70-NOFTZ-NEXT: cvt.u32.u16 %r10, %rs1;
+; SM70-NOFTZ-NEXT: shl.b32 %r11, %r10, 16;
+; SM70-NOFTZ-NEXT: $L__BB79_1: // %atomicrmw.start
+; SM70-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NOFTZ-NEXT: shr.u32 %r8, %r23, %r1;
+; SM70-NOFTZ-NEXT: shl.b32 %r9, %r8, 16;
+; SM70-NOFTZ-NEXT: sub.rn.f32 %r12, %r9, %r11;
+; SM70-NOFTZ-NEXT: bfe.u32 %r13, %r12, 16, 1;
+; SM70-NOFTZ-NEXT: add.s32 %r14, %r13, %r12;
+; SM70-NOFTZ-NEXT: add.s32 %r15, %r14, 32767;
+; SM70-NOFTZ-NEXT: setp.nan.f32 %p1, %r12, %r12;
+; SM70-NOFTZ-NEXT: or.b32 %r16, %r12, 4194304;
+; SM70-NOFTZ-NEXT: selp.b32 %r17, %r16, %r15, %p1;
+; SM70-NOFTZ-NEXT: shr.u32 %r18, %r17, 16;
+; SM70-NOFTZ-NEXT: shl.b32 %r19, %r18, %r1;
+; SM70-NOFTZ-NEXT: and.b32 %r20, %r23, %r2;
+; SM70-NOFTZ-NEXT: or.b32 %r21, %r20, %r19;
+; SM70-NOFTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r23, %r21;
+; SM70-NOFTZ-NEXT: setp.ne.b32 %p2, %r3, %r23;
+; SM70-NOFTZ-NEXT: mov.b32 %r23, %r3;
+; SM70-NOFTZ-NEXT: @%p2 bra $L__BB79_1;
+; SM70-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NOFTZ-NEXT: shr.u32 %r22, %r3, %r1;
+; SM70-NOFTZ-NEXT: fence.acq_rel.cta;
+; SM70-NOFTZ-NEXT: st.param.b16 [func_retval0], %r22;
+; SM70-NOFTZ-NEXT: ret;
+;
+; SM70-FTZ-LABEL: fsub_acq_rel_bfloat_global_cta(
+; SM70-FTZ: {
+; SM70-FTZ-NEXT: .reg .pred %p<3>;
+; SM70-FTZ-NEXT: .reg .b16 %rs<2>;
+; SM70-FTZ-NEXT: .reg .b32 %r<24>;
+; SM70-FTZ-NEXT: .reg .b64 %rd<3>;
+; SM70-FTZ-EMPTY:
+; SM70-FTZ-NEXT: // %bb.0:
+; SM70-FTZ-NEXT: ld.param.b16 %rs1, [fsub_acq_rel_bfloat_global_cta_param_1];
+; SM70-FTZ-NEXT: ld.param.b64 %rd2, [fsub_acq_rel_bfloat_global_cta_param_0];
+; SM70-FTZ-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-FTZ-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-FTZ-NEXT: and.b32 %r5, %r4, 3;
+; SM70-FTZ-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-FTZ-NEXT: mov.b32 %r6, 65535;
+; SM70-FTZ-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-FTZ-NEXT: not.b32 %r2, %r7;
+; SM70-FTZ-NEXT: ld.relaxed.cta.global.b32 %r23, [%rd1];
+; SM70-FTZ-NEXT: cvt.u32.u16 %r10, %rs1;
+; SM70-FTZ-NEXT: shl.b32 %r11, %r10, 16;
+; SM70-FTZ-NEXT: $L__BB79_1: // %atomicrmw.start
+; SM70-FTZ-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-FTZ-NEXT: shr.u32 %r8, %r23, %r1;
+; SM70-FTZ-NEXT: shl.b32 %r9, %r8, 16;
+; SM70-FTZ-NEXT: sub.rn.ftz.f32 %r12, %r9, %r11;
+; SM70-FTZ-NEXT: bfe.u32 %r13, %r12, 16, 1;
+; SM70-FTZ-NEXT: add.s32 %r14, %r13, %r12;
+; SM70-FTZ-NEXT: add.s32 %r15, %r14, 32767;
+; SM70-FTZ-NEXT: setp.nan.ftz.f32 %p1, %r12, %r12;
+; SM70-FTZ-NEXT: or.b32 %r16, %r12, 4194304;
+; SM70-FTZ-NEXT: selp.b32 %r17, %r16, %r15, %p1;
+; SM70-FTZ-NEXT: shr.u32 %r18, %r17, 16;
+; SM70-FTZ-NEXT: shl.b32 %r19, %r18, %r1;
+; SM70-FTZ-NEXT: and.b32 %r20, %r23, %r2;
+; SM70-FTZ-NEXT: or.b32 %r21, %r20, %r19;
+; SM70-FTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r23, %r21;
+; SM70-FTZ-NEXT: setp.ne.b32 %p2, %r3, %r23;
+; SM70-FTZ-NEXT: mov.b32 %r23, %r3;
+; SM70-FTZ-NEXT: @%p2 bra $L__BB79_1;
+; SM70-FTZ-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-FTZ-NEXT: shr.u32 %r22, %r3, %r1;
+; SM70-FTZ-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-NEXT: st.param.b16 [func_retval0], %r22;
+; SM70-FTZ-NEXT: ret;
%retval = atomicrmw fsub ptr addrspace(1) %addr, bfloat %val syncscope("block") acq_rel
ret bfloat %retval
}
define bfloat @fmin_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat %val) {
-; SM70-LABEL: fmin_acq_rel_bfloat_global_cta(
-; SM70: {
-; SM70-NEXT: .reg .pred %p<3>;
-; SM70-NEXT: .reg .b16 %rs<2>;
-; SM70-NEXT: .reg .b32 %r<24>;
-; SM70-NEXT: .reg .b64 %rd<3>;
-; SM70-EMPTY:
-; SM70-NEXT: // %bb.0:
-; SM70-NEXT: ld.param.b16 %rs1, [fmin_acq_rel_bfloat_global_cta_param_1];
-; SM70-NEXT: ld.param.b64 %rd2, [fmin_acq_rel_bfloat_global_cta_param_0];
-; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: and.b64 %rd1, %rd2, -4;
-; SM70-NEXT: cvt.u32.u64 %r4, %rd2;
-; SM70-NEXT: and.b32 %r5, %r4, 3;
-; SM70-NEXT: shl.b32 %r1, %r5, 3;
-; SM70-NEXT: mov.b32 %r6, 65535;
-; SM70-NEXT: shl.b32 %r7, %r6, %r1;
-; SM70-NEXT: not.b32 %r2, %r7;
-; SM70-NEXT: ld.relaxed.cta.global.b32 %r23, [%rd1];
-; SM70-NEXT: cvt.u32.u16 %r10, %rs1;
-; SM70-NEXT: shl.b32 %r11, %r10, 16;
-; SM70-NEXT: $L__BB80_1: // %atomicrmw.start
-; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: shr.u32 %r8, %r23, %r1;
-; SM70-NEXT: shl.b32 %r9, %r8, 16;
-; SM70-NEXT: min.f32 %r12, %r9, %r11;
-; SM70-NEXT: bfe.u32 %r13, %r12, 16, 1;
-; SM70-NEXT: add.s32 %r14, %r13, %r12;
-; SM70-NEXT: add.s32 %r15, %r14, 32767;
-; SM70-NEXT: setp.nan.f32 %p1, %r12, %r12;
-; SM70-NEXT: or.b32 %r16, %r12, 4194304;
-; SM70-NEXT: selp.b32 %r17, %r16, %r15, %p1;
-; SM70-NEXT: shr.u32 %r18, %r17, 16;
-; SM70-NEXT: shl.b32 %r19, %r18, %r1;
-; SM70-NEXT: and.b32 %r20, %r23, %r2;
-; SM70-NEXT: or.b32 %r21, %r20, %r19;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r23, %r21;
-; SM70-NEXT: setp.ne.b32 %p2, %r3, %r23;
-; SM70-NEXT: mov.b32 %r23, %r3;
-; SM70-NEXT: @%p2 bra $L__BB80_1;
-; SM70-NEXT: // %bb.2: // %atomicrmw.end
-; SM70-NEXT: shr.u32 %r22, %r3, %r1;
-; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: st.param.b16 [func_retval0], %r22;
-; SM70-NEXT: ret;
+; SM70-NOFTZ-LABEL: fmin_acq_rel_bfloat_global_cta(
+; SM70-NOFTZ: {
+; SM70-NOFTZ-NEXT: .reg .pred %p<3>;
+; SM70-NOFTZ-NEXT: .reg .b16 %rs<2>;
+; SM70-NOFTZ-NEXT: .reg .b32 %r<24>;
+; SM70-NOFTZ-NEXT: .reg .b64 %rd<3>;
+; SM70-NOFTZ-EMPTY:
+; SM70-NOFTZ-NEXT: // %bb.0:
+; SM70-NOFTZ-NEXT: ld.param.b16 %rs1, [fmin_acq_rel_bfloat_global_cta_param_1];
+; SM70-NOFTZ-NEXT: ld.param.b64 %rd2, [fmin_acq_rel_bfloat_global_cta_param_0];
+; SM70-NOFTZ-NEXT: fence.acq_rel.cta;
+; SM70-NOFTZ-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NOFTZ-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-NOFTZ-NEXT: and.b32 %r5, %r4, 3;
+; SM70-NOFTZ-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-NOFTZ-NEXT: mov.b32 %r6, 65535;
+; SM70-NOFTZ-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-NOFTZ-NEXT: not.b32 %r2, %r7;
+; SM70-NOFTZ-NEXT: ld.relaxed.cta.global.b32 %r23, [%rd1];
+; SM70-NOFTZ-NEXT: cvt.u32.u16 %r10, %rs1;
+; SM70-NOFTZ-NEXT: shl.b32 %r11, %r10, 16;
+; SM70-NOFTZ-NEXT: $L__BB80_1: // %atomicrmw.start
+; SM70-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NOFTZ-NEXT: shr.u32 %r8, %r23, %r1;
+; SM70-NOFTZ-NEXT: shl.b32 %r9, %r8, 16;
+; SM70-NOFTZ-NEXT: min.f32 %r12, %r9, %r11;
+; SM70-NOFTZ-NEXT: bfe.u32 %r13, %r12, 16, 1;
+; SM70-NOFTZ-NEXT: add.s32 %r14, %r13, %r12;
+; SM70-NOFTZ-NEXT: add.s32 %r15, %r14, 32767;
+; SM70-NOFTZ-NEXT: setp.nan.f32 %p1, %r12, %r12;
+; SM70-NOFTZ-NEXT: or.b32 %r16, %r12, 4194304;
+; SM70-NOFTZ-NEXT: selp.b32 %r17, %r16, %r15, %p1;
+; SM70-NOFTZ-NEXT: shr.u32 %r18, %r17, 16;
+; SM70-NOFTZ-NEXT: shl.b32 %r19, %r18, %r1;
+; SM70-NOFTZ-NEXT: and.b32 %r20, %r23, %r2;
+; SM70-NOFTZ-NEXT: or.b32 %r21, %r20, %r19;
+; SM70-NOFTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r23, %r21;
+; SM70-NOFTZ-NEXT: setp.ne.b32 %p2, %r3, %r23;
+; SM70-NOFTZ-NEXT: mov.b32 %r23, %r3;
+; SM70-NOFTZ-NEXT: @%p2 bra $L__BB80_1;
+; SM70-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NOFTZ-NEXT: shr.u32 %r22, %r3, %r1;
+; SM70-NOFTZ-NEXT: fence.acq_rel.cta;
+; SM70-NOFTZ-NEXT: st.param.b16 [func_retval0], %r22;
+; SM70-NOFTZ-NEXT: ret;
+;
+; SM70-FTZ-LABEL: fmin_acq_rel_bfloat_global_cta(
+; SM70-FTZ: {
+; SM70-FTZ-NEXT: .reg .pred %p<3>;
+; SM70-FTZ-NEXT: .reg .b16 %rs<2>;
+; SM70-FTZ-NEXT: .reg .b32 %r<24>;
+; SM70-FTZ-NEXT: .reg .b64 %rd<3>;
+; SM70-FTZ-EMPTY:
+; SM70-FTZ-NEXT: // %bb.0:
+; SM70-FTZ-NEXT: ld.param.b16 %rs1, [fmin_acq_rel_bfloat_global_cta_param_1];
+; SM70-FTZ-NEXT: ld.param.b64 %rd2, [fmin_acq_rel_bfloat_global_cta_param_0];
+; SM70-FTZ-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-FTZ-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-FTZ-NEXT: and.b32 %r5, %r4, 3;
+; SM70-FTZ-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-FTZ-NEXT: mov.b32 %r6, 65535;
+; SM70-FTZ-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-FTZ-NEXT: not.b32 %r2, %r7;
+; SM70-FTZ-NEXT: ld.relaxed.cta.global.b32 %r23, [%rd1];
+; SM70-FTZ-NEXT: cvt.u32.u16 %r10, %rs1;
+; SM70-FTZ-NEXT: shl.b32 %r11, %r10, 16;
+; SM70-FTZ-NEXT: $L__BB80_1: // %atomicrmw.start
+; SM70-FTZ-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-FTZ-NEXT: shr.u32 %r8, %r23, %r1;
+; SM70-FTZ-NEXT: shl.b32 %r9, %r8, 16;
+; SM70-FTZ-NEXT: min.ftz.f32 %r12, %r9, %r11;
+; SM70-FTZ-NEXT: bfe.u32 %r13, %r12, 16, 1;
+; SM70-FTZ-NEXT: add.s32 %r14, %r13, %r12;
+; SM70-FTZ-NEXT: add.s32 %r15, %r14, 32767;
+; SM70-FTZ-NEXT: setp.nan.ftz.f32 %p1, %r12, %r12;
+; SM70-FTZ-NEXT: or.b32 %r16, %r12, 4194304;
+; SM70-FTZ-NEXT: selp.b32 %r17, %r16, %r15, %p1;
+; SM70-FTZ-NEXT: shr.u32 %r18, %r17, 16;
+; SM70-FTZ-NEXT: shl.b32 %r19, %r18, %r1;
+; SM70-FTZ-NEXT: and.b32 %r20, %r23, %r2;
+; SM70-FTZ-NEXT: or.b32 %r21, %r20, %r19;
+; SM70-FTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r23, %r21;
+; SM70-FTZ-NEXT: setp.ne.b32 %p2, %r3, %r23;
+; SM70-FTZ-NEXT: mov.b32 %r23, %r3;
+; SM70-FTZ-NEXT: @%p2 bra $L__BB80_1;
+; SM70-FTZ-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-FTZ-NEXT: shr.u32 %r22, %r3, %r1;
+; SM70-FTZ-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-NEXT: st.param.b16 [func_retval0], %r22;
+; SM70-FTZ-NEXT: ret;
%retval = atomicrmw fmin ptr addrspace(1) %addr, bfloat %val syncscope("block") acq_rel
ret bfloat %retval
}
define bfloat @fmax_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat %val) {
-; SM70-LABEL: fmax_acq_rel_bfloat_global_cta(
-; SM70: {
-; SM70-NEXT: .reg .pred %p<3>;
-; SM70-NEXT: .reg .b16 %rs<2>;
-; SM70-NEXT: .reg .b32 %r<24>;
-; SM70-NEXT: .reg .b64 %rd<3>;
-; SM70-EMPTY:
-; SM70-NEXT: // %bb.0:
-; SM70-NEXT: ld.param.b16 %rs1, [fmax_acq_rel_bfloat_global_cta_param_1];
-; SM70-NEXT: ld.param.b64 %rd2, [fmax_acq_rel_bfloat_global_cta_param_0];
-; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: and.b64 %rd1, %rd2, -4;
-; SM70-NEXT: cvt.u32.u64 %r4, %rd2;
-; SM70-NEXT: and.b32 %r5, %r4, 3;
-; SM70-NEXT: shl.b32 %r1, %r5, 3;
-; SM70-NEXT: mov.b32 %r6, 65535;
-; SM70-NEXT: shl.b32 %r7, %r6, %r1;
-; SM70-NEXT: not.b32 %r2, %r7;
-; SM70-NEXT: ld.relaxed.cta.global.b32 %r23, [%rd1];
-; SM70-NEXT: cvt.u32.u16 %r10, %rs1;
-; SM70-NEXT: shl.b32 %r11, %r10, 16;
-; SM70-NEXT: $L__BB81_1: // %atomicrmw.start
-; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: shr.u32 %r8, %r23, %r1;
-; SM70-NEXT: shl.b32 %r9, %r8, 16;
-; SM70-NEXT: max.f32 %r12, %r9, %r11;
-; SM70-NEXT: bfe.u32 %r13, %r12, 16, 1;
-; SM70-NEXT: add.s32 %r14, %r13, %r12;
-; SM70-NEXT: add.s32 %r15, %r14, 32767;
-; SM70-NEXT: setp.nan.f32 %p1, %r12, %r12;
-; SM70-NEXT: or.b32 %r16, %r12, 4194304;
-; SM70-NEXT: selp.b32 %r17, %r16, %r15, %p1;
-; SM70-NEXT: shr.u32 %r18, %r17, 16;
-; SM70-NEXT: shl.b32 %r19, %r18, %r1;
-; SM70-NEXT: and.b32 %r20, %r23, %r2;
-; SM70-NEXT: or.b32 %r21, %r20, %r19;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r23, %r21;
-; SM70-NEXT: setp.ne.b32 %p2, %r3, %r23;
-; SM70-NEXT: mov.b32 %r23, %r3;
-; SM70-NEXT: @%p2 bra $L__BB81_1;
-; SM70-NEXT: // %bb.2: // %atomicrmw.end
-; SM70-NEXT: shr.u32 %r22, %r3, %r1;
-; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: st.param.b16 [func_retval0], %r22;
-; SM70-NEXT: ret;
+; SM70-NOFTZ-LABEL: fmax_acq_rel_bfloat_global_cta(
+; SM70-NOFTZ: {
+; SM70-NOFTZ-NEXT: .reg .pred %p<3>;
+; SM70-NOFTZ-NEXT: .reg .b16 %rs<2>;
+; SM70-NOFTZ-NEXT: .reg .b32 %r<24>;
+; SM70-NOFTZ-NEXT: .reg .b64 %rd<3>;
+; SM70-NOFTZ-EMPTY:
+; SM70-NOFTZ-NEXT: // %bb.0:
+; SM70-NOFTZ-NEXT: ld.param.b16 %rs1, [fmax_acq_rel_bfloat_global_cta_param_1];
+; SM70-NOFTZ-NEXT: ld.param.b64 %rd2, [fmax_acq_rel_bfloat_global_cta_param_0];
+; SM70-NOFTZ-NEXT: fence.acq_rel.cta;
+; SM70-NOFTZ-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NOFTZ-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-NOFTZ-NEXT: and.b32 %r5, %r4, 3;
+; SM70-NOFTZ-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-NOFTZ-NEXT: mov.b32 %r6, 65535;
+; SM70-NOFTZ-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-NOFTZ-NEXT: not.b32 %r2, %r7;
+; SM70-NOFTZ-NEXT: ld.relaxed.cta.global.b32 %r23, [%rd1];
+; SM70-NOFTZ-NEXT: cvt.u32.u16 %r10, %rs1;
+; SM70-NOFTZ-NEXT: shl.b32 %r11, %r10, 16;
+; SM70-NOFTZ-NEXT: $L__BB81_1: // %atomicrmw.start
+; SM70-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NOFTZ-NEXT: shr.u32 %r8, %r23, %r1;
+; SM70-NOFTZ-NEXT: shl.b32 %r9, %r8, 16;
+; SM70-NOFTZ-NEXT: max.f32 %r12, %r9, %r11;
+; SM70-NOFTZ-NEXT: bfe.u32 %r13, %r12, 16, 1;
+; SM70-NOFTZ-NEXT: add.s32 %r14, %r13, %r12;
+; SM70-NOFTZ-NEXT: add.s32 %r15, %r14, 32767;
+; SM70-NOFTZ-NEXT: setp.nan.f32 %p1, %r12, %r12;
+; SM70-NOFTZ-NEXT: or.b32 %r16, %r12, 4194304;
+; SM70-NOFTZ-NEXT: selp.b32 %r17, %r16, %r15, %p1;
+; SM70-NOFTZ-NEXT: shr.u32 %r18, %r17, 16;
+; SM70-NOFTZ-NEXT: shl.b32 %r19, %r18, %r1;
+; SM70-NOFTZ-NEXT: and.b32 %r20, %r23, %r2;
+; SM70-NOFTZ-NEXT: or.b32 %r21, %r20, %r19;
+; SM70-NOFTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r23, %r21;
+; SM70-NOFTZ-NEXT: setp.ne.b32 %p2, %r3, %r23;
+; SM70-NOFTZ-NEXT: mov.b32 %r23, %r3;
+; SM70-NOFTZ-NEXT: @%p2 bra $L__BB81_1;
+; SM70-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NOFTZ-NEXT: shr.u32 %r22, %r3, %r1;
+; SM70-NOFTZ-NEXT: fence.acq_rel.cta;
+; SM70-NOFTZ-NEXT: st.param.b16 [func_retval0], %r22;
+; SM70-NOFTZ-NEXT: ret;
+;
+; SM70-FTZ-LABEL: fmax_acq_rel_bfloat_global_cta(
+; SM70-FTZ: {
+; SM70-FTZ-NEXT: .reg .pred %p<3>;
+; SM70-FTZ-NEXT: .reg .b16 %rs<2>;
+; SM70-FTZ-NEXT: .reg .b32 %r<24>;
+; SM70-FTZ-NEXT: .reg .b64 %rd<3>;
+; SM70-FTZ-EMPTY:
+; SM70-FTZ-NEXT: // %bb.0:
+; SM70-FTZ-NEXT: ld.param.b16 %rs1, [fmax_acq_rel_bfloat_global_cta_param_1];
+; SM70-FTZ-NEXT: ld.param.b64 %rd2, [fmax_acq_rel_bfloat_global_cta_param_0];
+; SM70-FTZ-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-FTZ-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-FTZ-NEXT: and.b32 %r5, %r4, 3;
+; SM70-FTZ-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-FTZ-NEXT: mov.b32 %r6, 65535;
+; SM70-FTZ-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-FTZ-NEXT: not.b32 %r2, %r7;
+; SM70-FTZ-NEXT: ld.relaxed.cta.global.b32 %r23, [%rd1];
+; SM70-FTZ-NEXT: cvt.u32.u16 %r10, %rs1;
+; SM70-FTZ-NEXT: shl.b32 %r11, %r10, 16;
+; SM70-FTZ-NEXT: $L__BB81_1: // %atomicrmw.start
+; SM70-FTZ-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-FTZ-NEXT: shr.u32 %r8, %r23, %r1;
+; SM70-FTZ-NEXT: shl.b32 %r9, %r8, 16;
+; SM70-FTZ-NEXT: max.ftz.f32 %r12, %r9, %r11;
+; SM70-FTZ-NEXT: bfe.u32 %r13, %r12, 16, 1;
+; SM70-FTZ-NEXT: add.s32 %r14, %r13, %r12;
+; SM70-FTZ-NEXT: add.s32 %r15, %r14, 32767;
+; SM70-FTZ-NEXT: setp.nan.ftz.f32 %p1, %r12, %r12;
+; SM70-FTZ-NEXT: or.b32 %r16, %r12, 4194304;
+; SM70-FTZ-NEXT: selp.b32 %r17, %r16, %r15, %p1;
+; SM70-FTZ-NEXT: shr.u32 %r18, %r17, 16;
+; SM70-FTZ-NEXT: shl.b32 %r19, %r18, %r1;
+; SM70-FTZ-NEXT: and.b32 %r20, %r23, %r2;
+; SM70-FTZ-NEXT: or.b32 %r21, %r20, %r19;
+; SM70-FTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r23, %r21;
+; SM70-FTZ-NEXT: setp.ne.b32 %p2, %r3, %r23;
+; SM70-FTZ-NEXT: mov.b32 %r23, %r3;
+; SM70-FTZ-NEXT: @%p2 bra $L__BB81_1;
+; SM70-FTZ-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-FTZ-NEXT: shr.u32 %r22, %r3, %r1;
+; SM70-FTZ-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-NEXT: st.param.b16 [func_retval0], %r22;
+; SM70-FTZ-NEXT: ret;
%retval = atomicrmw fmax ptr addrspace(1) %addr, bfloat %val syncscope("block") acq_rel
ret bfloat %retval
}
define bfloat @fminimum_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat %val) {
-; SM70-LABEL: fminimum_acq_rel_bfloat_global_cta(
-; SM70: {
-; SM70-NEXT: .reg .pred %p<7>;
-; SM70-NEXT: .reg .b16 %rs<8>;
-; SM70-NEXT: .reg .b32 %r<20>;
-; SM70-NEXT: .reg .b64 %rd<3>;
-; SM70-EMPTY:
-; SM70-NEXT: // %bb.0:
-; SM70-NEXT: ld.param.b16 %rs1, [fminimum_acq_rel_bfloat_global_cta_param_1];
-; SM70-NEXT: ld.param.b64 %rd2, [fminimum_acq_rel_bfloat_global_cta_param_0];
-; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: and.b64 %rd1, %rd2, -4;
-; SM70-NEXT: cvt.u32.u64 %r4, %rd2;
-; SM70-NEXT: and.b32 %r5, %r4, 3;
-; SM70-NEXT: shl.b32 %r1, %r5, 3;
-; SM70-NEXT: mov.b32 %r6, 65535;
-; SM70-NEXT: shl.b32 %r7, %r6, %r1;
-; SM70-NEXT: not.b32 %r2, %r7;
-; SM70-NEXT: ld.relaxed.cta.global.b32 %r19, [%rd1];
-; SM70-NEXT: cvt.u32.u16 %r10, %rs1;
-; SM70-NEXT: shl.b32 %r11, %r10, 16;
-; SM70-NEXT: setp.eq.b16 %p4, %rs1, -32768;
-; SM70-NEXT: $L__BB82_1: // %atomicrmw.start
-; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: shr.u32 %r8, %r19, %r1;
-; SM70-NEXT: cvt.u16.u32 %rs2, %r8;
-; SM70-NEXT: shl.b32 %r9, %r8, 16;
-; SM70-NEXT: setp.lt.f32 %p1, %r9, %r11;
-; SM70-NEXT: selp.b16 %rs3, %rs2, %rs1, %p1;
-; SM70-NEXT: setp.nan.f32 %p2, %r9, %r11;
-; SM70-NEXT: selp.b16 %rs4, 0x7FC0, %rs3, %p2;
-; SM70-NEXT: setp.eq.b16 %p3, %rs2, -32768;
-; SM70-NEXT: selp.b16 %rs5, %rs2, %rs4, %p3;
-; SM70-NEXT: selp.b16 %rs6, %rs1, %rs5, %p4;
-; SM70-NEXT: cvt.u32.u16 %r12, %rs4;
-; SM70-NEXT: shl.b32 %r13, %r12, 16;
-; SM70-NEXT: setp.eq.f32 %p5, %r13, 0f00000000;
-; SM70-NEXT: selp.b16 %rs7, %rs6, %rs4, %p5;
-; SM70-NEXT: cvt.u32.u16 %r14, %rs7;
-; SM70-NEXT: shl.b32 %r15, %r14, %r1;
-; SM70-NEXT: and.b32 %r16, %r19, %r2;
-; SM70-NEXT: or.b32 %r17, %r16, %r15;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r19, %r17;
-; SM70-NEXT: setp.ne.b32 %p6, %r3, %r19;
-; SM70-NEXT: mov.b32 %r19, %r3;
-; SM70-NEXT: @%p6 bra $L__BB82_1;
-; SM70-NEXT: // %bb.2: // %atomicrmw.end
-; SM70-NEXT: shr.u32 %r18, %r3, %r1;
-; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: st.param.b16 [func_retval0], %r18;
-; SM70-NEXT: ret;
+; SM70-NOFTZ-LABEL: fminimum_acq_rel_bfloat_global_cta(
+; SM70-NOFTZ: {
+; SM70-NOFTZ-NEXT: .reg .pred %p<7>;
+; SM70-NOFTZ-NEXT: .reg .b16 %rs<8>;
+; SM70-NOFTZ-NEXT: .reg .b32 %r<20>;
+; SM70-NOFTZ-NEXT: .reg .b64 %rd<3>;
+; SM70-NOFTZ-EMPTY:
+; SM70-NOFTZ-NEXT: // %bb.0:
+; SM70-NOFTZ-NEXT: ld.param.b16 %rs1, [fminimum_acq_rel_bfloat_global_cta_param_1];
+; SM70-NOFTZ-NEXT: ld.param.b64 %rd2, [fminimum_acq_rel_bfloat_global_cta_param_0];
+; SM70-NOFTZ-NEXT: fence.acq_rel.cta;
+; SM70-NOFTZ-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NOFTZ-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-NOFTZ-NEXT: and.b32 %r5, %r4, 3;
+; SM70-NOFTZ-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-NOFTZ-NEXT: mov.b32 %r6, 65535;
+; SM70-NOFTZ-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-NOFTZ-NEXT: not.b32 %r2, %r7;
+; SM70-NOFTZ-NEXT: ld.relaxed.cta.global.b32 %r19, [%rd1];
+; SM70-NOFTZ-NEXT: cvt.u32.u16 %r10, %rs1;
+; SM70-NOFTZ-NEXT: shl.b32 %r11, %r10, 16;
+; SM70-NOFTZ-NEXT: setp.eq.b16 %p4, %rs1, -32768;
+; SM70-NOFTZ-NEXT: $L__BB82_1: // %atomicrmw.start
+; SM70-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NOFTZ-NEXT: shr.u32 %r8, %r19, %r1;
+; SM70-NOFTZ-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM70-NOFTZ-NEXT: shl.b32 %r9, %r8, 16;
+; SM70-NOFTZ-NEXT: setp.lt.f32 %p1, %r9, %r11;
+; SM70-NOFTZ-NEXT: selp.b16 %rs3, %rs2, %rs1, %p1;
+; SM70-NOFTZ-NEXT: setp.nan.f32 %p2, %r9, %r11;
+; SM70-NOFTZ-NEXT: selp.b16 %rs4, 0x7FC0, %rs3, %p2;
+; SM70-NOFTZ-NEXT: setp.eq.b16 %p3, %rs2, -32768;
+; SM70-NOFTZ-NEXT: selp.b16 %rs5, %rs2, %rs4, %p3;
+; SM70-NOFTZ-NEXT: selp.b16 %rs6, %rs1, %rs5, %p4;
+; SM70-NOFTZ-NEXT: cvt.u32.u16 %r12, %rs4;
+; SM70-NOFTZ-NEXT: shl.b32 %r13, %r12, 16;
+; SM70-NOFTZ-NEXT: setp.eq.f32 %p5, %r13, 0f00000000;
+; SM70-NOFTZ-NEXT: selp.b16 %rs7, %rs6, %rs4, %p5;
+; SM70-NOFTZ-NEXT: cvt.u32.u16 %r14, %rs7;
+; SM70-NOFTZ-NEXT: shl.b32 %r15, %r14, %r1;
+; SM70-NOFTZ-NEXT: and.b32 %r16, %r19, %r2;
+; SM70-NOFTZ-NEXT: or.b32 %r17, %r16, %r15;
+; SM70-NOFTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r19, %r17;
+; SM70-NOFTZ-NEXT: setp.ne.b32 %p6, %r3, %r19;
+; SM70-NOFTZ-NEXT: mov.b32 %r19, %r3;
+; SM70-NOFTZ-NEXT: @%p6 bra $L__BB82_1;
+; SM70-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NOFTZ-NEXT: shr.u32 %r18, %r3, %r1;
+; SM70-NOFTZ-NEXT: fence.acq_rel.cta;
+; SM70-NOFTZ-NEXT: st.param.b16 [func_retval0], %r18;
+; SM70-NOFTZ-NEXT: ret;
+;
+; SM70-FTZ-LABEL: fminimum_acq_rel_bfloat_global_cta(
+; SM70-FTZ: {
+; SM70-FTZ-NEXT: .reg .pred %p<7>;
+; SM70-FTZ-NEXT: .reg .b16 %rs<8>;
+; SM70-FTZ-NEXT: .reg .b32 %r<20>;
+; SM70-FTZ-NEXT: .reg .b64 %rd<3>;
+; SM70-FTZ-EMPTY:
+; SM70-FTZ-NEXT: // %bb.0:
+; SM70-FTZ-NEXT: ld.param.b16 %rs1, [fminimum_acq_rel_bfloat_global_cta_param_1];
+; SM70-FTZ-NEXT: ld.param.b64 %rd2, [fminimum_acq_rel_bfloat_global_cta_param_0];
+; SM70-FTZ-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-FTZ-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-FTZ-NEXT: and.b32 %r5, %r4, 3;
+; SM70-FTZ-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-FTZ-NEXT: mov.b32 %r6, 65535;
+; SM70-FTZ-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-FTZ-NEXT: not.b32 %r2, %r7;
+; SM70-FTZ-NEXT: ld.relaxed.cta.global.b32 %r19, [%rd1];
+; SM70-FTZ-NEXT: cvt.u32.u16 %r10, %rs1;
+; SM70-FTZ-NEXT: shl.b32 %r11, %r10, 16;
+; SM70-FTZ-NEXT: setp.eq.b16 %p4, %rs1, -32768;
+; SM70-FTZ-NEXT: $L__BB82_1: // %atomicrmw.start
+; SM70-FTZ-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-FTZ-NEXT: shr.u32 %r8, %r19, %r1;
+; SM70-FTZ-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM70-FTZ-NEXT: shl.b32 %r9, %r8, 16;
+; SM70-FTZ-NEXT: setp.lt.ftz.f32 %p1, %r9, %r11;
+; SM70-FTZ-NEXT: selp.b16 %rs3, %rs2, %rs1, %p1;
+; SM70-FTZ-NEXT: setp.nan.ftz.f32 %p2, %r9, %r11;
+; SM70-FTZ-NEXT: selp.b16 %rs4, 0x7FC0, %rs3, %p2;
+; SM70-FTZ-NEXT: setp.eq.b16 %p3, %rs2, -32768;
+; SM70-FTZ-NEXT: selp.b16 %rs5, %rs2, %rs4, %p3;
+; SM70-FTZ-NEXT: selp.b16 %rs6, %rs1, %rs5, %p4;
+; SM70-FTZ-NEXT: cvt.u32.u16 %r12, %rs4;
+; SM70-FTZ-NEXT: shl.b32 %r13, %r12, 16;
+; SM70-FTZ-NEXT: setp.eq.ftz.f32 %p5, %r13, 0f00000000;
+; SM70-FTZ-NEXT: selp.b16 %rs7, %rs6, %rs4, %p5;
+; SM70-FTZ-NEXT: cvt.u32.u16 %r14, %rs7;
+; SM70-FTZ-NEXT: shl.b32 %r15, %r14, %r1;
+; SM70-FTZ-NEXT: and.b32 %r16, %r19, %r2;
+; SM70-FTZ-NEXT: or.b32 %r17, %r16, %r15;
+; SM70-FTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r19, %r17;
+; SM70-FTZ-NEXT: setp.ne.b32 %p6, %r3, %r19;
+; SM70-FTZ-NEXT: mov.b32 %r19, %r3;
+; SM70-FTZ-NEXT: @%p6 bra $L__BB82_1;
+; SM70-FTZ-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-FTZ-NEXT: shr.u32 %r18, %r3, %r1;
+; SM70-FTZ-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-NEXT: st.param.b16 [func_retval0], %r18;
+; SM70-FTZ-NEXT: ret;
%retval = atomicrmw fminimum ptr addrspace(1) %addr, bfloat %val syncscope("block") acq_rel
ret bfloat %retval
}
define bfloat @fmaximum_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat %val) {
-; SM70-LABEL: fmaximum_acq_rel_bfloat_global_cta(
-; SM70: {
-; SM70-NEXT: .reg .pred %p<7>;
-; SM70-NEXT: .reg .b16 %rs<8>;
-; SM70-NEXT: .reg .b32 %r<20>;
-; SM70-NEXT: .reg .b64 %rd<3>;
-; SM70-EMPTY:
-; SM70-NEXT: // %bb.0:
-; SM70-NEXT: ld.param.b16 %rs1, [fmaximum_acq_rel_bfloat_global_cta_param_1];
-; SM70-NEXT: ld.param.b64 %rd2, [fmaximum_acq_rel_bfloat_global_cta_param_0];
-; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: and.b64 %rd1, %rd2, -4;
-; SM70-NEXT: cvt.u32.u64 %r4, %rd2;
-; SM70-NEXT: and.b32 %r5, %r4, 3;
-; SM70-NEXT: shl.b32 %r1, %r5, 3;
-; SM70-NEXT: mov.b32 %r6, 65535;
-; SM70-NEXT: shl.b32 %r7, %r6, %r1;
-; SM70-NEXT: not.b32 %r2, %r7;
-; SM70-NEXT: ld.relaxed.cta.global.b32 %r19, [%rd1];
-; SM70-NEXT: cvt.u32.u16 %r10, %rs1;
-; SM70-NEXT: shl.b32 %r11, %r10, 16;
-; SM70-NEXT: setp.eq.b16 %p4, %rs1, 0;
-; SM70-NEXT: $L__BB83_1: // %atomicrmw.start
-; SM70-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NEXT: shr.u32 %r8, %r19, %r1;
-; SM70-NEXT: cvt.u16.u32 %rs2, %r8;
-; SM70-NEXT: shl.b32 %r9, %r8, 16;
-; SM70-NEXT: setp.gt.f32 %p1, %r9, %r11;
-; SM70-NEXT: selp.b16 %rs3, %rs2, %rs1, %p1;
-; SM70-NEXT: setp.nan.f32 %p2, %r9, %r11;
-; SM70-NEXT: selp.b16 %rs4, 0x7FC0, %rs3, %p2;
-; SM70-NEXT: setp.eq.b16 %p3, %rs2, 0;
-; SM70-NEXT: selp.b16 %rs5, %rs2, %rs4, %p3;
-; SM70-NEXT: selp.b16 %rs6, %rs1, %rs5, %p4;
-; SM70-NEXT: cvt.u32.u16 %r12, %rs4;
-; SM70-NEXT: shl.b32 %r13, %r12, 16;
-; SM70-NEXT: setp.eq.f32 %p5, %r13, 0f00000000;
-; SM70-NEXT: selp.b16 %rs7, %rs6, %rs4, %p5;
-; SM70-NEXT: cvt.u32.u16 %r14, %rs7;
-; SM70-NEXT: shl.b32 %r15, %r14, %r1;
-; SM70-NEXT: and.b32 %r16, %r19, %r2;
-; SM70-NEXT: or.b32 %r17, %r16, %r15;
-; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r19, %r17;
-; SM70-NEXT: setp.ne.b32 %p6, %r3, %r19;
-; SM70-NEXT: mov.b32 %r19, %r3;
-; SM70-NEXT: @%p6 bra $L__BB83_1;
-; SM70-NEXT: // %bb.2: // %atomicrmw.end
-; SM70-NEXT: shr.u32 %r18, %r3, %r1;
-; SM70-NEXT: fence.acq_rel.cta;
-; SM70-NEXT: st.param.b16 [func_retval0], %r18;
-; SM70-NEXT: ret;
+; SM70-NOFTZ-LABEL: fmaximum_acq_rel_bfloat_global_cta(
+; SM70-NOFTZ: {
+; SM70-NOFTZ-NEXT: .reg .pred %p<7>;
+; SM70-NOFTZ-NEXT: .reg .b16 %rs<8>;
+; SM70-NOFTZ-NEXT: .reg .b32 %r<20>;
+; SM70-NOFTZ-NEXT: .reg .b64 %rd<3>;
+; SM70-NOFTZ-EMPTY:
+; SM70-NOFTZ-NEXT: // %bb.0:
+; SM70-NOFTZ-NEXT: ld.param.b16 %rs1, [fmaximum_acq_rel_bfloat_global_cta_param_1];
+; SM70-NOFTZ-NEXT: ld.param.b64 %rd2, [fmaximum_acq_rel_bfloat_global_cta_param_0];
+; SM70-NOFTZ-NEXT: fence.acq_rel.cta;
+; SM70-NOFTZ-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NOFTZ-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-NOFTZ-NEXT: and.b32 %r5, %r4, 3;
+; SM70-NOFTZ-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-NOFTZ-NEXT: mov.b32 %r6, 65535;
+; SM70-NOFTZ-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-NOFTZ-NEXT: not.b32 %r2, %r7;
+; SM70-NOFTZ-NEXT: ld.relaxed.cta.global.b32 %r19, [%rd1];
+; SM70-NOFTZ-NEXT: cvt.u32.u16 %r10, %rs1;
+; SM70-NOFTZ-NEXT: shl.b32 %r11, %r10, 16;
+; SM70-NOFTZ-NEXT: setp.eq.b16 %p4, %rs1, 0;
+; SM70-NOFTZ-NEXT: $L__BB83_1: // %atomicrmw.start
+; SM70-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NOFTZ-NEXT: shr.u32 %r8, %r19, %r1;
+; SM70-NOFTZ-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM70-NOFTZ-NEXT: shl.b32 %r9, %r8, 16;
+; SM70-NOFTZ-NEXT: setp.gt.f32 %p1, %r9, %r11;
+; SM70-NOFTZ-NEXT: selp.b16 %rs3, %rs2, %rs1, %p1;
+; SM70-NOFTZ-NEXT: setp.nan.f32 %p2, %r9, %r11;
+; SM70-NOFTZ-NEXT: selp.b16 %rs4, 0x7FC0, %rs3, %p2;
+; SM70-NOFTZ-NEXT: setp.eq.b16 %p3, %rs2, 0;
+; SM70-NOFTZ-NEXT: selp.b16 %rs5, %rs2, %rs4, %p3;
+; SM70-NOFTZ-NEXT: selp.b16 %rs6, %rs1, %rs5, %p4;
+; SM70-NOFTZ-NEXT: cvt.u32.u16 %r12, %rs4;
+; SM70-NOFTZ-NEXT: shl.b32 %r13, %r12, 16;
+; SM70-NOFTZ-NEXT: setp.eq.f32 %p5, %r13, 0f00000000;
+; SM70-NOFTZ-NEXT: selp.b16 %rs7, %rs6, %rs4, %p5;
+; SM70-NOFTZ-NEXT: cvt.u32.u16 %r14, %rs7;
+; SM70-NOFTZ-NEXT: shl.b32 %r15, %r14, %r1;
+; SM70-NOFTZ-NEXT: and.b32 %r16, %r19, %r2;
+; SM70-NOFTZ-NEXT: or.b32 %r17, %r16, %r15;
+; SM70-NOFTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r19, %r17;
+; SM70-NOFTZ-NEXT: setp.ne.b32 %p6, %r3, %r19;
+; SM70-NOFTZ-NEXT: mov.b32 %r19, %r3;
+; SM70-NOFTZ-NEXT: @%p6 bra $L__BB83_1;
+; SM70-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NOFTZ-NEXT: shr.u32 %r18, %r3, %r1;
+; SM70-NOFTZ-NEXT: fence.acq_rel.cta;
+; SM70-NOFTZ-NEXT: st.param.b16 [func_retval0], %r18;
+; SM70-NOFTZ-NEXT: ret;
+;
+; SM70-FTZ-LABEL: fmaximum_acq_rel_bfloat_global_cta(
+; SM70-FTZ: {
+; SM70-FTZ-NEXT: .reg .pred %p<7>;
+; SM70-FTZ-NEXT: .reg .b16 %rs<8>;
+; SM70-FTZ-NEXT: .reg .b32 %r<20>;
+; SM70-FTZ-NEXT: .reg .b64 %rd<3>;
+; SM70-FTZ-EMPTY:
+; SM70-FTZ-NEXT: // %bb.0:
+; SM70-FTZ-NEXT: ld.param.b16 %rs1, [fmaximum_acq_rel_bfloat_global_cta_param_1];
+; SM70-FTZ-NEXT: ld.param.b64 %rd2, [fmaximum_acq_rel_bfloat_global_cta_param_0];
+; SM70-FTZ-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-FTZ-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-FTZ-NEXT: and.b32 %r5, %r4, 3;
+; SM70-FTZ-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-FTZ-NEXT: mov.b32 %r6, 65535;
+; SM70-FTZ-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-FTZ-NEXT: not.b32 %r2, %r7;
+; SM70-FTZ-NEXT: ld.relaxed.cta.global.b32 %r19, [%rd1];
+; SM70-FTZ-NEXT: cvt.u32.u16 %r10, %rs1;
+; SM70-FTZ-NEXT: shl.b32 %r11, %r10, 16;
+; SM70-FTZ-NEXT: setp.eq.b16 %p4, %rs1, 0;
+; SM70-FTZ-NEXT: $L__BB83_1: // %atomicrmw.start
+; SM70-FTZ-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-FTZ-NEXT: shr.u32 %r8, %r19, %r1;
+; SM70-FTZ-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM70-FTZ-NEXT: shl.b32 %r9, %r8, 16;
+; SM70-FTZ-NEXT: setp.gt.ftz.f32 %p1, %r9, %r11;
+; SM70-FTZ-NEXT: selp.b16 %rs3, %rs2, %rs1, %p1;
+; SM70-FTZ-NEXT: setp.nan.ftz.f32 %p2, %r9, %r11;
+; SM70-FTZ-NEXT: selp.b16 %rs4, 0x7FC0, %rs3, %p2;
+; SM70-FTZ-NEXT: setp.eq.b16 %p3, %rs2, 0;
+; SM70-FTZ-NEXT: selp.b16 %rs5, %rs2, %rs4, %p3;
+; SM70-FTZ-NEXT: selp.b16 %rs6, %rs1, %rs5, %p4;
+; SM70-FTZ-NEXT: cvt.u32.u16 %r12, %rs4;
+; SM70-FTZ-NEXT: shl.b32 %r13, %r12, 16;
+; SM70-FTZ-NEXT: setp.eq.ftz.f32 %p5, %r13, 0f00000000;
+; SM70-FTZ-NEXT: selp.b16 %rs7, %rs6, %rs4, %p5;
+; SM70-FTZ-NEXT: cvt.u32.u16 %r14, %rs7;
+; SM70-FTZ-NEXT: shl.b32 %r15, %r14, %r1;
+; SM70-FTZ-NEXT: and.b32 %r16, %r19, %r2;
+; SM70-FTZ-NEXT: or.b32 %r17, %r16, %r15;
+; SM70-FTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r19, %r17;
+; SM70-FTZ-NEXT: setp.ne.b32 %p6, %r3, %r19;
+; SM70-FTZ-NEXT: mov.b32 %r19, %r3;
+; SM70-FTZ-NEXT: @%p6 bra $L__BB83_1;
+; SM70-FTZ-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-FTZ-NEXT: shr.u32 %r18, %r3, %r1;
+; SM70-FTZ-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-NEXT: st.param.b16 [func_retval0], %r18;
+; SM70-FTZ-NEXT: ret;
%retval = atomicrmw fmaximum ptr addrspace(1) %addr, bfloat %val syncscope("block") acq_rel
ret bfloat %retval
}
@@ -3228,17 +3926,40 @@ define i64 @nand_acq_rel_i64_global_sys(ptr addrspace(1) %addr, i64 %val) {
}
define float @fadd_acq_rel_float_global_sys(ptr addrspace(1) %addr, float %val) {
-; SM70-LABEL: fadd_acq_rel_float_global_sys(
-; SM70: {
-; SM70-NEXT: .reg .b32 %r<3>;
-; SM70-NEXT: .reg .b64 %rd<2>;
-; SM70-EMPTY:
-; SM70-NEXT: // %bb.0:
-; SM70-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_sys_param_0];
-; SM70-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_global_sys_param_1];
-; SM70-NEXT: atom.acq_rel.sys.global.add.f32 %r2, [%rd1], %r1;
-; SM70-NEXT: st.param.b32 [func_retval0], %r2;
-; SM70-NEXT: ret;
+; SM70-NOFTZ-LABEL: fadd_acq_rel_float_global_sys(
+; SM70-NOFTZ: {
+; SM70-NOFTZ-NEXT: .reg .pred %p<2>;
+; SM70-NOFTZ-NEXT: .reg .b32 %r<5>;
+; SM70-NOFTZ-NEXT: .reg .b64 %rd<2>;
+; SM70-NOFTZ-EMPTY:
+; SM70-NOFTZ-NEXT: // %bb.0:
+; SM70-NOFTZ-NEXT: ld.param.b32 %r2, [fadd_acq_rel_float_global_sys_param_1];
+; SM70-NOFTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_sys_param_0];
+; SM70-NOFTZ-NEXT: fence.acq_rel.sys;
+; SM70-NOFTZ-NEXT: ld.relaxed.sys.global.b32 %r4, [%rd1];
+; SM70-NOFTZ-NEXT: $L__BB106_1: // %atomicrmw.start
+; SM70-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NOFTZ-NEXT: add.rn.f32 %r3, %r4, %r2;
+; SM70-NOFTZ-NEXT: atom.relaxed.sys.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM70-NOFTZ-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM70-NOFTZ-NEXT: mov.b32 %r4, %r1;
+; SM70-NOFTZ-NEXT: @%p1 bra $L__BB106_1;
+; SM70-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NOFTZ-NEXT: fence.acq_rel.sys;
+; SM70-NOFTZ-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NOFTZ-NEXT: ret;
+;
+; SM70-FTZ-LABEL: fadd_acq_rel_float_global_sys(
+; SM70-FTZ: {
+; SM70-FTZ-NEXT: .reg .b32 %r<3>;
+; SM70-FTZ-NEXT: .reg .b64 %rd<2>;
+; SM70-FTZ-EMPTY:
+; SM70-FTZ-NEXT: // %bb.0:
+; SM70-FTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_sys_param_0];
+; SM70-FTZ-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_global_sys_param_1];
+; SM70-FTZ-NEXT: atom.acq_rel.sys.global.add.f32 %r2, [%rd1], %r1;
+; SM70-FTZ-NEXT: st.param.b32 [func_retval0], %r2;
+; SM70-FTZ-NEXT: ret;
%retval = atomicrmw fadd ptr addrspace(1) %addr, float %val syncscope("") acq_rel
ret float %retval
}
@@ -3377,17 +4098,40 @@ define i64 @nand_acq_rel_i64_global_cluster(ptr addrspace(1) %addr, i64 %val) {
}
define float @fadd_acq_rel_float_global_cluster(ptr addrspace(1) %addr, float %val) {
-; SM70-LABEL: fadd_acq_rel_float_global_cluster(
-; SM70: {
-; SM70-NEXT: .reg .b32 %r<3>;
-; SM70-NEXT: .reg .b64 %rd<2>;
-; SM70-EMPTY:
-; SM70-NEXT: // %bb.0:
-; SM70-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_cluster_param_0];
-; SM70-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_global_cluster_param_1];
-; SM70-NEXT: atom.acq_rel.cta.global.add.f32 %r2, [%rd1], %r1;
-; SM70-NEXT: st.param.b32 [func_retval0], %r2;
-; SM70-NEXT: ret;
+; SM70-NOFTZ-LABEL: fadd_acq_rel_float_global_cluster(
+; SM70-NOFTZ: {
+; SM70-NOFTZ-NEXT: .reg .pred %p<2>;
+; SM70-NOFTZ-NEXT: .reg .b32 %r<5>;
+; SM70-NOFTZ-NEXT: .reg .b64 %rd<2>;
+; SM70-NOFTZ-EMPTY:
+; SM70-NOFTZ-NEXT: // %bb.0:
+; SM70-NOFTZ-NEXT: ld.param.b32 %r2, [fadd_acq_rel_float_global_cluster_param_1];
+; SM70-NOFTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_cluster_param_0];
+; SM70-NOFTZ-NEXT: fence.acq_rel.cta;
+; SM70-NOFTZ-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM70-NOFTZ-NEXT: $L__BB114_1: // %atomicrmw.start
+; SM70-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NOFTZ-NEXT: add.rn.f32 %r3, %r4, %r2;
+; SM70-NOFTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM70-NOFTZ-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM70-NOFTZ-NEXT: mov.b32 %r4, %r1;
+; SM70-NOFTZ-NEXT: @%p1 bra $L__BB114_1;
+; SM70-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NOFTZ-NEXT: fence.acq_rel.cta;
+; SM70-NOFTZ-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NOFTZ-NEXT: ret;
+;
+; SM70-FTZ-LABEL: fadd_acq_rel_float_global_cluster(
+; SM70-FTZ: {
+; SM70-FTZ-NEXT: .reg .b32 %r<3>;
+; SM70-FTZ-NEXT: .reg .b64 %rd<2>;
+; SM70-FTZ-EMPTY:
+; SM70-FTZ-NEXT: // %bb.0:
+; SM70-FTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_cluster_param_0];
+; SM70-FTZ-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_global_cluster_param_1];
+; SM70-FTZ-NEXT: atom.acq_rel.cta.global.add.f32 %r2, [%rd1], %r1;
+; SM70-FTZ-NEXT: st.param.b32 [func_retval0], %r2;
+; SM70-FTZ-NEXT: ret;
%retval = atomicrmw fadd ptr addrspace(1) %addr, float %val syncscope("cluster") acq_rel
ret float %retval
}
@@ -3526,17 +4270,40 @@ define i64 @nand_acq_rel_i64_global_gpu(ptr addrspace(1) %addr, i64 %val) {
}
define float @fadd_acq_rel_float_global_gpu(ptr addrspace(1) %addr, float %val) {
-; SM70-LABEL: fadd_acq_rel_float_global_gpu(
-; SM70: {
-; SM70-NEXT: .reg .b32 %r<3>;
-; SM70-NEXT: .reg .b64 %rd<2>;
-; SM70-EMPTY:
-; SM70-NEXT: // %bb.0:
-; SM70-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_gpu_param_0];
-; SM70-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_global_gpu_param_1];
-; SM70-NEXT: atom.acq_rel.gpu.global.add.f32 %r2, [%rd1], %r1;
-; SM70-NEXT: st.param.b32 [func_retval0], %r2;
-; SM70-NEXT: ret;
+; SM70-NOFTZ-LABEL: fadd_acq_rel_float_global_gpu(
+; SM70-NOFTZ: {
+; SM70-NOFTZ-NEXT: .reg .pred %p<2>;
+; SM70-NOFTZ-NEXT: .reg .b32 %r<5>;
+; SM70-NOFTZ-NEXT: .reg .b64 %rd<2>;
+; SM70-NOFTZ-EMPTY:
+; SM70-NOFTZ-NEXT: // %bb.0:
+; SM70-NOFTZ-NEXT: ld.param.b32 %r2, [fadd_acq_rel_float_global_gpu_param_1];
+; SM70-NOFTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_gpu_param_0];
+; SM70-NOFTZ-NEXT: fence.acq_rel.gpu;
+; SM70-NOFTZ-NEXT: ld.relaxed.gpu.global.b32 %r4, [%rd1];
+; SM70-NOFTZ-NEXT: $L__BB122_1: // %atomicrmw.start
+; SM70-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NOFTZ-NEXT: add.rn.f32 %r3, %r4, %r2;
+; SM70-NOFTZ-NEXT: atom.relaxed.gpu.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM70-NOFTZ-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM70-NOFTZ-NEXT: mov.b32 %r4, %r1;
+; SM70-NOFTZ-NEXT: @%p1 bra $L__BB122_1;
+; SM70-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NOFTZ-NEXT: fence.acq_rel.gpu;
+; SM70-NOFTZ-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NOFTZ-NEXT: ret;
+;
+; SM70-FTZ-LABEL: fadd_acq_rel_float_global_gpu(
+; SM70-FTZ: {
+; SM70-FTZ-NEXT: .reg .b32 %r<3>;
+; SM70-FTZ-NEXT: .reg .b64 %rd<2>;
+; SM70-FTZ-EMPTY:
+; SM70-FTZ-NEXT: // %bb.0:
+; SM70-FTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_gpu_param_0];
+; SM70-FTZ-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_global_gpu_param_1];
+; SM70-FTZ-NEXT: atom.acq_rel.gpu.global.add.f32 %r2, [%rd1], %r1;
+; SM70-FTZ-NEXT: st.param.b32 [func_retval0], %r2;
+; SM70-FTZ-NEXT: ret;
%retval = atomicrmw fadd ptr addrspace(1) %addr, float %val syncscope("device") acq_rel
ret float %retval
}
@@ -3675,17 +4442,51 @@ define i64 @nand_acq_rel_i64_generic_cta(ptr %addr, i64 %val) {
}
define float @fadd_acq_rel_float_generic_cta(ptr %addr, float %val) {
-; SM70-LABEL: fadd_acq_rel_float_generic_cta(
-; SM70: {
-; SM70-NEXT: .reg .b32 %r<3>;
-; SM70-NEXT: .reg .b64 %rd<2>;
-; SM70-EMPTY:
-; SM70-NEXT: // %bb.0:
-; SM70-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_generic_cta_param_0];
-; SM70-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_generic_cta_param_1];
-; SM70-NEXT: atom.acq_rel.cta.add.f32 %r2, [%rd1], %r1;
-; SM70-NEXT: st.param.b32 [func_retval0], %r2;
-; SM70-NEXT: ret;
+; SM70-NOFTZ-LABEL: fadd_acq_rel_float_generic_cta(
+; SM70-NOFTZ: {
+; SM70-NOFTZ-NEXT: .reg .pred %p<2>;
+; SM70-NOFTZ-NEXT: .reg .b32 %r<5>;
+; SM70-NOFTZ-NEXT: .reg .b64 %rd<2>;
+; SM70-NOFTZ-EMPTY:
+; SM70-NOFTZ-NEXT: // %bb.0:
+; SM70-NOFTZ-NEXT: ld.param.b32 %r2, [fadd_acq_rel_float_generic_cta_param_1];
+; SM70-NOFTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_generic_cta_param_0];
+; SM70-NOFTZ-NEXT: fence.acq_rel.cta;
+; SM70-NOFTZ-NEXT: ld.relaxed.cta.b32 %r4, [%rd1];
+; SM70-NOFTZ-NEXT: $L__BB130_1: // %atomicrmw.start
+; SM70-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NOFTZ-NEXT: add.rn.f32 %r3, %r4, %r2;
+; SM70-NOFTZ-NEXT: atom.relaxed.cta.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM70-NOFTZ-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM70-NOFTZ-NEXT: mov.b32 %r4, %r1;
+; SM70-NOFTZ-NEXT: @%p1 bra $L__BB130_1;
+; SM70-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NOFTZ-NEXT: fence.acq_rel.cta;
+; SM70-NOFTZ-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NOFTZ-NEXT: ret;
+;
+; SM70-FTZ-LABEL: fadd_acq_rel_float_generic_cta(
+; SM70-FTZ: {
+; SM70-FTZ-NEXT: .reg .pred %p<2>;
+; SM70-FTZ-NEXT: .reg .b32 %r<5>;
+; SM70-FTZ-NEXT: .reg .b64 %rd<2>;
+; SM70-FTZ-EMPTY:
+; SM70-FTZ-NEXT: // %bb.0:
+; SM70-FTZ-NEXT: ld.param.b32 %r2, [fadd_acq_rel_float_generic_cta_param_1];
+; SM70-FTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_generic_cta_param_0];
+; SM70-FTZ-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-NEXT: ld.relaxed.cta.b32 %r4, [%rd1];
+; SM70-FTZ-NEXT: $L__BB130_1: // %atomicrmw.start
+; SM70-FTZ-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-FTZ-NEXT: add.rn.ftz.f32 %r3, %r4, %r2;
+; SM70-FTZ-NEXT: atom.relaxed.cta.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM70-FTZ-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM70-FTZ-NEXT: mov.b32 %r4, %r1;
+; SM70-FTZ-NEXT: @%p1 bra $L__BB130_1;
+; SM70-FTZ-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-FTZ-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-FTZ-NEXT: ret;
%retval = atomicrmw fadd ptr %addr, float %val syncscope("block") acq_rel
ret float %retval
}
@@ -3824,17 +4625,40 @@ define i64 @nand_acq_rel_i64_shared_cta(ptr addrspace(3) %addr, i64 %val) {
}
define float @fadd_acq_rel_float_shared_cta(ptr addrspace(3) %addr, float %val) {
-; SM70-LABEL: fadd_acq_rel_float_shared_cta(
-; SM70: {
-; SM70-NEXT: .reg .b32 %r<3>;
-; SM70-NEXT: .reg .b64 %rd<2>;
-; SM70-EMPTY:
-; SM70-NEXT: // %bb.0:
-; SM70-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_shared_cta_param_0];
-; SM70-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_shared_cta_param_1];
-; SM70-NEXT: atom.acq_rel.cta.shared.add.f32 %r2, [%rd1], %r1;
-; SM70-NEXT: st.param.b32 [func_retval0], %r2;
-; SM70-NEXT: ret;
+; SM70-NOFTZ-LABEL: fadd_acq_rel_float_shared_cta(
+; SM70-NOFTZ: {
+; SM70-NOFTZ-NEXT: .reg .b32 %r<3>;
+; SM70-NOFTZ-NEXT: .reg .b64 %rd<2>;
+; SM70-NOFTZ-EMPTY:
+; SM70-NOFTZ-NEXT: // %bb.0:
+; SM70-NOFTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_shared_cta_param_0];
+; SM70-NOFTZ-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_shared_cta_param_1];
+; SM70-NOFTZ-NEXT: atom.acq_rel.cta.shared.add.f32 %r2, [%rd1], %r1;
+; SM70-NOFTZ-NEXT: st.param.b32 [func_retval0], %r2;
+; SM70-NOFTZ-NEXT: ret;
+;
+; SM70-FTZ-LABEL: fadd_acq_rel_float_shared_cta(
+; SM70-FTZ: {
+; SM70-FTZ-NEXT: .reg .pred %p<2>;
+; SM70-FTZ-NEXT: .reg .b32 %r<5>;
+; SM70-FTZ-NEXT: .reg .b64 %rd<2>;
+; SM70-FTZ-EMPTY:
+; SM70-FTZ-NEXT: // %bb.0:
+; SM70-FTZ-NEXT: ld.param.b32 %r2, [fadd_acq_rel_float_shared_cta_param_1];
+; SM70-FTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_shared_cta_param_0];
+; SM70-FTZ-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-NEXT: ld.relaxed.cta.shared.b32 %r4, [%rd1];
+; SM70-FTZ-NEXT: $L__BB138_1: // %atomicrmw.start
+; SM70-FTZ-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-FTZ-NEXT: add.rn.ftz.f32 %r3, %r4, %r2;
+; SM70-FTZ-NEXT: atom.relaxed.cta.shared.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM70-FTZ-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM70-FTZ-NEXT: mov.b32 %r4, %r1;
+; SM70-FTZ-NEXT: @%p1 bra $L__BB138_1;
+; SM70-FTZ-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-FTZ-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-FTZ-NEXT: ret;
%retval = atomicrmw fadd ptr addrspace(3) %addr, float %val syncscope("block") acq_rel
ret float %retval
}
diff --git a/llvm/test/CodeGen/NVPTX/atomicrmw-sm90.ll b/llvm/test/CodeGen/NVPTX/atomicrmw-sm90.ll
index e072e921f80ea..46819ca53638c 100644
--- a/llvm/test/CodeGen/NVPTX/atomicrmw-sm90.ll
+++ b/llvm/test/CodeGen/NVPTX/atomicrmw-sm90.ll
@@ -1,6 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
-; RUN: llc < %s -march=nvptx64 -mcpu=sm_90 -mattr=+ptx87 | FileCheck %s --check-prefix=SM90
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_90 -mattr=+ptx87 | FileCheck %s --check-prefixes=SM90,SM90-NOFTZ
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_90 -mattr=+ptx87 -denormal-fp-math-f32=preserve-sign | FileCheck %s --check-prefixes=SM90,SM90-FTZ
; RUN: %if ptxas-sm_90 && ptxas-isa-8.7 %{ llc < %s -march=nvptx64 -mcpu=sm_90 -mattr=+ptx87 | %ptxas-verify -arch=sm_90 %}
+; RUN: %if ptxas-sm_90 && ptxas-isa-8.7 %{ llc < %s -march=nvptx64 -mcpu=sm_90 -mattr=+ptx87 -denormal-fp-math-f32=preserve-sign | %ptxas-verify -arch=sm_90 %}
define i8 @xchg_acq_rel_i8_global_cta(ptr addrspace(1) %addr, i8 %val) {
; SM90-LABEL: xchg_acq_rel_i8_global_cta(
@@ -1737,152 +1739,290 @@ define i64 @usub_sat_acq_rel_i64_global_cta(ptr addrspace(1) %addr, i64 %val) {
}
define float @fadd_acq_rel_float_global_cta(ptr addrspace(1) %addr, float %val) {
-; SM90-LABEL: fadd_acq_rel_float_global_cta(
-; SM90: {
-; SM90-NEXT: .reg .b32 %r<3>;
-; SM90-NEXT: .reg .b64 %rd<2>;
-; SM90-EMPTY:
-; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_cta_param_0];
-; SM90-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_global_cta_param_1];
-; SM90-NEXT: atom.acq_rel.cta.global.add.f32 %r2, [%rd1], %r1;
-; SM90-NEXT: st.param.b32 [func_retval0], %r2;
-; SM90-NEXT: ret;
+; SM90-NOFTZ-LABEL: fadd_acq_rel_float_global_cta(
+; SM90-NOFTZ: {
+; SM90-NOFTZ-NEXT: .reg .pred %p<2>;
+; SM90-NOFTZ-NEXT: .reg .b32 %r<5>;
+; SM90-NOFTZ-NEXT: .reg .b64 %rd<2>;
+; SM90-NOFTZ-EMPTY:
+; SM90-NOFTZ-NEXT: // %bb.0:
+; SM90-NOFTZ-NEXT: ld.param.b32 %r2, [fadd_acq_rel_float_global_cta_param_1];
+; SM90-NOFTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_cta_param_0];
+; SM90-NOFTZ-NEXT: fence.release.cta;
+; SM90-NOFTZ-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM90-NOFTZ-NEXT: $L__BB60_1: // %atomicrmw.start
+; SM90-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NOFTZ-NEXT: add.rn.f32 %r3, %r4, %r2;
+; SM90-NOFTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM90-NOFTZ-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM90-NOFTZ-NEXT: mov.b32 %r4, %r1;
+; SM90-NOFTZ-NEXT: @%p1 bra $L__BB60_1;
+; SM90-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NOFTZ-NEXT: fence.acquire.cta;
+; SM90-NOFTZ-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NOFTZ-NEXT: ret;
+;
+; SM90-FTZ-LABEL: fadd_acq_rel_float_global_cta(
+; SM90-FTZ: {
+; SM90-FTZ-NEXT: .reg .b32 %r<3>;
+; SM90-FTZ-NEXT: .reg .b64 %rd<2>;
+; SM90-FTZ-EMPTY:
+; SM90-FTZ-NEXT: // %bb.0:
+; SM90-FTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_cta_param_0];
+; SM90-FTZ-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_global_cta_param_1];
+; SM90-FTZ-NEXT: atom.acq_rel.cta.global.add.f32 %r2, [%rd1], %r1;
+; SM90-FTZ-NEXT: st.param.b32 [func_retval0], %r2;
+; SM90-FTZ-NEXT: ret;
%retval = atomicrmw fadd ptr addrspace(1) %addr, float %val syncscope("block") acq_rel
ret float %retval
}
define float @fsub_acq_rel_float_global_cta(ptr addrspace(1) %addr, float %val) {
-; SM90-LABEL: fsub_acq_rel_float_global_cta(
-; SM90: {
-; SM90-NEXT: .reg .pred %p<2>;
-; SM90-NEXT: .reg .b32 %r<5>;
-; SM90-NEXT: .reg .b64 %rd<2>;
-; SM90-EMPTY:
-; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b32 %r2, [fsub_acq_rel_float_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_float_global_cta_param_0];
-; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
-; SM90-NEXT: $L__BB61_1: // %atomicrmw.start
-; SM90-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NEXT: sub.rn.f32 %r3, %r4, %r2;
-; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
-; SM90-NEXT: setp.ne.b32 %p1, %r1, %r4;
-; SM90-NEXT: mov.b32 %r4, %r1;
-; SM90-NEXT: @%p1 bra $L__BB61_1;
-; SM90-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b32 [func_retval0], %r1;
-; SM90-NEXT: ret;
+; SM90-NOFTZ-LABEL: fsub_acq_rel_float_global_cta(
+; SM90-NOFTZ: {
+; SM90-NOFTZ-NEXT: .reg .pred %p<2>;
+; SM90-NOFTZ-NEXT: .reg .b32 %r<5>;
+; SM90-NOFTZ-NEXT: .reg .b64 %rd<2>;
+; SM90-NOFTZ-EMPTY:
+; SM90-NOFTZ-NEXT: // %bb.0:
+; SM90-NOFTZ-NEXT: ld.param.b32 %r2, [fsub_acq_rel_float_global_cta_param_1];
+; SM90-NOFTZ-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_float_global_cta_param_0];
+; SM90-NOFTZ-NEXT: fence.release.cta;
+; SM90-NOFTZ-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM90-NOFTZ-NEXT: $L__BB61_1: // %atomicrmw.start
+; SM90-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NOFTZ-NEXT: sub.rn.f32 %r3, %r4, %r2;
+; SM90-NOFTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM90-NOFTZ-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM90-NOFTZ-NEXT: mov.b32 %r4, %r1;
+; SM90-NOFTZ-NEXT: @%p1 bra $L__BB61_1;
+; SM90-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NOFTZ-NEXT: fence.acquire.cta;
+; SM90-NOFTZ-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NOFTZ-NEXT: ret;
+;
+; SM90-FTZ-LABEL: fsub_acq_rel_float_global_cta(
+; SM90-FTZ: {
+; SM90-FTZ-NEXT: .reg .pred %p<2>;
+; SM90-FTZ-NEXT: .reg .b32 %r<5>;
+; SM90-FTZ-NEXT: .reg .b64 %rd<2>;
+; SM90-FTZ-EMPTY:
+; SM90-FTZ-NEXT: // %bb.0:
+; SM90-FTZ-NEXT: ld.param.b32 %r2, [fsub_acq_rel_float_global_cta_param_1];
+; SM90-FTZ-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_float_global_cta_param_0];
+; SM90-FTZ-NEXT: fence.release.cta;
+; SM90-FTZ-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM90-FTZ-NEXT: $L__BB61_1: // %atomicrmw.start
+; SM90-FTZ-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-FTZ-NEXT: sub.rn.ftz.f32 %r3, %r4, %r2;
+; SM90-FTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM90-FTZ-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM90-FTZ-NEXT: mov.b32 %r4, %r1;
+; SM90-FTZ-NEXT: @%p1 bra $L__BB61_1;
+; SM90-FTZ-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-FTZ-NEXT: fence.acquire.cta;
+; SM90-FTZ-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-FTZ-NEXT: ret;
%retval = atomicrmw fsub ptr addrspace(1) %addr, float %val syncscope("block") acq_rel
ret float %retval
}
define float @fmin_acq_rel_float_global_cta(ptr addrspace(1) %addr, float %val) {
-; SM90-LABEL: fmin_acq_rel_float_global_cta(
-; SM90: {
-; SM90-NEXT: .reg .pred %p<2>;
-; SM90-NEXT: .reg .b32 %r<5>;
-; SM90-NEXT: .reg .b64 %rd<2>;
-; SM90-EMPTY:
-; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b32 %r2, [fmin_acq_rel_float_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_float_global_cta_param_0];
-; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
-; SM90-NEXT: $L__BB62_1: // %atomicrmw.start
-; SM90-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NEXT: min.f32 %r3, %r4, %r2;
-; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
-; SM90-NEXT: setp.ne.b32 %p1, %r1, %r4;
-; SM90-NEXT: mov.b32 %r4, %r1;
-; SM90-NEXT: @%p1 bra $L__BB62_1;
-; SM90-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b32 [func_retval0], %r1;
-; SM90-NEXT: ret;
+; SM90-NOFTZ-LABEL: fmin_acq_rel_float_global_cta(
+; SM90-NOFTZ: {
+; SM90-NOFTZ-NEXT: .reg .pred %p<2>;
+; SM90-NOFTZ-NEXT: .reg .b32 %r<5>;
+; SM90-NOFTZ-NEXT: .reg .b64 %rd<2>;
+; SM90-NOFTZ-EMPTY:
+; SM90-NOFTZ-NEXT: // %bb.0:
+; SM90-NOFTZ-NEXT: ld.param.b32 %r2, [fmin_acq_rel_float_global_cta_param_1];
+; SM90-NOFTZ-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_float_global_cta_param_0];
+; SM90-NOFTZ-NEXT: fence.release.cta;
+; SM90-NOFTZ-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM90-NOFTZ-NEXT: $L__BB62_1: // %atomicrmw.start
+; SM90-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NOFTZ-NEXT: min.f32 %r3, %r4, %r2;
+; SM90-NOFTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM90-NOFTZ-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM90-NOFTZ-NEXT: mov.b32 %r4, %r1;
+; SM90-NOFTZ-NEXT: @%p1 bra $L__BB62_1;
+; SM90-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NOFTZ-NEXT: fence.acquire.cta;
+; SM90-NOFTZ-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NOFTZ-NEXT: ret;
+;
+; SM90-FTZ-LABEL: fmin_acq_rel_float_global_cta(
+; SM90-FTZ: {
+; SM90-FTZ-NEXT: .reg .pred %p<2>;
+; SM90-FTZ-NEXT: .reg .b32 %r<5>;
+; SM90-FTZ-NEXT: .reg .b64 %rd<2>;
+; SM90-FTZ-EMPTY:
+; SM90-FTZ-NEXT: // %bb.0:
+; SM90-FTZ-NEXT: ld.param.b32 %r2, [fmin_acq_rel_float_global_cta_param_1];
+; SM90-FTZ-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_float_global_cta_param_0];
+; SM90-FTZ-NEXT: fence.release.cta;
+; SM90-FTZ-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM90-FTZ-NEXT: $L__BB62_1: // %atomicrmw.start
+; SM90-FTZ-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-FTZ-NEXT: min.ftz.f32 %r3, %r4, %r2;
+; SM90-FTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM90-FTZ-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM90-FTZ-NEXT: mov.b32 %r4, %r1;
+; SM90-FTZ-NEXT: @%p1 bra $L__BB62_1;
+; SM90-FTZ-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-FTZ-NEXT: fence.acquire.cta;
+; SM90-FTZ-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-FTZ-NEXT: ret;
%retval = atomicrmw fmin ptr addrspace(1) %addr, float %val syncscope("block") acq_rel
ret float %retval
}
define float @fmax_acq_rel_float_global_cta(ptr addrspace(1) %addr, float %val) {
-; SM90-LABEL: fmax_acq_rel_float_global_cta(
-; SM90: {
-; SM90-NEXT: .reg .pred %p<2>;
-; SM90-NEXT: .reg .b32 %r<5>;
-; SM90-NEXT: .reg .b64 %rd<2>;
-; SM90-EMPTY:
-; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b32 %r2, [fmax_acq_rel_float_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_float_global_cta_param_0];
-; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
-; SM90-NEXT: $L__BB63_1: // %atomicrmw.start
-; SM90-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NEXT: max.f32 %r3, %r4, %r2;
-; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
-; SM90-NEXT: setp.ne.b32 %p1, %r1, %r4;
-; SM90-NEXT: mov.b32 %r4, %r1;
-; SM90-NEXT: @%p1 bra $L__BB63_1;
-; SM90-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b32 [func_retval0], %r1;
-; SM90-NEXT: ret;
+; SM90-NOFTZ-LABEL: fmax_acq_rel_float_global_cta(
+; SM90-NOFTZ: {
+; SM90-NOFTZ-NEXT: .reg .pred %p<2>;
+; SM90-NOFTZ-NEXT: .reg .b32 %r<5>;
+; SM90-NOFTZ-NEXT: .reg .b64 %rd<2>;
+; SM90-NOFTZ-EMPTY:
+; SM90-NOFTZ-NEXT: // %bb.0:
+; SM90-NOFTZ-NEXT: ld.param.b32 %r2, [fmax_acq_rel_float_global_cta_param_1];
+; SM90-NOFTZ-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_float_global_cta_param_0];
+; SM90-NOFTZ-NEXT: fence.release.cta;
+; SM90-NOFTZ-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM90-NOFTZ-NEXT: $L__BB63_1: // %atomicrmw.start
+; SM90-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NOFTZ-NEXT: max.f32 %r3, %r4, %r2;
+; SM90-NOFTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM90-NOFTZ-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM90-NOFTZ-NEXT: mov.b32 %r4, %r1;
+; SM90-NOFTZ-NEXT: @%p1 bra $L__BB63_1;
+; SM90-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NOFTZ-NEXT: fence.acquire.cta;
+; SM90-NOFTZ-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NOFTZ-NEXT: ret;
+;
+; SM90-FTZ-LABEL: fmax_acq_rel_float_global_cta(
+; SM90-FTZ: {
+; SM90-FTZ-NEXT: .reg .pred %p<2>;
+; SM90-FTZ-NEXT: .reg .b32 %r<5>;
+; SM90-FTZ-NEXT: .reg .b64 %rd<2>;
+; SM90-FTZ-EMPTY:
+; SM90-FTZ-NEXT: // %bb.0:
+; SM90-FTZ-NEXT: ld.param.b32 %r2, [fmax_acq_rel_float_global_cta_param_1];
+; SM90-FTZ-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_float_global_cta_param_0];
+; SM90-FTZ-NEXT: fence.release.cta;
+; SM90-FTZ-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM90-FTZ-NEXT: $L__BB63_1: // %atomicrmw.start
+; SM90-FTZ-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-FTZ-NEXT: max.ftz.f32 %r3, %r4, %r2;
+; SM90-FTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM90-FTZ-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM90-FTZ-NEXT: mov.b32 %r4, %r1;
+; SM90-FTZ-NEXT: @%p1 bra $L__BB63_1;
+; SM90-FTZ-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-FTZ-NEXT: fence.acquire.cta;
+; SM90-FTZ-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-FTZ-NEXT: ret;
%retval = atomicrmw fmax ptr addrspace(1) %addr, float %val syncscope("block") acq_rel
ret float %retval
}
define float @fminimum_acq_rel_float_global_cta(ptr addrspace(1) %addr, float %val) {
-; SM90-LABEL: fminimum_acq_rel_float_global_cta(
-; SM90: {
-; SM90-NEXT: .reg .pred %p<2>;
-; SM90-NEXT: .reg .b32 %r<5>;
-; SM90-NEXT: .reg .b64 %rd<2>;
-; SM90-EMPTY:
-; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b32 %r2, [fminimum_acq_rel_float_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_float_global_cta_param_0];
-; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
-; SM90-NEXT: $L__BB64_1: // %atomicrmw.start
-; SM90-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NEXT: min.NaN.f32 %r3, %r4, %r2;
-; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
-; SM90-NEXT: setp.ne.b32 %p1, %r1, %r4;
-; SM90-NEXT: mov.b32 %r4, %r1;
-; SM90-NEXT: @%p1 bra $L__BB64_1;
-; SM90-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b32 [func_retval0], %r1;
-; SM90-NEXT: ret;
+; SM90-NOFTZ-LABEL: fminimum_acq_rel_float_global_cta(
+; SM90-NOFTZ: {
+; SM90-NOFTZ-NEXT: .reg .pred %p<2>;
+; SM90-NOFTZ-NEXT: .reg .b32 %r<5>;
+; SM90-NOFTZ-NEXT: .reg .b64 %rd<2>;
+; SM90-NOFTZ-EMPTY:
+; SM90-NOFTZ-NEXT: // %bb.0:
+; SM90-NOFTZ-NEXT: ld.param.b32 %r2, [fminimum_acq_rel_float_global_cta_param_1];
+; SM90-NOFTZ-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_float_global_cta_param_0];
+; SM90-NOFTZ-NEXT: fence.release.cta;
+; SM90-NOFTZ-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM90-NOFTZ-NEXT: $L__BB64_1: // %atomicrmw.start
+; SM90-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NOFTZ-NEXT: min.NaN.f32 %r3, %r4, %r2;
+; SM90-NOFTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM90-NOFTZ-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM90-NOFTZ-NEXT: mov.b32 %r4, %r1;
+; SM90-NOFTZ-NEXT: @%p1 bra $L__BB64_1;
+; SM90-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NOFTZ-NEXT: fence.acquire.cta;
+; SM90-NOFTZ-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NOFTZ-NEXT: ret;
+;
+; SM90-FTZ-LABEL: fminimum_acq_rel_float_global_cta(
+; SM90-FTZ: {
+; SM90-FTZ-NEXT: .reg .pred %p<2>;
+; SM90-FTZ-NEXT: .reg .b32 %r<5>;
+; SM90-FTZ-NEXT: .reg .b64 %rd<2>;
+; SM90-FTZ-EMPTY:
+; SM90-FTZ-NEXT: // %bb.0:
+; SM90-FTZ-NEXT: ld.param.b32 %r2, [fminimum_acq_rel_float_global_cta_param_1];
+; SM90-FTZ-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_float_global_cta_param_0];
+; SM90-FTZ-NEXT: fence.release.cta;
+; SM90-FTZ-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM90-FTZ-NEXT: $L__BB64_1: // %atomicrmw.start
+; SM90-FTZ-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-FTZ-NEXT: min.ftz.NaN.f32 %r3, %r4, %r2;
+; SM90-FTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM90-FTZ-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM90-FTZ-NEXT: mov.b32 %r4, %r1;
+; SM90-FTZ-NEXT: @%p1 bra $L__BB64_1;
+; SM90-FTZ-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-FTZ-NEXT: fence.acquire.cta;
+; SM90-FTZ-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-FTZ-NEXT: ret;
%retval = atomicrmw fminimum ptr addrspace(1) %addr, float %val syncscope("block") acq_rel
ret float %retval
}
define float @fmaximum_acq_rel_float_global_cta(ptr addrspace(1) %addr, float %val) {
-; SM90-LABEL: fmaximum_acq_rel_float_global_cta(
-; SM90: {
-; SM90-NEXT: .reg .pred %p<2>;
-; SM90-NEXT: .reg .b32 %r<5>;
-; SM90-NEXT: .reg .b64 %rd<2>;
-; SM90-EMPTY:
-; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b32 %r2, [fmaximum_acq_rel_float_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_float_global_cta_param_0];
-; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
-; SM90-NEXT: $L__BB65_1: // %atomicrmw.start
-; SM90-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NEXT: max.NaN.f32 %r3, %r4, %r2;
-; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
-; SM90-NEXT: setp.ne.b32 %p1, %r1, %r4;
-; SM90-NEXT: mov.b32 %r4, %r1;
-; SM90-NEXT: @%p1 bra $L__BB65_1;
-; SM90-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b32 [func_retval0], %r1;
-; SM90-NEXT: ret;
+; SM90-NOFTZ-LABEL: fmaximum_acq_rel_float_global_cta(
+; SM90-NOFTZ: {
+; SM90-NOFTZ-NEXT: .reg .pred %p<2>;
+; SM90-NOFTZ-NEXT: .reg .b32 %r<5>;
+; SM90-NOFTZ-NEXT: .reg .b64 %rd<2>;
+; SM90-NOFTZ-EMPTY:
+; SM90-NOFTZ-NEXT: // %bb.0:
+; SM90-NOFTZ-NEXT: ld.param.b32 %r2, [fmaximum_acq_rel_float_global_cta_param_1];
+; SM90-NOFTZ-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_float_global_cta_param_0];
+; SM90-NOFTZ-NEXT: fence.release.cta;
+; SM90-NOFTZ-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM90-NOFTZ-NEXT: $L__BB65_1: // %atomicrmw.start
+; SM90-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NOFTZ-NEXT: max.NaN.f32 %r3, %r4, %r2;
+; SM90-NOFTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM90-NOFTZ-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM90-NOFTZ-NEXT: mov.b32 %r4, %r1;
+; SM90-NOFTZ-NEXT: @%p1 bra $L__BB65_1;
+; SM90-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NOFTZ-NEXT: fence.acquire.cta;
+; SM90-NOFTZ-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NOFTZ-NEXT: ret;
+;
+; SM90-FTZ-LABEL: fmaximum_acq_rel_float_global_cta(
+; SM90-FTZ: {
+; SM90-FTZ-NEXT: .reg .pred %p<2>;
+; SM90-FTZ-NEXT: .reg .b32 %r<5>;
+; SM90-FTZ-NEXT: .reg .b64 %rd<2>;
+; SM90-FTZ-EMPTY:
+; SM90-FTZ-NEXT: // %bb.0:
+; SM90-FTZ-NEXT: ld.param.b32 %r2, [fmaximum_acq_rel_float_global_cta_param_1];
+; SM90-FTZ-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_float_global_cta_param_0];
+; SM90-FTZ-NEXT: fence.release.cta;
+; SM90-FTZ-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM90-FTZ-NEXT: $L__BB65_1: // %atomicrmw.start
+; SM90-FTZ-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-FTZ-NEXT: max.ftz.NaN.f32 %r3, %r4, %r2;
+; SM90-FTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM90-FTZ-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM90-FTZ-NEXT: mov.b32 %r4, %r1;
+; SM90-FTZ-NEXT: @%p1 bra $L__BB65_1;
+; SM90-FTZ-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-FTZ-NEXT: fence.acquire.cta;
+; SM90-FTZ-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-FTZ-NEXT: ret;
%retval = atomicrmw fmaximum ptr addrspace(1) %addr, float %val syncscope("block") acq_rel
ret float %retval
}
@@ -2049,227 +2189,455 @@ define double @fmaximum_acq_rel_double_global_cta(ptr addrspace(1) %addr, double
}
define half @fadd_acq_rel_half_global_cta(ptr addrspace(1) %addr, half %val) {
-; SM90-LABEL: fadd_acq_rel_half_global_cta(
-; SM90: {
-; SM90-NEXT: .reg .b16 %rs<3>;
-; SM90-NEXT: .reg .b64 %rd<2>;
-; SM90-EMPTY:
-; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_half_global_cta_param_0];
-; SM90-NEXT: ld.param.b16 %rs1, [fadd_acq_rel_half_global_cta_param_1];
-; SM90-NEXT: atom.acq_rel.cta.global.add.noftz.f16 %rs2, [%rd1], %rs1;
-; SM90-NEXT: st.param.b16 [func_retval0], %rs2;
-; SM90-NEXT: ret;
+; SM90-NOFTZ-LABEL: fadd_acq_rel_half_global_cta(
+; SM90-NOFTZ: {
+; SM90-NOFTZ-NEXT: .reg .b16 %rs<3>;
+; SM90-NOFTZ-NEXT: .reg .b64 %rd<2>;
+; SM90-NOFTZ-EMPTY:
+; SM90-NOFTZ-NEXT: // %bb.0:
+; SM90-NOFTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_half_global_cta_param_0];
+; SM90-NOFTZ-NEXT: ld.param.b16 %rs1, [fadd_acq_rel_half_global_cta_param_1];
+; SM90-NOFTZ-NEXT: atom.acq_rel.cta.global.add.noftz.f16 %rs2, [%rd1], %rs1;
+; SM90-NOFTZ-NEXT: st.param.b16 [func_retval0], %rs2;
+; SM90-NOFTZ-NEXT: ret;
+;
+; SM90-FTZ-LABEL: fadd_acq_rel_half_global_cta(
+; SM90-FTZ: {
+; SM90-FTZ-NEXT: .reg .pred %p<2>;
+; SM90-FTZ-NEXT: .reg .b16 %rs<4>;
+; SM90-FTZ-NEXT: .reg .b32 %r<15>;
+; SM90-FTZ-NEXT: .reg .b64 %rd<3>;
+; SM90-FTZ-EMPTY:
+; SM90-FTZ-NEXT: // %bb.0:
+; SM90-FTZ-NEXT: ld.param.b16 %rs1, [fadd_acq_rel_half_global_cta_param_1];
+; SM90-FTZ-NEXT: ld.param.b64 %rd2, [fadd_acq_rel_half_global_cta_param_0];
+; SM90-FTZ-NEXT: fence.release.cta;
+; SM90-FTZ-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-FTZ-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM90-FTZ-NEXT: and.b32 %r5, %r4, 3;
+; SM90-FTZ-NEXT: shl.b32 %r1, %r5, 3;
+; SM90-FTZ-NEXT: mov.b32 %r6, 65535;
+; SM90-FTZ-NEXT: shl.b32 %r7, %r6, %r1;
+; SM90-FTZ-NEXT: not.b32 %r2, %r7;
+; SM90-FTZ-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM90-FTZ-NEXT: $L__BB72_1: // %atomicrmw.start
+; SM90-FTZ-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-FTZ-NEXT: shr.u32 %r8, %r14, %r1;
+; SM90-FTZ-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM90-FTZ-NEXT: add.rn.ftz.f16 %rs3, %rs2, %rs1;
+; SM90-FTZ-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM90-FTZ-NEXT: shl.b32 %r10, %r9, %r1;
+; SM90-FTZ-NEXT: and.b32 %r11, %r14, %r2;
+; SM90-FTZ-NEXT: or.b32 %r12, %r11, %r10;
+; SM90-FTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM90-FTZ-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM90-FTZ-NEXT: mov.b32 %r14, %r3;
+; SM90-FTZ-NEXT: @%p1 bra $L__BB72_1;
+; SM90-FTZ-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-FTZ-NEXT: shr.u32 %r13, %r3, %r1;
+; SM90-FTZ-NEXT: fence.acquire.cta;
+; SM90-FTZ-NEXT: st.param.b16 [func_retval0], %r13;
+; SM90-FTZ-NEXT: ret;
%retval = atomicrmw fadd ptr addrspace(1) %addr, half %val syncscope("block") acq_rel
ret half %retval
}
define half @fsub_acq_rel_half_global_cta(ptr addrspace(1) %addr, half %val) {
-; SM90-LABEL: fsub_acq_rel_half_global_cta(
-; SM90: {
-; SM90-NEXT: .reg .pred %p<2>;
-; SM90-NEXT: .reg .b16 %rs<4>;
-; SM90-NEXT: .reg .b32 %r<15>;
-; SM90-NEXT: .reg .b64 %rd<3>;
-; SM90-EMPTY:
-; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b16 %rs1, [fsub_acq_rel_half_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd2, [fsub_acq_rel_half_global_cta_param_0];
-; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: and.b64 %rd1, %rd2, -4;
-; SM90-NEXT: cvt.u32.u64 %r4, %rd2;
-; SM90-NEXT: and.b32 %r5, %r4, 3;
-; SM90-NEXT: shl.b32 %r1, %r5, 3;
-; SM90-NEXT: mov.b32 %r6, 65535;
-; SM90-NEXT: shl.b32 %r7, %r6, %r1;
-; SM90-NEXT: not.b32 %r2, %r7;
-; SM90-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
-; SM90-NEXT: $L__BB73_1: // %atomicrmw.start
-; SM90-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NEXT: shr.u32 %r8, %r14, %r1;
-; SM90-NEXT: cvt.u16.u32 %rs2, %r8;
-; SM90-NEXT: sub.rn.f16 %rs3, %rs2, %rs1;
-; SM90-NEXT: cvt.u32.u16 %r9, %rs3;
-; SM90-NEXT: shl.b32 %r10, %r9, %r1;
-; SM90-NEXT: and.b32 %r11, %r14, %r2;
-; SM90-NEXT: or.b32 %r12, %r11, %r10;
-; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
-; SM90-NEXT: setp.ne.b32 %p1, %r3, %r14;
-; SM90-NEXT: mov.b32 %r14, %r3;
-; SM90-NEXT: @%p1 bra $L__BB73_1;
-; SM90-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NEXT: shr.u32 %r13, %r3, %r1;
-; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b16 [func_retval0], %r13;
-; SM90-NEXT: ret;
+; SM90-NOFTZ-LABEL: fsub_acq_rel_half_global_cta(
+; SM90-NOFTZ: {
+; SM90-NOFTZ-NEXT: .reg .pred %p<2>;
+; SM90-NOFTZ-NEXT: .reg .b16 %rs<4>;
+; SM90-NOFTZ-NEXT: .reg .b32 %r<15>;
+; SM90-NOFTZ-NEXT: .reg .b64 %rd<3>;
+; SM90-NOFTZ-EMPTY:
+; SM90-NOFTZ-NEXT: // %bb.0:
+; SM90-NOFTZ-NEXT: ld.param.b16 %rs1, [fsub_acq_rel_half_global_cta_param_1];
+; SM90-NOFTZ-NEXT: ld.param.b64 %rd2, [fsub_acq_rel_half_global_cta_param_0];
+; SM90-NOFTZ-NEXT: fence.release.cta;
+; SM90-NOFTZ-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NOFTZ-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM90-NOFTZ-NEXT: and.b32 %r5, %r4, 3;
+; SM90-NOFTZ-NEXT: shl.b32 %r1, %r5, 3;
+; SM90-NOFTZ-NEXT: mov.b32 %r6, 65535;
+; SM90-NOFTZ-NEXT: shl.b32 %r7, %r6, %r1;
+; SM90-NOFTZ-NEXT: not.b32 %r2, %r7;
+; SM90-NOFTZ-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM90-NOFTZ-NEXT: $L__BB73_1: // %atomicrmw.start
+; SM90-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NOFTZ-NEXT: shr.u32 %r8, %r14, %r1;
+; SM90-NOFTZ-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM90-NOFTZ-NEXT: sub.rn.f16 %rs3, %rs2, %rs1;
+; SM90-NOFTZ-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM90-NOFTZ-NEXT: shl.b32 %r10, %r9, %r1;
+; SM90-NOFTZ-NEXT: and.b32 %r11, %r14, %r2;
+; SM90-NOFTZ-NEXT: or.b32 %r12, %r11, %r10;
+; SM90-NOFTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM90-NOFTZ-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM90-NOFTZ-NEXT: mov.b32 %r14, %r3;
+; SM90-NOFTZ-NEXT: @%p1 bra $L__BB73_1;
+; SM90-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NOFTZ-NEXT: shr.u32 %r13, %r3, %r1;
+; SM90-NOFTZ-NEXT: fence.acquire.cta;
+; SM90-NOFTZ-NEXT: st.param.b16 [func_retval0], %r13;
+; SM90-NOFTZ-NEXT: ret;
+;
+; SM90-FTZ-LABEL: fsub_acq_rel_half_global_cta(
+; SM90-FTZ: {
+; SM90-FTZ-NEXT: .reg .pred %p<2>;
+; SM90-FTZ-NEXT: .reg .b16 %rs<4>;
+; SM90-FTZ-NEXT: .reg .b32 %r<15>;
+; SM90-FTZ-NEXT: .reg .b64 %rd<3>;
+; SM90-FTZ-EMPTY:
+; SM90-FTZ-NEXT: // %bb.0:
+; SM90-FTZ-NEXT: ld.param.b16 %rs1, [fsub_acq_rel_half_global_cta_param_1];
+; SM90-FTZ-NEXT: ld.param.b64 %rd2, [fsub_acq_rel_half_global_cta_param_0];
+; SM90-FTZ-NEXT: fence.release.cta;
+; SM90-FTZ-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-FTZ-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM90-FTZ-NEXT: and.b32 %r5, %r4, 3;
+; SM90-FTZ-NEXT: shl.b32 %r1, %r5, 3;
+; SM90-FTZ-NEXT: mov.b32 %r6, 65535;
+; SM90-FTZ-NEXT: shl.b32 %r7, %r6, %r1;
+; SM90-FTZ-NEXT: not.b32 %r2, %r7;
+; SM90-FTZ-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM90-FTZ-NEXT: $L__BB73_1: // %atomicrmw.start
+; SM90-FTZ-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-FTZ-NEXT: shr.u32 %r8, %r14, %r1;
+; SM90-FTZ-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM90-FTZ-NEXT: sub.rn.ftz.f16 %rs3, %rs2, %rs1;
+; SM90-FTZ-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM90-FTZ-NEXT: shl.b32 %r10, %r9, %r1;
+; SM90-FTZ-NEXT: and.b32 %r11, %r14, %r2;
+; SM90-FTZ-NEXT: or.b32 %r12, %r11, %r10;
+; SM90-FTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM90-FTZ-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM90-FTZ-NEXT: mov.b32 %r14, %r3;
+; SM90-FTZ-NEXT: @%p1 bra $L__BB73_1;
+; SM90-FTZ-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-FTZ-NEXT: shr.u32 %r13, %r3, %r1;
+; SM90-FTZ-NEXT: fence.acquire.cta;
+; SM90-FTZ-NEXT: st.param.b16 [func_retval0], %r13;
+; SM90-FTZ-NEXT: ret;
%retval = atomicrmw fsub ptr addrspace(1) %addr, half %val syncscope("block") acq_rel
ret half %retval
}
define half @fmin_acq_rel_half_global_cta(ptr addrspace(1) %addr, half %val) {
-; SM90-LABEL: fmin_acq_rel_half_global_cta(
-; SM90: {
-; SM90-NEXT: .reg .pred %p<2>;
-; SM90-NEXT: .reg .b16 %rs<4>;
-; SM90-NEXT: .reg .b32 %r<15>;
-; SM90-NEXT: .reg .b64 %rd<3>;
-; SM90-EMPTY:
-; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b16 %rs1, [fmin_acq_rel_half_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd2, [fmin_acq_rel_half_global_cta_param_0];
-; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: and.b64 %rd1, %rd2, -4;
-; SM90-NEXT: cvt.u32.u64 %r4, %rd2;
-; SM90-NEXT: and.b32 %r5, %r4, 3;
-; SM90-NEXT: shl.b32 %r1, %r5, 3;
-; SM90-NEXT: mov.b32 %r6, 65535;
-; SM90-NEXT: shl.b32 %r7, %r6, %r1;
-; SM90-NEXT: not.b32 %r2, %r7;
-; SM90-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
-; SM90-NEXT: $L__BB74_1: // %atomicrmw.start
-; SM90-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NEXT: shr.u32 %r8, %r14, %r1;
-; SM90-NEXT: cvt.u16.u32 %rs2, %r8;
-; SM90-NEXT: min.f16 %rs3, %rs2, %rs1;
-; SM90-NEXT: cvt.u32.u16 %r9, %rs3;
-; SM90-NEXT: shl.b32 %r10, %r9, %r1;
-; SM90-NEXT: and.b32 %r11, %r14, %r2;
-; SM90-NEXT: or.b32 %r12, %r11, %r10;
-; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
-; SM90-NEXT: setp.ne.b32 %p1, %r3, %r14;
-; SM90-NEXT: mov.b32 %r14, %r3;
-; SM90-NEXT: @%p1 bra $L__BB74_1;
-; SM90-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NEXT: shr.u32 %r13, %r3, %r1;
-; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b16 [func_retval0], %r13;
-; SM90-NEXT: ret;
+; SM90-NOFTZ-LABEL: fmin_acq_rel_half_global_cta(
+; SM90-NOFTZ: {
+; SM90-NOFTZ-NEXT: .reg .pred %p<2>;
+; SM90-NOFTZ-NEXT: .reg .b16 %rs<4>;
+; SM90-NOFTZ-NEXT: .reg .b32 %r<15>;
+; SM90-NOFTZ-NEXT: .reg .b64 %rd<3>;
+; SM90-NOFTZ-EMPTY:
+; SM90-NOFTZ-NEXT: // %bb.0:
+; SM90-NOFTZ-NEXT: ld.param.b16 %rs1, [fmin_acq_rel_half_global_cta_param_1];
+; SM90-NOFTZ-NEXT: ld.param.b64 %rd2, [fmin_acq_rel_half_global_cta_param_0];
+; SM90-NOFTZ-NEXT: fence.release.cta;
+; SM90-NOFTZ-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NOFTZ-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM90-NOFTZ-NEXT: and.b32 %r5, %r4, 3;
+; SM90-NOFTZ-NEXT: shl.b32 %r1, %r5, 3;
+; SM90-NOFTZ-NEXT: mov.b32 %r6, 65535;
+; SM90-NOFTZ-NEXT: shl.b32 %r7, %r6, %r1;
+; SM90-NOFTZ-NEXT: not.b32 %r2, %r7;
+; SM90-NOFTZ-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM90-NOFTZ-NEXT: $L__BB74_1: // %atomicrmw.start
+; SM90-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NOFTZ-NEXT: shr.u32 %r8, %r14, %r1;
+; SM90-NOFTZ-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM90-NOFTZ-NEXT: min.f16 %rs3, %rs2, %rs1;
+; SM90-NOFTZ-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM90-NOFTZ-NEXT: shl.b32 %r10, %r9, %r1;
+; SM90-NOFTZ-NEXT: and.b32 %r11, %r14, %r2;
+; SM90-NOFTZ-NEXT: or.b32 %r12, %r11, %r10;
+; SM90-NOFTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM90-NOFTZ-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM90-NOFTZ-NEXT: mov.b32 %r14, %r3;
+; SM90-NOFTZ-NEXT: @%p1 bra $L__BB74_1;
+; SM90-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NOFTZ-NEXT: shr.u32 %r13, %r3, %r1;
+; SM90-NOFTZ-NEXT: fence.acquire.cta;
+; SM90-NOFTZ-NEXT: st.param.b16 [func_retval0], %r13;
+; SM90-NOFTZ-NEXT: ret;
+;
+; SM90-FTZ-LABEL: fmin_acq_rel_half_global_cta(
+; SM90-FTZ: {
+; SM90-FTZ-NEXT: .reg .pred %p<2>;
+; SM90-FTZ-NEXT: .reg .b16 %rs<4>;
+; SM90-FTZ-NEXT: .reg .b32 %r<15>;
+; SM90-FTZ-NEXT: .reg .b64 %rd<3>;
+; SM90-FTZ-EMPTY:
+; SM90-FTZ-NEXT: // %bb.0:
+; SM90-FTZ-NEXT: ld.param.b16 %rs1, [fmin_acq_rel_half_global_cta_param_1];
+; SM90-FTZ-NEXT: ld.param.b64 %rd2, [fmin_acq_rel_half_global_cta_param_0];
+; SM90-FTZ-NEXT: fence.release.cta;
+; SM90-FTZ-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-FTZ-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM90-FTZ-NEXT: and.b32 %r5, %r4, 3;
+; SM90-FTZ-NEXT: shl.b32 %r1, %r5, 3;
+; SM90-FTZ-NEXT: mov.b32 %r6, 65535;
+; SM90-FTZ-NEXT: shl.b32 %r7, %r6, %r1;
+; SM90-FTZ-NEXT: not.b32 %r2, %r7;
+; SM90-FTZ-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM90-FTZ-NEXT: $L__BB74_1: // %atomicrmw.start
+; SM90-FTZ-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-FTZ-NEXT: shr.u32 %r8, %r14, %r1;
+; SM90-FTZ-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM90-FTZ-NEXT: min.ftz.f16 %rs3, %rs2, %rs1;
+; SM90-FTZ-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM90-FTZ-NEXT: shl.b32 %r10, %r9, %r1;
+; SM90-FTZ-NEXT: and.b32 %r11, %r14, %r2;
+; SM90-FTZ-NEXT: or.b32 %r12, %r11, %r10;
+; SM90-FTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM90-FTZ-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM90-FTZ-NEXT: mov.b32 %r14, %r3;
+; SM90-FTZ-NEXT: @%p1 bra $L__BB74_1;
+; SM90-FTZ-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-FTZ-NEXT: shr.u32 %r13, %r3, %r1;
+; SM90-FTZ-NEXT: fence.acquire.cta;
+; SM90-FTZ-NEXT: st.param.b16 [func_retval0], %r13;
+; SM90-FTZ-NEXT: ret;
%retval = atomicrmw fmin ptr addrspace(1) %addr, half %val syncscope("block") acq_rel
ret half %retval
}
define half @fmax_acq_rel_half_global_cta(ptr addrspace(1) %addr, half %val) {
-; SM90-LABEL: fmax_acq_rel_half_global_cta(
-; SM90: {
-; SM90-NEXT: .reg .pred %p<2>;
-; SM90-NEXT: .reg .b16 %rs<4>;
-; SM90-NEXT: .reg .b32 %r<15>;
-; SM90-NEXT: .reg .b64 %rd<3>;
-; SM90-EMPTY:
-; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b16 %rs1, [fmax_acq_rel_half_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd2, [fmax_acq_rel_half_global_cta_param_0];
-; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: and.b64 %rd1, %rd2, -4;
-; SM90-NEXT: cvt.u32.u64 %r4, %rd2;
-; SM90-NEXT: and.b32 %r5, %r4, 3;
-; SM90-NEXT: shl.b32 %r1, %r5, 3;
-; SM90-NEXT: mov.b32 %r6, 65535;
-; SM90-NEXT: shl.b32 %r7, %r6, %r1;
-; SM90-NEXT: not.b32 %r2, %r7;
-; SM90-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
-; SM90-NEXT: $L__BB75_1: // %atomicrmw.start
-; SM90-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NEXT: shr.u32 %r8, %r14, %r1;
-; SM90-NEXT: cvt.u16.u32 %rs2, %r8;
-; SM90-NEXT: max.f16 %rs3, %rs2, %rs1;
-; SM90-NEXT: cvt.u32.u16 %r9, %rs3;
-; SM90-NEXT: shl.b32 %r10, %r9, %r1;
-; SM90-NEXT: and.b32 %r11, %r14, %r2;
-; SM90-NEXT: or.b32 %r12, %r11, %r10;
-; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
-; SM90-NEXT: setp.ne.b32 %p1, %r3, %r14;
-; SM90-NEXT: mov.b32 %r14, %r3;
-; SM90-NEXT: @%p1 bra $L__BB75_1;
-; SM90-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NEXT: shr.u32 %r13, %r3, %r1;
-; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b16 [func_retval0], %r13;
-; SM90-NEXT: ret;
+; SM90-NOFTZ-LABEL: fmax_acq_rel_half_global_cta(
+; SM90-NOFTZ: {
+; SM90-NOFTZ-NEXT: .reg .pred %p<2>;
+; SM90-NOFTZ-NEXT: .reg .b16 %rs<4>;
+; SM90-NOFTZ-NEXT: .reg .b32 %r<15>;
+; SM90-NOFTZ-NEXT: .reg .b64 %rd<3>;
+; SM90-NOFTZ-EMPTY:
+; SM90-NOFTZ-NEXT: // %bb.0:
+; SM90-NOFTZ-NEXT: ld.param.b16 %rs1, [fmax_acq_rel_half_global_cta_param_1];
+; SM90-NOFTZ-NEXT: ld.param.b64 %rd2, [fmax_acq_rel_half_global_cta_param_0];
+; SM90-NOFTZ-NEXT: fence.release.cta;
+; SM90-NOFTZ-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NOFTZ-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM90-NOFTZ-NEXT: and.b32 %r5, %r4, 3;
+; SM90-NOFTZ-NEXT: shl.b32 %r1, %r5, 3;
+; SM90-NOFTZ-NEXT: mov.b32 %r6, 65535;
+; SM90-NOFTZ-NEXT: shl.b32 %r7, %r6, %r1;
+; SM90-NOFTZ-NEXT: not.b32 %r2, %r7;
+; SM90-NOFTZ-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM90-NOFTZ-NEXT: $L__BB75_1: // %atomicrmw.start
+; SM90-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NOFTZ-NEXT: shr.u32 %r8, %r14, %r1;
+; SM90-NOFTZ-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM90-NOFTZ-NEXT: max.f16 %rs3, %rs2, %rs1;
+; SM90-NOFTZ-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM90-NOFTZ-NEXT: shl.b32 %r10, %r9, %r1;
+; SM90-NOFTZ-NEXT: and.b32 %r11, %r14, %r2;
+; SM90-NOFTZ-NEXT: or.b32 %r12, %r11, %r10;
+; SM90-NOFTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM90-NOFTZ-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM90-NOFTZ-NEXT: mov.b32 %r14, %r3;
+; SM90-NOFTZ-NEXT: @%p1 bra $L__BB75_1;
+; SM90-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NOFTZ-NEXT: shr.u32 %r13, %r3, %r1;
+; SM90-NOFTZ-NEXT: fence.acquire.cta;
+; SM90-NOFTZ-NEXT: st.param.b16 [func_retval0], %r13;
+; SM90-NOFTZ-NEXT: ret;
+;
+; SM90-FTZ-LABEL: fmax_acq_rel_half_global_cta(
+; SM90-FTZ: {
+; SM90-FTZ-NEXT: .reg .pred %p<2>;
+; SM90-FTZ-NEXT: .reg .b16 %rs<4>;
+; SM90-FTZ-NEXT: .reg .b32 %r<15>;
+; SM90-FTZ-NEXT: .reg .b64 %rd<3>;
+; SM90-FTZ-EMPTY:
+; SM90-FTZ-NEXT: // %bb.0:
+; SM90-FTZ-NEXT: ld.param.b16 %rs1, [fmax_acq_rel_half_global_cta_param_1];
+; SM90-FTZ-NEXT: ld.param.b64 %rd2, [fmax_acq_rel_half_global_cta_param_0];
+; SM90-FTZ-NEXT: fence.release.cta;
+; SM90-FTZ-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-FTZ-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM90-FTZ-NEXT: and.b32 %r5, %r4, 3;
+; SM90-FTZ-NEXT: shl.b32 %r1, %r5, 3;
+; SM90-FTZ-NEXT: mov.b32 %r6, 65535;
+; SM90-FTZ-NEXT: shl.b32 %r7, %r6, %r1;
+; SM90-FTZ-NEXT: not.b32 %r2, %r7;
+; SM90-FTZ-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM90-FTZ-NEXT: $L__BB75_1: // %atomicrmw.start
+; SM90-FTZ-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-FTZ-NEXT: shr.u32 %r8, %r14, %r1;
+; SM90-FTZ-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM90-FTZ-NEXT: max.ftz.f16 %rs3, %rs2, %rs1;
+; SM90-FTZ-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM90-FTZ-NEXT: shl.b32 %r10, %r9, %r1;
+; SM90-FTZ-NEXT: and.b32 %r11, %r14, %r2;
+; SM90-FTZ-NEXT: or.b32 %r12, %r11, %r10;
+; SM90-FTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM90-FTZ-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM90-FTZ-NEXT: mov.b32 %r14, %r3;
+; SM90-FTZ-NEXT: @%p1 bra $L__BB75_1;
+; SM90-FTZ-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-FTZ-NEXT: shr.u32 %r13, %r3, %r1;
+; SM90-FTZ-NEXT: fence.acquire.cta;
+; SM90-FTZ-NEXT: st.param.b16 [func_retval0], %r13;
+; SM90-FTZ-NEXT: ret;
%retval = atomicrmw fmax ptr addrspace(1) %addr, half %val syncscope("block") acq_rel
ret half %retval
}
define half @fminimum_acq_rel_half_global_cta(ptr addrspace(1) %addr, half %val) {
-; SM90-LABEL: fminimum_acq_rel_half_global_cta(
-; SM90: {
-; SM90-NEXT: .reg .pred %p<2>;
-; SM90-NEXT: .reg .b16 %rs<4>;
-; SM90-NEXT: .reg .b32 %r<15>;
-; SM90-NEXT: .reg .b64 %rd<3>;
-; SM90-EMPTY:
-; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b16 %rs1, [fminimum_acq_rel_half_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd2, [fminimum_acq_rel_half_global_cta_param_0];
-; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: and.b64 %rd1, %rd2, -4;
-; SM90-NEXT: cvt.u32.u64 %r4, %rd2;
-; SM90-NEXT: and.b32 %r5, %r4, 3;
-; SM90-NEXT: shl.b32 %r1, %r5, 3;
-; SM90-NEXT: mov.b32 %r6, 65535;
-; SM90-NEXT: shl.b32 %r7, %r6, %r1;
-; SM90-NEXT: not.b32 %r2, %r7;
-; SM90-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
-; SM90-NEXT: $L__BB76_1: // %atomicrmw.start
-; SM90-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NEXT: shr.u32 %r8, %r14, %r1;
-; SM90-NEXT: cvt.u16.u32 %rs2, %r8;
-; SM90-NEXT: min.NaN.f16 %rs3, %rs2, %rs1;
-; SM90-NEXT: cvt.u32.u16 %r9, %rs3;
-; SM90-NEXT: shl.b32 %r10, %r9, %r1;
-; SM90-NEXT: and.b32 %r11, %r14, %r2;
-; SM90-NEXT: or.b32 %r12, %r11, %r10;
-; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
-; SM90-NEXT: setp.ne.b32 %p1, %r3, %r14;
-; SM90-NEXT: mov.b32 %r14, %r3;
-; SM90-NEXT: @%p1 bra $L__BB76_1;
-; SM90-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NEXT: shr.u32 %r13, %r3, %r1;
-; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b16 [func_retval0], %r13;
-; SM90-NEXT: ret;
+; SM90-NOFTZ-LABEL: fminimum_acq_rel_half_global_cta(
+; SM90-NOFTZ: {
+; SM90-NOFTZ-NEXT: .reg .pred %p<2>;
+; SM90-NOFTZ-NEXT: .reg .b16 %rs<4>;
+; SM90-NOFTZ-NEXT: .reg .b32 %r<15>;
+; SM90-NOFTZ-NEXT: .reg .b64 %rd<3>;
+; SM90-NOFTZ-EMPTY:
+; SM90-NOFTZ-NEXT: // %bb.0:
+; SM90-NOFTZ-NEXT: ld.param.b16 %rs1, [fminimum_acq_rel_half_global_cta_param_1];
+; SM90-NOFTZ-NEXT: ld.param.b64 %rd2, [fminimum_acq_rel_half_global_cta_param_0];
+; SM90-NOFTZ-NEXT: fence.release.cta;
+; SM90-NOFTZ-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NOFTZ-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM90-NOFTZ-NEXT: and.b32 %r5, %r4, 3;
+; SM90-NOFTZ-NEXT: shl.b32 %r1, %r5, 3;
+; SM90-NOFTZ-NEXT: mov.b32 %r6, 65535;
+; SM90-NOFTZ-NEXT: shl.b32 %r7, %r6, %r1;
+; SM90-NOFTZ-NEXT: not.b32 %r2, %r7;
+; SM90-NOFTZ-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM90-NOFTZ-NEXT: $L__BB76_1: // %atomicrmw.start
+; SM90-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NOFTZ-NEXT: shr.u32 %r8, %r14, %r1;
+; SM90-NOFTZ-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM90-NOFTZ-NEXT: min.NaN.f16 %rs3, %rs2, %rs1;
+; SM90-NOFTZ-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM90-NOFTZ-NEXT: shl.b32 %r10, %r9, %r1;
+; SM90-NOFTZ-NEXT: and.b32 %r11, %r14, %r2;
+; SM90-NOFTZ-NEXT: or.b32 %r12, %r11, %r10;
+; SM90-NOFTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM90-NOFTZ-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM90-NOFTZ-NEXT: mov.b32 %r14, %r3;
+; SM90-NOFTZ-NEXT: @%p1 bra $L__BB76_1;
+; SM90-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NOFTZ-NEXT: shr.u32 %r13, %r3, %r1;
+; SM90-NOFTZ-NEXT: fence.acquire.cta;
+; SM90-NOFTZ-NEXT: st.param.b16 [func_retval0], %r13;
+; SM90-NOFTZ-NEXT: ret;
+;
+; SM90-FTZ-LABEL: fminimum_acq_rel_half_global_cta(
+; SM90-FTZ: {
+; SM90-FTZ-NEXT: .reg .pred %p<2>;
+; SM90-FTZ-NEXT: .reg .b16 %rs<4>;
+; SM90-FTZ-NEXT: .reg .b32 %r<15>;
+; SM90-FTZ-NEXT: .reg .b64 %rd<3>;
+; SM90-FTZ-EMPTY:
+; SM90-FTZ-NEXT: // %bb.0:
+; SM90-FTZ-NEXT: ld.param.b16 %rs1, [fminimum_acq_rel_half_global_cta_param_1];
+; SM90-FTZ-NEXT: ld.param.b64 %rd2, [fminimum_acq_rel_half_global_cta_param_0];
+; SM90-FTZ-NEXT: fence.release.cta;
+; SM90-FTZ-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-FTZ-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM90-FTZ-NEXT: and.b32 %r5, %r4, 3;
+; SM90-FTZ-NEXT: shl.b32 %r1, %r5, 3;
+; SM90-FTZ-NEXT: mov.b32 %r6, 65535;
+; SM90-FTZ-NEXT: shl.b32 %r7, %r6, %r1;
+; SM90-FTZ-NEXT: not.b32 %r2, %r7;
+; SM90-FTZ-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM90-FTZ-NEXT: $L__BB76_1: // %atomicrmw.start
+; SM90-FTZ-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-FTZ-NEXT: shr.u32 %r8, %r14, %r1;
+; SM90-FTZ-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM90-FTZ-NEXT: min.ftz.NaN.f16 %rs3, %rs2, %rs1;
+; SM90-FTZ-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM90-FTZ-NEXT: shl.b32 %r10, %r9, %r1;
+; SM90-FTZ-NEXT: and.b32 %r11, %r14, %r2;
+; SM90-FTZ-NEXT: or.b32 %r12, %r11, %r10;
+; SM90-FTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM90-FTZ-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM90-FTZ-NEXT: mov.b32 %r14, %r3;
+; SM90-FTZ-NEXT: @%p1 bra $L__BB76_1;
+; SM90-FTZ-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-FTZ-NEXT: shr.u32 %r13, %r3, %r1;
+; SM90-FTZ-NEXT: fence.acquire.cta;
+; SM90-FTZ-NEXT: st.param.b16 [func_retval0], %r13;
+; SM90-FTZ-NEXT: ret;
%retval = atomicrmw fminimum ptr addrspace(1) %addr, half %val syncscope("block") acq_rel
ret half %retval
}
define half @fmaximum_acq_rel_half_global_cta(ptr addrspace(1) %addr, half %val) {
-; SM90-LABEL: fmaximum_acq_rel_half_global_cta(
-; SM90: {
-; SM90-NEXT: .reg .pred %p<2>;
-; SM90-NEXT: .reg .b16 %rs<4>;
-; SM90-NEXT: .reg .b32 %r<15>;
-; SM90-NEXT: .reg .b64 %rd<3>;
-; SM90-EMPTY:
-; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b16 %rs1, [fmaximum_acq_rel_half_global_cta_param_1];
-; SM90-NEXT: ld.param.b64 %rd2, [fmaximum_acq_rel_half_global_cta_param_0];
-; SM90-NEXT: fence.release.cta;
-; SM90-NEXT: and.b64 %rd1, %rd2, -4;
-; SM90-NEXT: cvt.u32.u64 %r4, %rd2;
-; SM90-NEXT: and.b32 %r5, %r4, 3;
-; SM90-NEXT: shl.b32 %r1, %r5, 3;
-; SM90-NEXT: mov.b32 %r6, 65535;
-; SM90-NEXT: shl.b32 %r7, %r6, %r1;
-; SM90-NEXT: not.b32 %r2, %r7;
-; SM90-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
-; SM90-NEXT: $L__BB77_1: // %atomicrmw.start
-; SM90-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NEXT: shr.u32 %r8, %r14, %r1;
-; SM90-NEXT: cvt.u16.u32 %rs2, %r8;
-; SM90-NEXT: max.NaN.f16 %rs3, %rs2, %rs1;
-; SM90-NEXT: cvt.u32.u16 %r9, %rs3;
-; SM90-NEXT: shl.b32 %r10, %r9, %r1;
-; SM90-NEXT: and.b32 %r11, %r14, %r2;
-; SM90-NEXT: or.b32 %r12, %r11, %r10;
-; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
-; SM90-NEXT: setp.ne.b32 %p1, %r3, %r14;
-; SM90-NEXT: mov.b32 %r14, %r3;
-; SM90-NEXT: @%p1 bra $L__BB77_1;
-; SM90-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NEXT: shr.u32 %r13, %r3, %r1;
-; SM90-NEXT: fence.acquire.cta;
-; SM90-NEXT: st.param.b16 [func_retval0], %r13;
-; SM90-NEXT: ret;
+; SM90-NOFTZ-LABEL: fmaximum_acq_rel_half_global_cta(
+; SM90-NOFTZ: {
+; SM90-NOFTZ-NEXT: .reg .pred %p<2>;
+; SM90-NOFTZ-NEXT: .reg .b16 %rs<4>;
+; SM90-NOFTZ-NEXT: .reg .b32 %r<15>;
+; SM90-NOFTZ-NEXT: .reg .b64 %rd<3>;
+; SM90-NOFTZ-EMPTY:
+; SM90-NOFTZ-NEXT: // %bb.0:
+; SM90-NOFTZ-NEXT: ld.param.b16 %rs1, [fmaximum_acq_rel_half_global_cta_param_1];
+; SM90-NOFTZ-NEXT: ld.param.b64 %rd2, [fmaximum_acq_rel_half_global_cta_param_0];
+; SM90-NOFTZ-NEXT: fence.release.cta;
+; SM90-NOFTZ-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NOFTZ-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM90-NOFTZ-NEXT: and.b32 %r5, %r4, 3;
+; SM90-NOFTZ-NEXT: shl.b32 %r1, %r5, 3;
+; SM90-NOFTZ-NEXT: mov.b32 %r6, 65535;
+; SM90-NOFTZ-NEXT: shl.b32 %r7, %r6, %r1;
+; SM90-NOFTZ-NEXT: not.b32 %r2, %r7;
+; SM90-NOFTZ-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM90-NOFTZ-NEXT: $L__BB77_1: // %atomicrmw.start
+; SM90-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NOFTZ-NEXT: shr.u32 %r8, %r14, %r1;
+; SM90-NOFTZ-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM90-NOFTZ-NEXT: max.NaN.f16 %rs3, %rs2, %rs1;
+; SM90-NOFTZ-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM90-NOFTZ-NEXT: shl.b32 %r10, %r9, %r1;
+; SM90-NOFTZ-NEXT: and.b32 %r11, %r14, %r2;
+; SM90-NOFTZ-NEXT: or.b32 %r12, %r11, %r10;
+; SM90-NOFTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM90-NOFTZ-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM90-NOFTZ-NEXT: mov.b32 %r14, %r3;
+; SM90-NOFTZ-NEXT: @%p1 bra $L__BB77_1;
+; SM90-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NOFTZ-NEXT: shr.u32 %r13, %r3, %r1;
+; SM90-NOFTZ-NEXT: fence.acquire.cta;
+; SM90-NOFTZ-NEXT: st.param.b16 [func_retval0], %r13;
+; SM90-NOFTZ-NEXT: ret;
+;
+; SM90-FTZ-LABEL: fmaximum_acq_rel_half_global_cta(
+; SM90-FTZ: {
+; SM90-FTZ-NEXT: .reg .pred %p<2>;
+; SM90-FTZ-NEXT: .reg .b16 %rs<4>;
+; SM90-FTZ-NEXT: .reg .b32 %r<15>;
+; SM90-FTZ-NEXT: .reg .b64 %rd<3>;
+; SM90-FTZ-EMPTY:
+; SM90-FTZ-NEXT: // %bb.0:
+; SM90-FTZ-NEXT: ld.param.b16 %rs1, [fmaximum_acq_rel_half_global_cta_param_1];
+; SM90-FTZ-NEXT: ld.param.b64 %rd2, [fmaximum_acq_rel_half_global_cta_param_0];
+; SM90-FTZ-NEXT: fence.release.cta;
+; SM90-FTZ-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-FTZ-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM90-FTZ-NEXT: and.b32 %r5, %r4, 3;
+; SM90-FTZ-NEXT: shl.b32 %r1, %r5, 3;
+; SM90-FTZ-NEXT: mov.b32 %r6, 65535;
+; SM90-FTZ-NEXT: shl.b32 %r7, %r6, %r1;
+; SM90-FTZ-NEXT: not.b32 %r2, %r7;
+; SM90-FTZ-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM90-FTZ-NEXT: $L__BB77_1: // %atomicrmw.start
+; SM90-FTZ-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-FTZ-NEXT: shr.u32 %r8, %r14, %r1;
+; SM90-FTZ-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM90-FTZ-NEXT: max.ftz.NaN.f16 %rs3, %rs2, %rs1;
+; SM90-FTZ-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM90-FTZ-NEXT: shl.b32 %r10, %r9, %r1;
+; SM90-FTZ-NEXT: and.b32 %r11, %r14, %r2;
+; SM90-FTZ-NEXT: or.b32 %r12, %r11, %r10;
+; SM90-FTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM90-FTZ-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM90-FTZ-NEXT: mov.b32 %r14, %r3;
+; SM90-FTZ-NEXT: @%p1 bra $L__BB77_1;
+; SM90-FTZ-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-FTZ-NEXT: shr.u32 %r13, %r3, %r1;
+; SM90-FTZ-NEXT: fence.acquire.cta;
+; SM90-FTZ-NEXT: st.param.b16 [func_retval0], %r13;
+; SM90-FTZ-NEXT: ret;
%retval = atomicrmw fmaximum ptr addrspace(1) %addr, half %val syncscope("block") acq_rel
ret half %retval
}
@@ -3100,17 +3468,40 @@ define i64 @nand_acq_rel_i64_global_sys(ptr addrspace(1) %addr, i64 %val) {
}
define float @fadd_acq_rel_float_global_sys(ptr addrspace(1) %addr, float %val) {
-; SM90-LABEL: fadd_acq_rel_float_global_sys(
-; SM90: {
-; SM90-NEXT: .reg .b32 %r<3>;
-; SM90-NEXT: .reg .b64 %rd<2>;
-; SM90-EMPTY:
-; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_sys_param_0];
-; SM90-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_global_sys_param_1];
-; SM90-NEXT: atom.acq_rel.sys.global.add.f32 %r2, [%rd1], %r1;
-; SM90-NEXT: st.param.b32 [func_retval0], %r2;
-; SM90-NEXT: ret;
+; SM90-NOFTZ-LABEL: fadd_acq_rel_float_global_sys(
+; SM90-NOFTZ: {
+; SM90-NOFTZ-NEXT: .reg .pred %p<2>;
+; SM90-NOFTZ-NEXT: .reg .b32 %r<5>;
+; SM90-NOFTZ-NEXT: .reg .b64 %rd<2>;
+; SM90-NOFTZ-EMPTY:
+; SM90-NOFTZ-NEXT: // %bb.0:
+; SM90-NOFTZ-NEXT: ld.param.b32 %r2, [fadd_acq_rel_float_global_sys_param_1];
+; SM90-NOFTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_sys_param_0];
+; SM90-NOFTZ-NEXT: fence.release.sys;
+; SM90-NOFTZ-NEXT: ld.relaxed.sys.global.b32 %r4, [%rd1];
+; SM90-NOFTZ-NEXT: $L__BB106_1: // %atomicrmw.start
+; SM90-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NOFTZ-NEXT: add.rn.f32 %r3, %r4, %r2;
+; SM90-NOFTZ-NEXT: atom.relaxed.sys.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM90-NOFTZ-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM90-NOFTZ-NEXT: mov.b32 %r4, %r1;
+; SM90-NOFTZ-NEXT: @%p1 bra $L__BB106_1;
+; SM90-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NOFTZ-NEXT: fence.acquire.sys;
+; SM90-NOFTZ-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NOFTZ-NEXT: ret;
+;
+; SM90-FTZ-LABEL: fadd_acq_rel_float_global_sys(
+; SM90-FTZ: {
+; SM90-FTZ-NEXT: .reg .b32 %r<3>;
+; SM90-FTZ-NEXT: .reg .b64 %rd<2>;
+; SM90-FTZ-EMPTY:
+; SM90-FTZ-NEXT: // %bb.0:
+; SM90-FTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_sys_param_0];
+; SM90-FTZ-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_global_sys_param_1];
+; SM90-FTZ-NEXT: atom.acq_rel.sys.global.add.f32 %r2, [%rd1], %r1;
+; SM90-FTZ-NEXT: st.param.b32 [func_retval0], %r2;
+; SM90-FTZ-NEXT: ret;
%retval = atomicrmw fadd ptr addrspace(1) %addr, float %val syncscope("") acq_rel
ret float %retval
}
@@ -3249,17 +3640,40 @@ define i64 @nand_acq_rel_i64_global_cluster(ptr addrspace(1) %addr, i64 %val) {
}
define float @fadd_acq_rel_float_global_cluster(ptr addrspace(1) %addr, float %val) {
-; SM90-LABEL: fadd_acq_rel_float_global_cluster(
-; SM90: {
-; SM90-NEXT: .reg .b32 %r<3>;
-; SM90-NEXT: .reg .b64 %rd<2>;
-; SM90-EMPTY:
-; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_cluster_param_0];
-; SM90-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_global_cluster_param_1];
-; SM90-NEXT: atom.acq_rel.cluster.global.add.f32 %r2, [%rd1], %r1;
-; SM90-NEXT: st.param.b32 [func_retval0], %r2;
-; SM90-NEXT: ret;
+; SM90-NOFTZ-LABEL: fadd_acq_rel_float_global_cluster(
+; SM90-NOFTZ: {
+; SM90-NOFTZ-NEXT: .reg .pred %p<2>;
+; SM90-NOFTZ-NEXT: .reg .b32 %r<5>;
+; SM90-NOFTZ-NEXT: .reg .b64 %rd<2>;
+; SM90-NOFTZ-EMPTY:
+; SM90-NOFTZ-NEXT: // %bb.0:
+; SM90-NOFTZ-NEXT: ld.param.b32 %r2, [fadd_acq_rel_float_global_cluster_param_1];
+; SM90-NOFTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_cluster_param_0];
+; SM90-NOFTZ-NEXT: fence.release.cluster;
+; SM90-NOFTZ-NEXT: ld.relaxed.cluster.global.b32 %r4, [%rd1];
+; SM90-NOFTZ-NEXT: $L__BB114_1: // %atomicrmw.start
+; SM90-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NOFTZ-NEXT: add.rn.f32 %r3, %r4, %r2;
+; SM90-NOFTZ-NEXT: atom.relaxed.cluster.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM90-NOFTZ-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM90-NOFTZ-NEXT: mov.b32 %r4, %r1;
+; SM90-NOFTZ-NEXT: @%p1 bra $L__BB114_1;
+; SM90-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NOFTZ-NEXT: fence.acquire.cluster;
+; SM90-NOFTZ-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NOFTZ-NEXT: ret;
+;
+; SM90-FTZ-LABEL: fadd_acq_rel_float_global_cluster(
+; SM90-FTZ: {
+; SM90-FTZ-NEXT: .reg .b32 %r<3>;
+; SM90-FTZ-NEXT: .reg .b64 %rd<2>;
+; SM90-FTZ-EMPTY:
+; SM90-FTZ-NEXT: // %bb.0:
+; SM90-FTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_cluster_param_0];
+; SM90-FTZ-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_global_cluster_param_1];
+; SM90-FTZ-NEXT: atom.acq_rel.cluster.global.add.f32 %r2, [%rd1], %r1;
+; SM90-FTZ-NEXT: st.param.b32 [func_retval0], %r2;
+; SM90-FTZ-NEXT: ret;
%retval = atomicrmw fadd ptr addrspace(1) %addr, float %val syncscope("cluster") acq_rel
ret float %retval
}
@@ -3398,17 +3812,40 @@ define i64 @nand_acq_rel_i64_global_gpu(ptr addrspace(1) %addr, i64 %val) {
}
define float @fadd_acq_rel_float_global_gpu(ptr addrspace(1) %addr, float %val) {
-; SM90-LABEL: fadd_acq_rel_float_global_gpu(
-; SM90: {
-; SM90-NEXT: .reg .b32 %r<3>;
-; SM90-NEXT: .reg .b64 %rd<2>;
-; SM90-EMPTY:
-; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_gpu_param_0];
-; SM90-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_global_gpu_param_1];
-; SM90-NEXT: atom.acq_rel.gpu.global.add.f32 %r2, [%rd1], %r1;
-; SM90-NEXT: st.param.b32 [func_retval0], %r2;
-; SM90-NEXT: ret;
+; SM90-NOFTZ-LABEL: fadd_acq_rel_float_global_gpu(
+; SM90-NOFTZ: {
+; SM90-NOFTZ-NEXT: .reg .pred %p<2>;
+; SM90-NOFTZ-NEXT: .reg .b32 %r<5>;
+; SM90-NOFTZ-NEXT: .reg .b64 %rd<2>;
+; SM90-NOFTZ-EMPTY:
+; SM90-NOFTZ-NEXT: // %bb.0:
+; SM90-NOFTZ-NEXT: ld.param.b32 %r2, [fadd_acq_rel_float_global_gpu_param_1];
+; SM90-NOFTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_gpu_param_0];
+; SM90-NOFTZ-NEXT: fence.release.gpu;
+; SM90-NOFTZ-NEXT: ld.relaxed.gpu.global.b32 %r4, [%rd1];
+; SM90-NOFTZ-NEXT: $L__BB122_1: // %atomicrmw.start
+; SM90-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NOFTZ-NEXT: add.rn.f32 %r3, %r4, %r2;
+; SM90-NOFTZ-NEXT: atom.relaxed.gpu.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM90-NOFTZ-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM90-NOFTZ-NEXT: mov.b32 %r4, %r1;
+; SM90-NOFTZ-NEXT: @%p1 bra $L__BB122_1;
+; SM90-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NOFTZ-NEXT: fence.acquire.gpu;
+; SM90-NOFTZ-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NOFTZ-NEXT: ret;
+;
+; SM90-FTZ-LABEL: fadd_acq_rel_float_global_gpu(
+; SM90-FTZ: {
+; SM90-FTZ-NEXT: .reg .b32 %r<3>;
+; SM90-FTZ-NEXT: .reg .b64 %rd<2>;
+; SM90-FTZ-EMPTY:
+; SM90-FTZ-NEXT: // %bb.0:
+; SM90-FTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_gpu_param_0];
+; SM90-FTZ-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_global_gpu_param_1];
+; SM90-FTZ-NEXT: atom.acq_rel.gpu.global.add.f32 %r2, [%rd1], %r1;
+; SM90-FTZ-NEXT: st.param.b32 [func_retval0], %r2;
+; SM90-FTZ-NEXT: ret;
%retval = atomicrmw fadd ptr addrspace(1) %addr, float %val syncscope("device") acq_rel
ret float %retval
}
@@ -3547,17 +3984,51 @@ define i64 @nand_acq_rel_i64_generic_cta(ptr %addr, i64 %val) {
}
define float @fadd_acq_rel_float_generic_cta(ptr %addr, float %val) {
-; SM90-LABEL: fadd_acq_rel_float_generic_cta(
-; SM90: {
-; SM90-NEXT: .reg .b32 %r<3>;
-; SM90-NEXT: .reg .b64 %rd<2>;
-; SM90-EMPTY:
-; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_generic_cta_param_0];
-; SM90-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_generic_cta_param_1];
-; SM90-NEXT: atom.acq_rel.cta.add.f32 %r2, [%rd1], %r1;
-; SM90-NEXT: st.param.b32 [func_retval0], %r2;
-; SM90-NEXT: ret;
+; SM90-NOFTZ-LABEL: fadd_acq_rel_float_generic_cta(
+; SM90-NOFTZ: {
+; SM90-NOFTZ-NEXT: .reg .pred %p<2>;
+; SM90-NOFTZ-NEXT: .reg .b32 %r<5>;
+; SM90-NOFTZ-NEXT: .reg .b64 %rd<2>;
+; SM90-NOFTZ-EMPTY:
+; SM90-NOFTZ-NEXT: // %bb.0:
+; SM90-NOFTZ-NEXT: ld.param.b32 %r2, [fadd_acq_rel_float_generic_cta_param_1];
+; SM90-NOFTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_generic_cta_param_0];
+; SM90-NOFTZ-NEXT: fence.release.cta;
+; SM90-NOFTZ-NEXT: ld.relaxed.cta.b32 %r4, [%rd1];
+; SM90-NOFTZ-NEXT: $L__BB130_1: // %atomicrmw.start
+; SM90-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NOFTZ-NEXT: add.rn.f32 %r3, %r4, %r2;
+; SM90-NOFTZ-NEXT: atom.relaxed.cta.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM90-NOFTZ-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM90-NOFTZ-NEXT: mov.b32 %r4, %r1;
+; SM90-NOFTZ-NEXT: @%p1 bra $L__BB130_1;
+; SM90-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NOFTZ-NEXT: fence.acquire.cta;
+; SM90-NOFTZ-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NOFTZ-NEXT: ret;
+;
+; SM90-FTZ-LABEL: fadd_acq_rel_float_generic_cta(
+; SM90-FTZ: {
+; SM90-FTZ-NEXT: .reg .pred %p<2>;
+; SM90-FTZ-NEXT: .reg .b32 %r<5>;
+; SM90-FTZ-NEXT: .reg .b64 %rd<2>;
+; SM90-FTZ-EMPTY:
+; SM90-FTZ-NEXT: // %bb.0:
+; SM90-FTZ-NEXT: ld.param.b32 %r2, [fadd_acq_rel_float_generic_cta_param_1];
+; SM90-FTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_generic_cta_param_0];
+; SM90-FTZ-NEXT: fence.release.cta;
+; SM90-FTZ-NEXT: ld.relaxed.cta.b32 %r4, [%rd1];
+; SM90-FTZ-NEXT: $L__BB130_1: // %atomicrmw.start
+; SM90-FTZ-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-FTZ-NEXT: add.rn.ftz.f32 %r3, %r4, %r2;
+; SM90-FTZ-NEXT: atom.relaxed.cta.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM90-FTZ-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM90-FTZ-NEXT: mov.b32 %r4, %r1;
+; SM90-FTZ-NEXT: @%p1 bra $L__BB130_1;
+; SM90-FTZ-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-FTZ-NEXT: fence.acquire.cta;
+; SM90-FTZ-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-FTZ-NEXT: ret;
%retval = atomicrmw fadd ptr %addr, float %val syncscope("block") acq_rel
ret float %retval
}
@@ -3696,17 +4167,40 @@ define i64 @nand_acq_rel_i64_shared_cta(ptr addrspace(3) %addr, i64 %val) {
}
define float @fadd_acq_rel_float_shared_cta(ptr addrspace(3) %addr, float %val) {
-; SM90-LABEL: fadd_acq_rel_float_shared_cta(
-; SM90: {
-; SM90-NEXT: .reg .b32 %r<3>;
-; SM90-NEXT: .reg .b64 %rd<2>;
-; SM90-EMPTY:
-; SM90-NEXT: // %bb.0:
-; SM90-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_shared_cta_param_0];
-; SM90-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_shared_cta_param_1];
-; SM90-NEXT: atom.acq_rel.cta.shared.add.f32 %r2, [%rd1], %r1;
-; SM90-NEXT: st.param.b32 [func_retval0], %r2;
-; SM90-NEXT: ret;
+; SM90-NOFTZ-LABEL: fadd_acq_rel_float_shared_cta(
+; SM90-NOFTZ: {
+; SM90-NOFTZ-NEXT: .reg .b32 %r<3>;
+; SM90-NOFTZ-NEXT: .reg .b64 %rd<2>;
+; SM90-NOFTZ-EMPTY:
+; SM90-NOFTZ-NEXT: // %bb.0:
+; SM90-NOFTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_shared_cta_param_0];
+; SM90-NOFTZ-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_shared_cta_param_1];
+; SM90-NOFTZ-NEXT: atom.acq_rel.cta.shared.add.f32 %r2, [%rd1], %r1;
+; SM90-NOFTZ-NEXT: st.param.b32 [func_retval0], %r2;
+; SM90-NOFTZ-NEXT: ret;
+;
+; SM90-FTZ-LABEL: fadd_acq_rel_float_shared_cta(
+; SM90-FTZ: {
+; SM90-FTZ-NEXT: .reg .pred %p<2>;
+; SM90-FTZ-NEXT: .reg .b32 %r<5>;
+; SM90-FTZ-NEXT: .reg .b64 %rd<2>;
+; SM90-FTZ-EMPTY:
+; SM90-FTZ-NEXT: // %bb.0:
+; SM90-FTZ-NEXT: ld.param.b32 %r2, [fadd_acq_rel_float_shared_cta_param_1];
+; SM90-FTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_shared_cta_param_0];
+; SM90-FTZ-NEXT: fence.release.cta;
+; SM90-FTZ-NEXT: ld.relaxed.cta.shared.b32 %r4, [%rd1];
+; SM90-FTZ-NEXT: $L__BB138_1: // %atomicrmw.start
+; SM90-FTZ-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-FTZ-NEXT: add.rn.ftz.f32 %r3, %r4, %r2;
+; SM90-FTZ-NEXT: atom.relaxed.cta.shared.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM90-FTZ-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM90-FTZ-NEXT: mov.b32 %r4, %r1;
+; SM90-FTZ-NEXT: @%p1 bra $L__BB138_1;
+; SM90-FTZ-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-FTZ-NEXT: fence.acquire.cta;
+; SM90-FTZ-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-FTZ-NEXT: ret;
%retval = atomicrmw fadd ptr addrspace(3) %addr, float %val syncscope("block") acq_rel
ret float %retval
}
diff --git a/llvm/test/CodeGen/NVPTX/atomicrmw.py b/llvm/test/CodeGen/NVPTX/atomicrmw.py
index ef1ce116f7cd0..966d62bf1b03a 100644
--- a/llvm/test/CodeGen/NVPTX/atomicrmw.py
+++ b/llvm/test/CodeGen/NVPTX/atomicrmw.py
@@ -54,9 +54,16 @@
"""
)
+# atomicrmw fadd's lowering depends on the function's FTZ (denormal) mode, so we
+# check codegen both with and without it. Lines common to both runs collapse to
+# the SM${sm} prefix; only the FTZ-sensitive ops diverge into SM${sm}-NOFTZ /
+# SM${sm}-FTZ. (-nvptx-allow-ftz-atomics is covered separately in
+# atomicrmw-allow-ftz-atomics.ll.)
run_statement = Template(
- """; RUN: llc < %s -march=nvptx64 -mcpu=sm_${sm} -mattr=+ptx${ptx} | FileCheck %s --check-prefix=SM${sm}
+ """; RUN: llc < %s -march=nvptx64 -mcpu=sm_${sm} -mattr=+ptx${ptx} | FileCheck %s --check-prefixes=SM${sm},SM${sm}-NOFTZ
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_${sm} -mattr=+ptx${ptx} -denormal-fp-math-f32=preserve-sign | FileCheck %s --check-prefixes=SM${sm},SM${sm}-FTZ
; RUN: %if ptxas-sm_${sm} && ptxas-isa-${ptxfp} %{ llc < %s -march=nvptx64 -mcpu=sm_${sm} -mattr=+ptx${ptx} | %ptxas-verify -arch=sm_${sm} %}
+; RUN: %if ptxas-sm_${sm} && ptxas-isa-${ptxfp} %{ llc < %s -march=nvptx64 -mcpu=sm_${sm} -mattr=+ptx${ptx} -denormal-fp-math-f32=preserve-sign | %ptxas-verify -arch=sm_${sm} %}
"""
)
diff --git a/llvm/test/CodeGen/NVPTX/atomics.ll b/llvm/test/CodeGen/NVPTX/atomics.ll
index 3820b13f7da1f..ab05bf1b7ea3f 100644
--- a/llvm/test/CodeGen/NVPTX/atomics.ll
+++ b/llvm/test/CodeGen/NVPTX/atomics.ll
@@ -8,14 +8,23 @@ declare float @llvm.nvvm.atomic.load.add.f32.p0(ptr %addr, float %val)
define float @atomic_add_f32_generic(ptr %addr, float %val) {
; CHECK-LABEL: atomic_add_f32_generic(
; CHECK: {
-; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .pred %p<2>;
+; CHECK-NEXT: .reg .b32 %r<5>;
; CHECK-NEXT: .reg .b64 %rd<2>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r2, [atomic_add_f32_generic_param_1];
; CHECK-NEXT: ld.param.b64 %rd1, [atomic_add_f32_generic_param_0];
-; CHECK-NEXT: ld.param.b32 %r1, [atomic_add_f32_generic_param_1];
-; CHECK-NEXT: atom.add.f32 %r2, [%rd1], %r1;
-; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ld.volatile.b32 %r4, [%rd1];
+; CHECK-NEXT: $L__BB0_1: // %atomicrmw.start
+; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: add.rn.f32 %r3, %r4, %r2;
+; CHECK-NEXT: atom.cas.b32 %r1, [%rd1], %r4, %r3;
+; CHECK-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; CHECK-NEXT: mov.b32 %r4, %r1;
+; CHECK-NEXT: @%p1 bra $L__BB0_1;
+; CHECK-NEXT: // %bb.2: // %atomicrmw.end
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
; CHECK-NEXT: ret;
%ret = call float @llvm.nvvm.atomic.load.add.f32.p0(ptr %addr, float %val)
ret float %ret
@@ -27,14 +36,23 @@ declare float @llvm.nvvm.atomic.load.add.f32.p1(ptr addrspace(1) %addr, float %v
define float @atomic_add_f32_addrspace1(ptr addrspace(1) %addr, float %val) {
; CHECK-LABEL: atomic_add_f32_addrspace1(
; CHECK: {
-; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .pred %p<2>;
+; CHECK-NEXT: .reg .b32 %r<5>;
; CHECK-NEXT: .reg .b64 %rd<2>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r2, [atomic_add_f32_addrspace1_param_1];
; CHECK-NEXT: ld.param.b64 %rd1, [atomic_add_f32_addrspace1_param_0];
-; CHECK-NEXT: ld.param.b32 %r1, [atomic_add_f32_addrspace1_param_1];
-; CHECK-NEXT: atom.global.add.f32 %r2, [%rd1], %r1;
-; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ld.volatile.global.b32 %r4, [%rd1];
+; CHECK-NEXT: $L__BB1_1: // %atomicrmw.start
+; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: add.rn.f32 %r3, %r4, %r2;
+; CHECK-NEXT: atom.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; CHECK-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; CHECK-NEXT: mov.b32 %r4, %r1;
+; CHECK-NEXT: @%p1 bra $L__BB1_1;
+; CHECK-NEXT: // %bb.2: // %atomicrmw.end
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
; CHECK-NEXT: ret;
%ret = call float @llvm.nvvm.atomic.load.add.f32.p1(ptr addrspace(1) %addr, float %val)
ret float %ret
More information about the llvm-commits
mailing list