[llvm] [NVPTX] Add atom.add.noftz support, avoid CAS loops when noftz is available (PR #223569)
Akshay Deodhar via llvm-commits
llvm-commits at lists.llvm.org
Mon Sep 28 14:47:56 PDT 2026
https://github.com/akshayrdeodhar updated https://github.com/llvm/llvm-project/pull/223569
>From ec3c86ed9edbe581adfbe2a585848ede8e719652 Mon Sep 17 00:00:00 2001
From: Akshay Deodhar <adeodhar at nvidia.com>
Date: Thu, 30 Jul 2026 01:36:37 +0000
Subject: [PATCH 1/3] [NVPTX] Improve atomic fadd codegen using f32
atom.add.noftz support
---
llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp | 5 +
llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp | 75 +-
llvm/lib/Target/NVPTX/NVPTXISelLowering.h | 4 +
llvm/lib/Target/NVPTX/NVPTXInstrInfo.td | 2 +
llvm/lib/Target/NVPTX/NVPTXIntrinsics.td | 14 +-
llvm/lib/Target/NVPTX/NVPTXSubtarget.h | 3 +
.../NVPTX/atomicrmw-allow-ftz-atomics.ll | 109 -
llvm/test/CodeGen/NVPTX/atomicrmw-ftz.ll | 2566 +++++++++++++++++
llvm/test/CodeGen/NVPTX/atomicrmw-sm60.ll | 16 +-
llvm/test/CodeGen/NVPTX/atomicrmw-sm70.ll | 16 +-
llvm/test/CodeGen/NVPTX/atomicrmw-sm90.ll | 133 +-
llvm/test/CodeGen/NVPTX/atomicrmw.py | 22 +-
llvm/test/CodeGen/NVPTX/atomics.ll | 8 +-
13 files changed, 2716 insertions(+), 257 deletions(-)
delete mode 100644 llvm/test/CodeGen/NVPTX/atomicrmw-allow-ftz-atomics.ll
create mode 100644 llvm/test/CodeGen/NVPTX/atomicrmw-ftz.ll
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp b/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp
index 186bfa185e135..4f438640092ff 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp
@@ -92,6 +92,7 @@ class NVPTXDAGToDAGISel : public SelectionDAGISel {
NVPTX::DivPrecisionLevel getDivF32Level(const SDNode *N) const;
bool usePrecSqrtF32(const SDNode *N) const;
bool useF32FTZ() const;
+ bool useLegacyF32AtomAdd() const;
bool allowFMA() const;
bool doRsqrtOpt() const;
bool doMADWideOpt() const;
@@ -221,6 +222,10 @@ bool NVPTXDAGToDAGISel::useF32FTZ() const {
return Subtarget->getTargetLowering()->useF32FTZ(*MF);
}
+bool NVPTXDAGToDAGISel::useLegacyF32AtomAdd() const {
+ return Subtarget->getTargetLowering()->useLegacyF32AtomAdd();
+}
+
bool NVPTXDAGToDAGISel::allowFMA() const {
const NVPTXTargetLowering *TL = Subtarget->getTargetLowering();
return TL->allowFMA(*MF, OptLevel);
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
index 35c96588f0f44..64b040f84e5f9 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
@@ -110,16 +110,29 @@ static cl::opt<bool> UsePrecSqrtF32(
cl::desc("NVPTX Specific: 0 use sqrt.approx, 1 use sqrt.rn."),
cl::init(true));
-// PTX atom.add.f32 has fixed FTZ behavior that may not match the function's
+// PTX atom.add.f32 might not support FTZ behavior that matches the function's
// (see shouldExpandAtomicRMWInIR), so we'd normally fall back to a CAS loop
-// when they disagree. This option (enabled by default) allows using atom.add
-// anyway, trading correct denormal handling for the speed of the native
-// instruction.
-static cl::opt<bool> AllowFTZAtomics(
- "nvptx-allow-ftz-atomics", cl::Hidden,
- cl::desc("NVPTX Specific: Lower atomicrmw fadd to atom.add even when its "
- "FTZ behavior does not match the function's denormal mode."),
- cl::init(true));
+// when they disagree. This option controls whether to use atom.add anyway,
+// trading correct denormal handling for the speed of the native instruction.
+enum class AtomicAddBehavior { Strict, LegacyF32Add, Fast };
+static cl::opt<AtomicAddBehavior> FTZAtomics(
+ "nvptx-ftz-atomics", cl::Hidden,
+ cl::desc("NVPTX Specific: How to lower atomicrmw fadd"),
+ cl::init(AtomicAddBehavior::Fast),
+ cl::values(
+ clEnumValN(AtomicAddBehavior::Strict, "strict",
+ "Conform to the function's denormal mode, using a CAS loop "
+ "where no atom.add matches it. PTX has no flushing add for "
+ "f64 or bf16, so those still use the native instruction."),
+ clEnumValN(AtomicAddBehavior::LegacyF32Add, "legacy-f32-add",
+ "Never emit atom.add.noftz.f32; keep the pre-PTX 9.4 "
+ "atom.add.f32 lowering for f32 even when it does not match "
+ "the function's denormal mode. Otherwise as 'fast'."),
+ clEnumValN(
+ AtomicAddBehavior::Fast, "fast",
+ "As 'strict', except when no atom.add instruction with FTZ "
+ "behavior matching the function's denormal mode is available, in "
+ "which case use the non-conforming one (default)")));
/// Whereas CUDA's implementation (see libdevice) uses ex2.approx for exp2(), it
/// does NOT use lg2.approx for log2, so this is disabled by default.
@@ -161,6 +174,10 @@ bool NVPTXTargetLowering::useF32FTZ(const MachineFunction &MF) const {
DenormalMode::PreserveSign;
}
+bool NVPTXTargetLowering::useLegacyF32AtomAdd() const {
+ return FTZAtomics == AtomicAddBehavior::LegacyF32Add;
+}
+
static bool IsPTXVectorType(MVT VT) {
switch (VT.SimpleTy) {
default:
@@ -7713,29 +7730,44 @@ NVPTXTargetLowering::shouldExpandAtomicRMWInIR(const AtomicRMWInst *AI) const {
// by the weird FTZ behavior PTX atom.add has:
// - atom.add.f32 on global memory flushes denormals
// - atom.add.f32 on shared memory does not flush denormals
+ // - atom.add.f32.noftz is supported for SM90+, PTX 9.4+
// - atom.add.f16 and atomic.add.bf16 never flush denormals
- //
- // We lower to atom.add only if the function's FTZ behavior matches that of
- // atom.add; otherwise, we lower to a CAS loop. But we always allow
+ // - atom.add.f64 never flushes denormals, and add.f64 never flushes denormals
+ // We lower to atom.add only if the function's FTZ behavior is natively supported. otherwise, we lower to a CAS loop. But we always allow
// atomic.add.bf16; even though it never flushes denormals, we never flush
// bf16 denormals when doing regular arithmetic, even when FTZ is enabled.
if (AI->isFloatingPointOperation() &&
AI->getOperation() == AtomicRMWInst::BinOp::FAdd) {
const Function *F = AI->getFunction();
+ const bool AllowFTZAtomics = FTZAtomics != AtomicAddBehavior::Strict;
+ // AllowFTZAtomics forces atom.add regardless of a denormal mode mismatch
- // AllowFTZAtomics forces atom.add regardless of the FTZ mismatch.
if (Ty->isFloatTy()) {
- const bool FTZ = F->getDenormalMode(APFloat::IEEEsingle()).Output ==
- DenormalMode::PreserveSign;
+ const auto Mode = F->getDenormalMode(APFloat::IEEEsingle());
+ const bool IEEE =
+ Mode.Input == DenormalMode::IEEE ||
+ Mode.Output == DenormalMode::IEEE; // must not flush denormals
+ const bool FTZ =
+ Mode.inputsAreZero(); // must have input denormals flushed
+ const bool NativeNoFTZ = STI.hasAtomAddNoFTZ();
bool UseNative = AllowFTZAtomics;
+ // Generic has no case below, so it keeps UseNative = AllowFTZAtomics:
+ // when AtomicAddBehavior is strict we always generate a CmpXchg loop,
+ // whose non-atomic fadd is correctly lowered to ftz/noftz based on the
+ // function's DenormalMode.
+ // TODO: Handle cases where input mode and output mode are mismatched,
+ // once the bug is fixed in nonatomic add.
switch (AI->getPointerAddressSpace()) {
case llvm::ADDRESS_SPACE_GLOBAL:
- UseNative |= FTZ;
+ UseNative |= (NativeNoFTZ || !IEEE);
break;
case llvm::ADDRESS_SPACE_SHARED:
case llvm::ADDRESS_SPACE_SHARED_CLUSTER:
UseNative |= !FTZ;
break;
+ case llvm::ADDRESS_SPACE_GENERIC:
+ UseNative |= (NativeNoFTZ && IEEE);
+ break;
}
if (UseNative)
return AtomicExpansionKind::None;
@@ -7743,17 +7775,20 @@ NVPTXTargetLowering::shouldExpandAtomicRMWInIR(const AtomicRMWInst *AI) const {
if (Ty->isHalfTy()) {
// atom.add.f16 never flushes denormals, so it only agrees with a
- // function that is not in FTZ mode for f16.
- const bool FTZ = F->getDenormalMode(APFloat::IEEEhalf()).Output ==
- DenormalMode::PreserveSign;
+ // function that is not in FTZ mode for f16. The f32 denormal mode is
+ // irrelevant here.
+ const bool FTZ = F->getDenormalMode(APFloat::IEEEhalf()).inputsAreZero();
if ((!FTZ || AllowFTZAtomics) && STI.hasFeature(NVPTX::SM70) &&
STI.hasFeature(NVPTX::PTX63))
return AtomicExpansionKind::None;
}
- if (Ty->isBFloatTy() && STI.hasFeature(NVPTX::SM90))
+ if (Ty->isBFloatTy() && STI.hasFeature(NVPTX::SM90) &&
+ STI.hasFeature(NVPTX::PTX78))
return AtomicExpansionKind::None;
+ // We cannot support FTZ mode for fp64, as neither atom.add.f64 nor add.f64
+ // has an ftz mode that flushes denormals
if (Ty->isDoubleTy() && STI.hasAtomAddF64())
return AtomicExpansionKind::None;
}
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.h b/llvm/lib/Target/NVPTX/NVPTXISelLowering.h
index 4942408f6449b..99f3d75aa9dc2 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.h
+++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.h
@@ -115,6 +115,10 @@ class NVPTXTargetLowering : public TargetLowering {
// to sign-preserving zero.
bool useF32FTZ(const MachineFunction &MF) const;
+ // Get whether we should use the default atom.add.f32 instruction even when
+ // atom.add.noftz.f32 is available, to preserve old numerical results
+ bool useLegacyF32AtomAdd() const;
+
SDValue getSqrtEstimate(SDValue Operand, SelectionDAG &DAG, int Enabled,
int &ExtraSteps, bool &UseOneConst,
bool Reciprocal) const override;
diff --git a/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td b/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td
index f54cd9a515896..0511d392bd4fb 100644
--- a/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td
+++ b/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td
@@ -148,6 +148,7 @@ class PredNot<Predicate p> : Predicate<"!(" # p.CondString # ")">;
class SubtargetPredicate : Predicate<"Subtarget->" # NAME # "()">;
+def hasAtomAddNoFTZ : SubtargetPredicate;
def hasAtomAddF64 : SubtargetPredicate;
def hasAtomScope : SubtargetPredicate;
def hasAtomBitwise64 : SubtargetPredicate;
@@ -160,6 +161,7 @@ def hasOptEnabled : Predicate<"TM.getOptLevel() != CodeGenOptLevel::None">;
def doF32FTZ : Predicate<"useF32FTZ()">;
def doNoF32FTZ : PredNot<doF32FTZ>;
+def noLegacyF32AtomAdd : Predicate<"!useLegacyF32AtomAdd()">;
def doRsqrtOpt : Predicate<"doRsqrtOpt()">;
def doMADWideOpt : Predicate<"doMADWideOpt()">;
diff --git a/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td b/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
index b622910978161..f8db000933920 100644
--- a/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
+++ b/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
@@ -3482,14 +3482,16 @@ multiclass F_ATOMIC_2<RegTyInfo t, SDPatternOperator op, string op_str,
(!cast<Instruction>(NAME # _r) ADDR:$addr, t.Ty:$b, (XForm.GetSem $this),
(XForm.GetScope $this), (XForm.GetAddSp $this),
(XForm.GetEvictionAndPrefetchHint $this),
- (XForm.GetCachePolicy $this))>;
+ (XForm.GetCachePolicy $this))>,
+ Requires<preds>;
if t.SupportsImm then
def : Pat<(op:$this addr:$addr, (t.Ty t.ImmNode:$b)),
(!cast<Instruction>(NAME # _i) ADDR:$addr, (t.Ty t.ImmNode:$b),
(XForm.GetSem $this), (XForm.GetScope $this), (XForm.GetAddSp $this),
(XForm.GetEvictionAndPrefetchHint $this),
- (XForm.GetCachePolicy $this))>;
+ (XForm.GetCachePolicy $this))>,
+ Requires<preds>;
}
multiclass F_ATOMIC_3<RegTyInfo t, string op_str, SDPatternOperator op, SDNode atomic> {
@@ -3539,7 +3541,13 @@ defm INT_PTX_ATOM_ADD_32 : F_ATOMIC_2<I32RT, atomic_load_add_i32, "add", "u32",
defm INT_PTX_ATOM_ADD_64 : F_ATOMIC_2<I64RT, atomic_load_add_i64, "add", "u64", atomic_load_add>;
defm INT_PTX_ATOM_ADD_F16 : F_ATOMIC_2<F16RT, atomic_load_fadd, "add.noftz", "f16", atomic_load_fadd, [SM70, PTX63]>;
-defm INT_PTX_ATOM_ADD_BF16 : F_ATOMIC_2<BF16RT, atomic_load_fadd, "add.noftz", "bf16", atomic_load_fadd, [SM90]>;
+defm INT_PTX_ATOM_ADD_BF16 : F_ATOMIC_2<BF16RT, atomic_load_fadd, "add.noftz", "bf16", atomic_load_fadd, [SM90, PTX78]>;
+
+// If the function denormal mode is IEEE, and noftz is supported, lower to NoFTZ
+defm INT_PTX_ATOM_ADD_NOFTZ_F32 : F_ATOMIC_2<F32RT, atomic_load_fadd, "add.noftz", "f32", atomic_load_fadd, [hasAtomAddNoFTZ, doNoF32FTZ, noLegacyF32AtomAdd]>;
+// Otherwise, lower to the default native instruction. Passes prior to ISel
+// must handle the case where there is a mismatch between default semantics and
+// the denormal mode
defm INT_PTX_ATOM_ADD_F32 : F_ATOMIC_2<F32RT, atomic_load_fadd, "add", "f32", atomic_load_fadd>;
defm INT_PTX_ATOM_ADD_F64 : F_ATOMIC_2<F64RT, atomic_load_fadd, "add", "f64", atomic_load_fadd, [hasAtomAddF64]>;
diff --git a/llvm/lib/Target/NVPTX/NVPTXSubtarget.h b/llvm/lib/Target/NVPTX/NVPTXSubtarget.h
index ee8adec2da060..4f56a8e0df7db 100644
--- a/llvm/lib/Target/NVPTX/NVPTXSubtarget.h
+++ b/llvm/lib/Target/NVPTX/NVPTXSubtarget.h
@@ -85,6 +85,9 @@ class NVPTXSubtarget : public NVPTXGenSubtargetInfo {
bool hasUsedBytesMaskPragma() const {
return hasFeature(NVPTX::SM50) && hasFeature(NVPTX::PTX83);
}
+ bool hasAtomAddNoFTZ() const {
+ return hasFeature(NVPTX::SM90) && hasFeature(NVPTX::PTX94);
+ }
bool hasAtomAddF64() const { return hasFeature(NVPTX::SM60); }
bool hasAtomScope() const { return hasFeature(NVPTX::SM60); }
bool hasAtomBitwise64() const { return hasFeature(NVPTX::SM32); }
diff --git a/llvm/test/CodeGen/NVPTX/atomicrmw-allow-ftz-atomics.ll b/llvm/test/CodeGen/NVPTX/atomicrmw-allow-ftz-atomics.ll
deleted file mode 100644
index 78c86bfb08e2f..0000000000000
--- a/llvm/test/CodeGen/NVPTX/atomicrmw-allow-ftz-atomics.ll
+++ /dev/null
@@ -1,109 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
-; -nvptx-allow-ftz-atomics forces atomicrmw fadd to lower to atom.add even when
-; the hardware's FTZ behavior doesn't match the function's denormal mode, where
-; we'd otherwise emit a CAS loop. Compare the lowering with the option disabled
-; (DISALLOW) against the option enabled or default (ALLOW).
-; RUN: llc < %s -march=nvptx64 -mcpu=sm_90 -mattr=+ptx87 -nvptx-allow-ftz-atomics=false | FileCheck %s --check-prefixes=CHECK,DISALLOW
-; RUN: llc < %s -march=nvptx64 -mcpu=sm_90 -mattr=+ptx87 -nvptx-allow-ftz-atomics=true | FileCheck %s --check-prefixes=CHECK,ALLOW
-; RUN: llc < %s -march=nvptx64 -mcpu=sm_90 -mattr=+ptx87 | FileCheck %s --check-prefixes=CHECK,ALLOW
-; RUN: %if ptxas-sm_90 && ptxas-isa-8.7 %{ llc < %s -march=nvptx64 -mcpu=sm_90 -mattr=+ptx87 -nvptx-allow-ftz-atomics=false | %ptxas-verify -arch=sm_90 %}
-; RUN: %if ptxas-sm_90 && ptxas-isa-8.7 %{ llc < %s -march=nvptx64 -mcpu=sm_90 -mattr=+ptx87 -nvptx-allow-ftz-atomics=true | %ptxas-verify -arch=sm_90 %}
-; RUN: %if ptxas-sm_90 && ptxas-isa-8.7 %{ llc < %s -march=nvptx64 -mcpu=sm_90 -mattr=+ptx87 | %ptxas-verify -arch=sm_90 %}
-
-; global f32: atom.add flushes, so we'd normally use a CAS loop (function is not FTZ)
-; when the option is disabled; when enabled (default), it lowers to atom.add.
-define float @fadd_f32_global(ptr addrspace(1) %addr, float %val) {
-;
-; DISALLOW-LABEL: fadd_f32_global(
-; DISALLOW: {
-; DISALLOW-NEXT: .reg .pred %p<2>;
-; DISALLOW-NEXT: .reg .b32 %r<5>;
-; DISALLOW-NEXT: .reg .b64 %rd<2>;
-; DISALLOW-EMPTY:
-; DISALLOW-NEXT: // %bb.0:
-; DISALLOW-NEXT: ld.param::func.b32 %r2, [fadd_f32_global_param_1];
-; DISALLOW-NEXT: ld.param::func.b64 %rd1, [fadd_f32_global_param_0];
-; DISALLOW-NEXT: ld.relaxed.sys.global.b32 %r4, [%rd1];
-; DISALLOW-NEXT: $L__BB0_1: // %atomicrmw.start
-; DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; DISALLOW-NEXT: add.rn.f32 %r3, %r4, %r2;
-; DISALLOW-NEXT: atom.relaxed.sys.global.cas.b32 %r1, [%rd1], %r4, %r3;
-; DISALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
-; DISALLOW-NEXT: mov.b32 %r4, %r1;
-; DISALLOW-NEXT: @%p1 bra $L__BB0_1;
-; DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
-; DISALLOW-NEXT: st.param::func.b32 [func_retval0], %r1;
-; DISALLOW-NEXT: ret;
-;
-; ALLOW-LABEL: fadd_f32_global(
-; ALLOW: {
-; ALLOW-NEXT: .reg .b32 %r<3>;
-; ALLOW-NEXT: .reg .b64 %rd<2>;
-; ALLOW-EMPTY:
-; ALLOW-NEXT: // %bb.0:
-; ALLOW-NEXT: ld.param::func.b64 %rd1, [fadd_f32_global_param_0];
-; ALLOW-NEXT: ld.param::func.b32 %r1, [fadd_f32_global_param_1];
-; ALLOW-NEXT: atom.relaxed.sys.global.add.f32 %r2, [%rd1], %r1;
-; ALLOW-NEXT: st.param::func.b32 [func_retval0], %r2;
-; ALLOW-NEXT: ret;
- %r = atomicrmw fadd ptr addrspace(1) %addr, float %val monotonic
- ret float %r
-}
-
-; generic f32: we can't know the memory space, so we'd normally use a CAS loop
-; when the option is disabled; when enabled (default), it lowers to atom.add.
-define float @fadd_f32_generic(ptr %addr, float %val) {
-;
-; DISALLOW-LABEL: fadd_f32_generic(
-; DISALLOW: {
-; DISALLOW-NEXT: .reg .pred %p<2>;
-; DISALLOW-NEXT: .reg .b32 %r<5>;
-; DISALLOW-NEXT: .reg .b64 %rd<2>;
-; DISALLOW-EMPTY:
-; DISALLOW-NEXT: // %bb.0:
-; DISALLOW-NEXT: ld.param::func.b32 %r2, [fadd_f32_generic_param_1];
-; DISALLOW-NEXT: ld.param::func.b64 %rd1, [fadd_f32_generic_param_0];
-; DISALLOW-NEXT: ld.relaxed.sys.b32 %r4, [%rd1];
-; DISALLOW-NEXT: $L__BB1_1: // %atomicrmw.start
-; DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; DISALLOW-NEXT: add.rn.f32 %r3, %r4, %r2;
-; DISALLOW-NEXT: atom.relaxed.sys.cas.b32 %r1, [%rd1], %r4, %r3;
-; DISALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
-; DISALLOW-NEXT: mov.b32 %r4, %r1;
-; DISALLOW-NEXT: @%p1 bra $L__BB1_1;
-; DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
-; DISALLOW-NEXT: st.param::func.b32 [func_retval0], %r1;
-; DISALLOW-NEXT: ret;
-;
-; ALLOW-LABEL: fadd_f32_generic(
-; ALLOW: {
-; ALLOW-NEXT: .reg .b32 %r<3>;
-; ALLOW-NEXT: .reg .b64 %rd<2>;
-; ALLOW-EMPTY:
-; ALLOW-NEXT: // %bb.0:
-; ALLOW-NEXT: ld.param::func.b64 %rd1, [fadd_f32_generic_param_0];
-; ALLOW-NEXT: ld.param::func.b32 %r1, [fadd_f32_generic_param_1];
-; ALLOW-NEXT: atom.relaxed.sys.add.f32 %r2, [%rd1], %r1;
-; ALLOW-NEXT: st.param::func.b32 [func_retval0], %r2;
-; ALLOW-NEXT: ret;
- %r = atomicrmw fadd ptr %addr, float %val monotonic
- ret float %r
-}
-
-; shared f32: atom.add does not flush, matching the non-FTZ function, so this is
-; atom.add either way.
-define float @fadd_f32_shared(ptr addrspace(3) %addr, float %val) {
-; CHECK-LABEL: fadd_f32_shared(
-; CHECK: {
-; CHECK-NEXT: .reg .b32 %r<3>;
-; CHECK-NEXT: .reg .b64 %rd<2>;
-; CHECK-EMPTY:
-; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: ld.param::func.b64 %rd1, [fadd_f32_shared_param_0];
-; CHECK-NEXT: ld.param::func.b32 %r1, [fadd_f32_shared_param_1];
-; CHECK-NEXT: atom.relaxed.sys.shared.add.f32 %r2, [%rd1], %r1;
-; CHECK-NEXT: st.param::func.b32 [func_retval0], %r2;
-; CHECK-NEXT: ret;
- %r = atomicrmw fadd ptr addrspace(3) %addr, float %val monotonic
- ret float %r
-}
diff --git a/llvm/test/CodeGen/NVPTX/atomicrmw-ftz.ll b/llvm/test/CodeGen/NVPTX/atomicrmw-ftz.ll
new file mode 100644
index 0000000000000..7ba318e8e4c12
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/atomicrmw-ftz.ll
@@ -0,0 +1,2566 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; -nvptx-ftz-atomics selects how atomicrmw fadd is lowered when the FTZ behavior
+; of a native atom.add does not match the function's denormal mode:
+; strict - always conform to the denormal mode (CAS loop if needed)
+; legacy-f32-add - keep the pre-PTX 9.4 flushing atom.add.f32 for f32
+; fast - conform when a conforming atom.add exists, else use
+; the non-conforming native instruction
+;
+; atom.add.noftz.f32 requires sm_90 and PTX 9.4, so the grid below fixes
+; -mattr=+ptx94 and varies the SM version to toggle its availability:
+; SM70 - noftz unavailable SM90 - noftz available
+; crossed with the three -nvptx-ftz-atomics modes and both denormal modes
+; (IEEE = default, PRESERVESIGN = -denormal-fp-math=preserve-sign). The
+; global / generic / shared address spaces are covered by the functions below,
+; since atom.add.f32 flushes on global but not on shared.
+
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_70 -mattr=+ptx94 -nvptx-ftz-atomics=strict | FileCheck %s --check-prefixes=SM70-IEEE-STRICT
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_70 -mattr=+ptx94 -nvptx-ftz-atomics=legacy-f32-add | FileCheck %s --check-prefixes=SM70-IEEE-LEGACY
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_70 -mattr=+ptx94 -nvptx-ftz-atomics=fast | FileCheck %s --check-prefixes=SM70-IEEE-FAST
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_70 -mattr=+ptx94 -denormal-fp-math=preserve-sign -nvptx-ftz-atomics=strict | FileCheck %s --check-prefixes=SM70-PRESERVESIGN-STRICT
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_70 -mattr=+ptx94 -denormal-fp-math=preserve-sign -nvptx-ftz-atomics=legacy-f32-add | FileCheck %s --check-prefixes=SM70-PRESERVESIGN-LEGACY
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_70 -mattr=+ptx94 -denormal-fp-math=preserve-sign -nvptx-ftz-atomics=fast | FileCheck %s --check-prefixes=SM70-PRESERVESIGN-FAST
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_90 -mattr=+ptx94 -nvptx-ftz-atomics=strict | FileCheck %s --check-prefixes=SM90-IEEE-STRICT
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_90 -mattr=+ptx94 -nvptx-ftz-atomics=legacy-f32-add | FileCheck %s --check-prefixes=SM90-IEEE-LEGACY
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_90 -mattr=+ptx94 -nvptx-ftz-atomics=fast | FileCheck %s --check-prefixes=SM90-IEEE-FAST
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_90 -mattr=+ptx94 -denormal-fp-math=preserve-sign -nvptx-ftz-atomics=strict | FileCheck %s --check-prefixes=SM90-PRESERVESIGN-STRICT
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_90 -mattr=+ptx94 -denormal-fp-math=preserve-sign -nvptx-ftz-atomics=legacy-f32-add | FileCheck %s --check-prefixes=SM90-PRESERVESIGN-LEGACY
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_90 -mattr=+ptx94 -denormal-fp-math=preserve-sign -nvptx-ftz-atomics=fast | FileCheck %s --check-prefixes=SM90-PRESERVESIGN-FAST
+
+; RUN: %if ptxas-sm_70 && ptxas-isa-9.4 %{ llc < %s -march=nvptx64 -mcpu=sm_70 -mattr=+ptx94 -nvptx-ftz-atomics=strict | %ptxas-verify -arch=sm_70 %}
+; RUN: %if ptxas-sm_70 && ptxas-isa-9.4 %{ llc < %s -march=nvptx64 -mcpu=sm_70 -mattr=+ptx94 -nvptx-ftz-atomics=legacy-f32-add | %ptxas-verify -arch=sm_70 %}
+; RUN: %if ptxas-sm_70 && ptxas-isa-9.4 %{ llc < %s -march=nvptx64 -mcpu=sm_70 -mattr=+ptx94 -nvptx-ftz-atomics=fast | %ptxas-verify -arch=sm_70 %}
+; RUN: %if ptxas-sm_70 && ptxas-isa-9.4 %{ llc < %s -march=nvptx64 -mcpu=sm_70 -mattr=+ptx94 -denormal-fp-math=preserve-sign -nvptx-ftz-atomics=strict | %ptxas-verify -arch=sm_70 %}
+; RUN: %if ptxas-sm_70 && ptxas-isa-9.4 %{ llc < %s -march=nvptx64 -mcpu=sm_70 -mattr=+ptx94 -denormal-fp-math=preserve-sign -nvptx-ftz-atomics=legacy-f32-add | %ptxas-verify -arch=sm_70 %}
+; RUN: %if ptxas-sm_70 && ptxas-isa-9.4 %{ llc < %s -march=nvptx64 -mcpu=sm_70 -mattr=+ptx94 -denormal-fp-math=preserve-sign -nvptx-ftz-atomics=fast | %ptxas-verify -arch=sm_70 %}
+; RUN: %if ptxas-sm_90 && ptxas-isa-9.4 %{ llc < %s -march=nvptx64 -mcpu=sm_90 -mattr=+ptx94 -nvptx-ftz-atomics=strict | %ptxas-verify -arch=sm_90 %}
+; RUN: %if ptxas-sm_90 && ptxas-isa-9.4 %{ llc < %s -march=nvptx64 -mcpu=sm_90 -mattr=+ptx94 -nvptx-ftz-atomics=legacy-f32-add | %ptxas-verify -arch=sm_90 %}
+; RUN: %if ptxas-sm_90 && ptxas-isa-9.4 %{ llc < %s -march=nvptx64 -mcpu=sm_90 -mattr=+ptx94 -nvptx-ftz-atomics=fast | %ptxas-verify -arch=sm_90 %}
+; RUN: %if ptxas-sm_90 && ptxas-isa-9.4 %{ llc < %s -march=nvptx64 -mcpu=sm_90 -mattr=+ptx94 -denormal-fp-math=preserve-sign -nvptx-ftz-atomics=strict | %ptxas-verify -arch=sm_90 %}
+; RUN: %if ptxas-sm_90 && ptxas-isa-9.4 %{ llc < %s -march=nvptx64 -mcpu=sm_90 -mattr=+ptx94 -denormal-fp-math=preserve-sign -nvptx-ftz-atomics=legacy-f32-add | %ptxas-verify -arch=sm_90 %}
+; RUN: %if ptxas-sm_90 && ptxas-isa-9.4 %{ llc < %s -march=nvptx64 -mcpu=sm_90 -mattr=+ptx94 -denormal-fp-math=preserve-sign -nvptx-ftz-atomics=fast | %ptxas-verify -arch=sm_90 %}
+
+; global f32: atom.add.f32 flushes denormals here, so it conforms only when the
+; function is in preserve-sign mode.
+define float @fadd_f32_global(ptr addrspace(1) %addr, float %val) {
+; SM70-IEEE-STRICT-LABEL: fadd_f32_global(
+; SM70-IEEE-STRICT: {
+; SM70-IEEE-STRICT-NEXT: .reg .pred %p<2>;
+; SM70-IEEE-STRICT-NEXT: .reg .b32 %r<5>;
+; SM70-IEEE-STRICT-NEXT: .reg .b64 %rd<2>;
+; SM70-IEEE-STRICT-EMPTY:
+; SM70-IEEE-STRICT-NEXT: // %bb.0:
+; SM70-IEEE-STRICT-NEXT: ld.param::func.b32 %r2, [fadd_f32_global_param_1];
+; SM70-IEEE-STRICT-NEXT: ld.param::func.b64 %rd1, [fadd_f32_global_param_0];
+; SM70-IEEE-STRICT-NEXT: ld.relaxed.sys.global.b32 %r4, [%rd1];
+; SM70-IEEE-STRICT-NEXT: $L__BB0_1: // %atomicrmw.start
+; SM70-IEEE-STRICT-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-IEEE-STRICT-NEXT: add.rn.f32 %r3, %r4, %r2;
+; SM70-IEEE-STRICT-NEXT: atom.relaxed.sys.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM70-IEEE-STRICT-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM70-IEEE-STRICT-NEXT: mov.b32 %r4, %r1;
+; SM70-IEEE-STRICT-NEXT: @%p1 bra $L__BB0_1;
+; SM70-IEEE-STRICT-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-IEEE-STRICT-NEXT: st.param::func.b32 [func_retval0], %r1;
+; SM70-IEEE-STRICT-NEXT: ret;
+;
+; SM70-IEEE-LEGACY-LABEL: fadd_f32_global(
+; SM70-IEEE-LEGACY: {
+; SM70-IEEE-LEGACY-NEXT: .reg .b32 %r<3>;
+; SM70-IEEE-LEGACY-NEXT: .reg .b64 %rd<2>;
+; SM70-IEEE-LEGACY-EMPTY:
+; SM70-IEEE-LEGACY-NEXT: // %bb.0:
+; SM70-IEEE-LEGACY-NEXT: ld.param::func.b64 %rd1, [fadd_f32_global_param_0];
+; SM70-IEEE-LEGACY-NEXT: ld.param::func.b32 %r1, [fadd_f32_global_param_1];
+; SM70-IEEE-LEGACY-NEXT: atom.relaxed.sys.global.add.f32 %r2, [%rd1], %r1;
+; SM70-IEEE-LEGACY-NEXT: st.param::func.b32 [func_retval0], %r2;
+; SM70-IEEE-LEGACY-NEXT: ret;
+;
+; SM70-IEEE-FAST-LABEL: fadd_f32_global(
+; SM70-IEEE-FAST: {
+; SM70-IEEE-FAST-NEXT: .reg .b32 %r<3>;
+; SM70-IEEE-FAST-NEXT: .reg .b64 %rd<2>;
+; SM70-IEEE-FAST-EMPTY:
+; SM70-IEEE-FAST-NEXT: // %bb.0:
+; SM70-IEEE-FAST-NEXT: ld.param::func.b64 %rd1, [fadd_f32_global_param_0];
+; SM70-IEEE-FAST-NEXT: ld.param::func.b32 %r1, [fadd_f32_global_param_1];
+; SM70-IEEE-FAST-NEXT: atom.relaxed.sys.global.add.f32 %r2, [%rd1], %r1;
+; SM70-IEEE-FAST-NEXT: st.param::func.b32 [func_retval0], %r2;
+; SM70-IEEE-FAST-NEXT: ret;
+;
+; SM70-PRESERVESIGN-STRICT-LABEL: fadd_f32_global(
+; SM70-PRESERVESIGN-STRICT: {
+; SM70-PRESERVESIGN-STRICT-NEXT: .reg .b32 %r<3>;
+; SM70-PRESERVESIGN-STRICT-NEXT: .reg .b64 %rd<2>;
+; SM70-PRESERVESIGN-STRICT-EMPTY:
+; SM70-PRESERVESIGN-STRICT-NEXT: // %bb.0:
+; SM70-PRESERVESIGN-STRICT-NEXT: ld.param::func.b64 %rd1, [fadd_f32_global_param_0];
+; SM70-PRESERVESIGN-STRICT-NEXT: ld.param::func.b32 %r1, [fadd_f32_global_param_1];
+; SM70-PRESERVESIGN-STRICT-NEXT: atom.relaxed.sys.global.add.f32 %r2, [%rd1], %r1;
+; SM70-PRESERVESIGN-STRICT-NEXT: st.param::func.b32 [func_retval0], %r2;
+; SM70-PRESERVESIGN-STRICT-NEXT: ret;
+;
+; SM70-PRESERVESIGN-LEGACY-LABEL: fadd_f32_global(
+; SM70-PRESERVESIGN-LEGACY: {
+; SM70-PRESERVESIGN-LEGACY-NEXT: .reg .b32 %r<3>;
+; SM70-PRESERVESIGN-LEGACY-NEXT: .reg .b64 %rd<2>;
+; SM70-PRESERVESIGN-LEGACY-EMPTY:
+; SM70-PRESERVESIGN-LEGACY-NEXT: // %bb.0:
+; SM70-PRESERVESIGN-LEGACY-NEXT: ld.param::func.b64 %rd1, [fadd_f32_global_param_0];
+; SM70-PRESERVESIGN-LEGACY-NEXT: ld.param::func.b32 %r1, [fadd_f32_global_param_1];
+; SM70-PRESERVESIGN-LEGACY-NEXT: atom.relaxed.sys.global.add.f32 %r2, [%rd1], %r1;
+; SM70-PRESERVESIGN-LEGACY-NEXT: st.param::func.b32 [func_retval0], %r2;
+; SM70-PRESERVESIGN-LEGACY-NEXT: ret;
+;
+; SM70-PRESERVESIGN-FAST-LABEL: fadd_f32_global(
+; SM70-PRESERVESIGN-FAST: {
+; SM70-PRESERVESIGN-FAST-NEXT: .reg .b32 %r<3>;
+; SM70-PRESERVESIGN-FAST-NEXT: .reg .b64 %rd<2>;
+; SM70-PRESERVESIGN-FAST-EMPTY:
+; SM70-PRESERVESIGN-FAST-NEXT: // %bb.0:
+; SM70-PRESERVESIGN-FAST-NEXT: ld.param::func.b64 %rd1, [fadd_f32_global_param_0];
+; SM70-PRESERVESIGN-FAST-NEXT: ld.param::func.b32 %r1, [fadd_f32_global_param_1];
+; SM70-PRESERVESIGN-FAST-NEXT: atom.relaxed.sys.global.add.f32 %r2, [%rd1], %r1;
+; SM70-PRESERVESIGN-FAST-NEXT: st.param::func.b32 [func_retval0], %r2;
+; SM70-PRESERVESIGN-FAST-NEXT: ret;
+;
+; SM90-IEEE-STRICT-LABEL: fadd_f32_global(
+; SM90-IEEE-STRICT: {
+; SM90-IEEE-STRICT-NEXT: .reg .b32 %r<3>;
+; SM90-IEEE-STRICT-NEXT: .reg .b64 %rd<2>;
+; SM90-IEEE-STRICT-EMPTY:
+; SM90-IEEE-STRICT-NEXT: // %bb.0:
+; SM90-IEEE-STRICT-NEXT: ld.param::func.b64 %rd1, [fadd_f32_global_param_0];
+; SM90-IEEE-STRICT-NEXT: ld.param::func.b32 %r1, [fadd_f32_global_param_1];
+; SM90-IEEE-STRICT-NEXT: atom.relaxed.sys.global.add.noftz.f32 %r2, [%rd1], %r1;
+; SM90-IEEE-STRICT-NEXT: st.param::func.b32 [func_retval0], %r2;
+; SM90-IEEE-STRICT-NEXT: ret;
+;
+; SM90-IEEE-LEGACY-LABEL: fadd_f32_global(
+; SM90-IEEE-LEGACY: {
+; SM90-IEEE-LEGACY-NEXT: .reg .b32 %r<3>;
+; SM90-IEEE-LEGACY-NEXT: .reg .b64 %rd<2>;
+; SM90-IEEE-LEGACY-EMPTY:
+; SM90-IEEE-LEGACY-NEXT: // %bb.0:
+; SM90-IEEE-LEGACY-NEXT: ld.param::func.b64 %rd1, [fadd_f32_global_param_0];
+; SM90-IEEE-LEGACY-NEXT: ld.param::func.b32 %r1, [fadd_f32_global_param_1];
+; SM90-IEEE-LEGACY-NEXT: atom.relaxed.sys.global.add.f32 %r2, [%rd1], %r1;
+; SM90-IEEE-LEGACY-NEXT: st.param::func.b32 [func_retval0], %r2;
+; SM90-IEEE-LEGACY-NEXT: ret;
+;
+; SM90-IEEE-FAST-LABEL: fadd_f32_global(
+; SM90-IEEE-FAST: {
+; SM90-IEEE-FAST-NEXT: .reg .b32 %r<3>;
+; SM90-IEEE-FAST-NEXT: .reg .b64 %rd<2>;
+; SM90-IEEE-FAST-EMPTY:
+; SM90-IEEE-FAST-NEXT: // %bb.0:
+; SM90-IEEE-FAST-NEXT: ld.param::func.b64 %rd1, [fadd_f32_global_param_0];
+; SM90-IEEE-FAST-NEXT: ld.param::func.b32 %r1, [fadd_f32_global_param_1];
+; SM90-IEEE-FAST-NEXT: atom.relaxed.sys.global.add.noftz.f32 %r2, [%rd1], %r1;
+; SM90-IEEE-FAST-NEXT: st.param::func.b32 [func_retval0], %r2;
+; SM90-IEEE-FAST-NEXT: ret;
+;
+; SM90-PRESERVESIGN-STRICT-LABEL: fadd_f32_global(
+; SM90-PRESERVESIGN-STRICT: {
+; SM90-PRESERVESIGN-STRICT-NEXT: .reg .b32 %r<3>;
+; SM90-PRESERVESIGN-STRICT-NEXT: .reg .b64 %rd<2>;
+; SM90-PRESERVESIGN-STRICT-EMPTY:
+; SM90-PRESERVESIGN-STRICT-NEXT: // %bb.0:
+; SM90-PRESERVESIGN-STRICT-NEXT: ld.param::func.b64 %rd1, [fadd_f32_global_param_0];
+; SM90-PRESERVESIGN-STRICT-NEXT: ld.param::func.b32 %r1, [fadd_f32_global_param_1];
+; SM90-PRESERVESIGN-STRICT-NEXT: atom.relaxed.sys.global.add.f32 %r2, [%rd1], %r1;
+; SM90-PRESERVESIGN-STRICT-NEXT: st.param::func.b32 [func_retval0], %r2;
+; SM90-PRESERVESIGN-STRICT-NEXT: ret;
+;
+; SM90-PRESERVESIGN-LEGACY-LABEL: fadd_f32_global(
+; SM90-PRESERVESIGN-LEGACY: {
+; SM90-PRESERVESIGN-LEGACY-NEXT: .reg .b32 %r<3>;
+; SM90-PRESERVESIGN-LEGACY-NEXT: .reg .b64 %rd<2>;
+; SM90-PRESERVESIGN-LEGACY-EMPTY:
+; SM90-PRESERVESIGN-LEGACY-NEXT: // %bb.0:
+; SM90-PRESERVESIGN-LEGACY-NEXT: ld.param::func.b64 %rd1, [fadd_f32_global_param_0];
+; SM90-PRESERVESIGN-LEGACY-NEXT: ld.param::func.b32 %r1, [fadd_f32_global_param_1];
+; SM90-PRESERVESIGN-LEGACY-NEXT: atom.relaxed.sys.global.add.f32 %r2, [%rd1], %r1;
+; SM90-PRESERVESIGN-LEGACY-NEXT: st.param::func.b32 [func_retval0], %r2;
+; SM90-PRESERVESIGN-LEGACY-NEXT: ret;
+;
+; SM90-PRESERVESIGN-FAST-LABEL: fadd_f32_global(
+; SM90-PRESERVESIGN-FAST: {
+; SM90-PRESERVESIGN-FAST-NEXT: .reg .b32 %r<3>;
+; SM90-PRESERVESIGN-FAST-NEXT: .reg .b64 %rd<2>;
+; SM90-PRESERVESIGN-FAST-EMPTY:
+; SM90-PRESERVESIGN-FAST-NEXT: // %bb.0:
+; SM90-PRESERVESIGN-FAST-NEXT: ld.param::func.b64 %rd1, [fadd_f32_global_param_0];
+; SM90-PRESERVESIGN-FAST-NEXT: ld.param::func.b32 %r1, [fadd_f32_global_param_1];
+; SM90-PRESERVESIGN-FAST-NEXT: atom.relaxed.sys.global.add.f32 %r2, [%rd1], %r1;
+; SM90-PRESERVESIGN-FAST-NEXT: st.param::func.b32 [func_retval0], %r2;
+; SM90-PRESERVESIGN-FAST-NEXT: ret;
+ %r = atomicrmw fadd ptr addrspace(1) %addr, float %val monotonic
+ ret float %r
+}
+
+; generic f32: the memory space is unknown, so atom.add.f32's flushing behavior
+; is unknown too.
+define float @fadd_f32_generic(ptr %addr, float %val) {
+; SM70-IEEE-STRICT-LABEL: fadd_f32_generic(
+; SM70-IEEE-STRICT: {
+; SM70-IEEE-STRICT-NEXT: .reg .pred %p<2>;
+; SM70-IEEE-STRICT-NEXT: .reg .b32 %r<5>;
+; SM70-IEEE-STRICT-NEXT: .reg .b64 %rd<2>;
+; SM70-IEEE-STRICT-EMPTY:
+; SM70-IEEE-STRICT-NEXT: // %bb.0:
+; SM70-IEEE-STRICT-NEXT: ld.param::func.b32 %r2, [fadd_f32_generic_param_1];
+; SM70-IEEE-STRICT-NEXT: ld.param::func.b64 %rd1, [fadd_f32_generic_param_0];
+; SM70-IEEE-STRICT-NEXT: ld.relaxed.sys.b32 %r4, [%rd1];
+; SM70-IEEE-STRICT-NEXT: $L__BB1_1: // %atomicrmw.start
+; SM70-IEEE-STRICT-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-IEEE-STRICT-NEXT: add.rn.f32 %r3, %r4, %r2;
+; SM70-IEEE-STRICT-NEXT: atom.relaxed.sys.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM70-IEEE-STRICT-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM70-IEEE-STRICT-NEXT: mov.b32 %r4, %r1;
+; SM70-IEEE-STRICT-NEXT: @%p1 bra $L__BB1_1;
+; SM70-IEEE-STRICT-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-IEEE-STRICT-NEXT: st.param::func.b32 [func_retval0], %r1;
+; SM70-IEEE-STRICT-NEXT: ret;
+;
+; SM70-IEEE-LEGACY-LABEL: fadd_f32_generic(
+; SM70-IEEE-LEGACY: {
+; SM70-IEEE-LEGACY-NEXT: .reg .b32 %r<3>;
+; SM70-IEEE-LEGACY-NEXT: .reg .b64 %rd<2>;
+; SM70-IEEE-LEGACY-EMPTY:
+; SM70-IEEE-LEGACY-NEXT: // %bb.0:
+; SM70-IEEE-LEGACY-NEXT: ld.param::func.b64 %rd1, [fadd_f32_generic_param_0];
+; SM70-IEEE-LEGACY-NEXT: ld.param::func.b32 %r1, [fadd_f32_generic_param_1];
+; SM70-IEEE-LEGACY-NEXT: atom.relaxed.sys.add.f32 %r2, [%rd1], %r1;
+; SM70-IEEE-LEGACY-NEXT: st.param::func.b32 [func_retval0], %r2;
+; SM70-IEEE-LEGACY-NEXT: ret;
+;
+; SM70-IEEE-FAST-LABEL: fadd_f32_generic(
+; SM70-IEEE-FAST: {
+; SM70-IEEE-FAST-NEXT: .reg .b32 %r<3>;
+; SM70-IEEE-FAST-NEXT: .reg .b64 %rd<2>;
+; SM70-IEEE-FAST-EMPTY:
+; SM70-IEEE-FAST-NEXT: // %bb.0:
+; SM70-IEEE-FAST-NEXT: ld.param::func.b64 %rd1, [fadd_f32_generic_param_0];
+; SM70-IEEE-FAST-NEXT: ld.param::func.b32 %r1, [fadd_f32_generic_param_1];
+; SM70-IEEE-FAST-NEXT: atom.relaxed.sys.add.f32 %r2, [%rd1], %r1;
+; SM70-IEEE-FAST-NEXT: st.param::func.b32 [func_retval0], %r2;
+; SM70-IEEE-FAST-NEXT: ret;
+;
+; SM70-PRESERVESIGN-STRICT-LABEL: fadd_f32_generic(
+; SM70-PRESERVESIGN-STRICT: {
+; SM70-PRESERVESIGN-STRICT-NEXT: .reg .pred %p<2>;
+; SM70-PRESERVESIGN-STRICT-NEXT: .reg .b32 %r<5>;
+; SM70-PRESERVESIGN-STRICT-NEXT: .reg .b64 %rd<2>;
+; SM70-PRESERVESIGN-STRICT-EMPTY:
+; SM70-PRESERVESIGN-STRICT-NEXT: // %bb.0:
+; SM70-PRESERVESIGN-STRICT-NEXT: ld.param::func.b32 %r2, [fadd_f32_generic_param_1];
+; SM70-PRESERVESIGN-STRICT-NEXT: ld.param::func.b64 %rd1, [fadd_f32_generic_param_0];
+; SM70-PRESERVESIGN-STRICT-NEXT: ld.relaxed.sys.b32 %r4, [%rd1];
+; SM70-PRESERVESIGN-STRICT-NEXT: $L__BB1_1: // %atomicrmw.start
+; SM70-PRESERVESIGN-STRICT-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-PRESERVESIGN-STRICT-NEXT: add.rn.ftz.f32 %r3, %r4, %r2;
+; SM70-PRESERVESIGN-STRICT-NEXT: atom.relaxed.sys.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM70-PRESERVESIGN-STRICT-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM70-PRESERVESIGN-STRICT-NEXT: mov.b32 %r4, %r1;
+; SM70-PRESERVESIGN-STRICT-NEXT: @%p1 bra $L__BB1_1;
+; SM70-PRESERVESIGN-STRICT-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-PRESERVESIGN-STRICT-NEXT: st.param::func.b32 [func_retval0], %r1;
+; SM70-PRESERVESIGN-STRICT-NEXT: ret;
+;
+; SM70-PRESERVESIGN-LEGACY-LABEL: fadd_f32_generic(
+; SM70-PRESERVESIGN-LEGACY: {
+; SM70-PRESERVESIGN-LEGACY-NEXT: .reg .b32 %r<3>;
+; SM70-PRESERVESIGN-LEGACY-NEXT: .reg .b64 %rd<2>;
+; SM70-PRESERVESIGN-LEGACY-EMPTY:
+; SM70-PRESERVESIGN-LEGACY-NEXT: // %bb.0:
+; SM70-PRESERVESIGN-LEGACY-NEXT: ld.param::func.b64 %rd1, [fadd_f32_generic_param_0];
+; SM70-PRESERVESIGN-LEGACY-NEXT: ld.param::func.b32 %r1, [fadd_f32_generic_param_1];
+; SM70-PRESERVESIGN-LEGACY-NEXT: atom.relaxed.sys.add.f32 %r2, [%rd1], %r1;
+; SM70-PRESERVESIGN-LEGACY-NEXT: st.param::func.b32 [func_retval0], %r2;
+; SM70-PRESERVESIGN-LEGACY-NEXT: ret;
+;
+; SM70-PRESERVESIGN-FAST-LABEL: fadd_f32_generic(
+; SM70-PRESERVESIGN-FAST: {
+; SM70-PRESERVESIGN-FAST-NEXT: .reg .b32 %r<3>;
+; SM70-PRESERVESIGN-FAST-NEXT: .reg .b64 %rd<2>;
+; SM70-PRESERVESIGN-FAST-EMPTY:
+; SM70-PRESERVESIGN-FAST-NEXT: // %bb.0:
+; SM70-PRESERVESIGN-FAST-NEXT: ld.param::func.b64 %rd1, [fadd_f32_generic_param_0];
+; SM70-PRESERVESIGN-FAST-NEXT: ld.param::func.b32 %r1, [fadd_f32_generic_param_1];
+; SM70-PRESERVESIGN-FAST-NEXT: atom.relaxed.sys.add.f32 %r2, [%rd1], %r1;
+; SM70-PRESERVESIGN-FAST-NEXT: st.param::func.b32 [func_retval0], %r2;
+; SM70-PRESERVESIGN-FAST-NEXT: ret;
+;
+; SM90-IEEE-STRICT-LABEL: fadd_f32_generic(
+; SM90-IEEE-STRICT: {
+; SM90-IEEE-STRICT-NEXT: .reg .b32 %r<3>;
+; SM90-IEEE-STRICT-NEXT: .reg .b64 %rd<2>;
+; SM90-IEEE-STRICT-EMPTY:
+; SM90-IEEE-STRICT-NEXT: // %bb.0:
+; SM90-IEEE-STRICT-NEXT: ld.param::func.b64 %rd1, [fadd_f32_generic_param_0];
+; SM90-IEEE-STRICT-NEXT: ld.param::func.b32 %r1, [fadd_f32_generic_param_1];
+; SM90-IEEE-STRICT-NEXT: atom.relaxed.sys.add.noftz.f32 %r2, [%rd1], %r1;
+; SM90-IEEE-STRICT-NEXT: st.param::func.b32 [func_retval0], %r2;
+; SM90-IEEE-STRICT-NEXT: ret;
+;
+; SM90-IEEE-LEGACY-LABEL: fadd_f32_generic(
+; SM90-IEEE-LEGACY: {
+; SM90-IEEE-LEGACY-NEXT: .reg .b32 %r<3>;
+; SM90-IEEE-LEGACY-NEXT: .reg .b64 %rd<2>;
+; SM90-IEEE-LEGACY-EMPTY:
+; SM90-IEEE-LEGACY-NEXT: // %bb.0:
+; SM90-IEEE-LEGACY-NEXT: ld.param::func.b64 %rd1, [fadd_f32_generic_param_0];
+; SM90-IEEE-LEGACY-NEXT: ld.param::func.b32 %r1, [fadd_f32_generic_param_1];
+; SM90-IEEE-LEGACY-NEXT: atom.relaxed.sys.add.f32 %r2, [%rd1], %r1;
+; SM90-IEEE-LEGACY-NEXT: st.param::func.b32 [func_retval0], %r2;
+; SM90-IEEE-LEGACY-NEXT: ret;
+;
+; SM90-IEEE-FAST-LABEL: fadd_f32_generic(
+; SM90-IEEE-FAST: {
+; SM90-IEEE-FAST-NEXT: .reg .b32 %r<3>;
+; SM90-IEEE-FAST-NEXT: .reg .b64 %rd<2>;
+; SM90-IEEE-FAST-EMPTY:
+; SM90-IEEE-FAST-NEXT: // %bb.0:
+; SM90-IEEE-FAST-NEXT: ld.param::func.b64 %rd1, [fadd_f32_generic_param_0];
+; SM90-IEEE-FAST-NEXT: ld.param::func.b32 %r1, [fadd_f32_generic_param_1];
+; SM90-IEEE-FAST-NEXT: atom.relaxed.sys.add.noftz.f32 %r2, [%rd1], %r1;
+; SM90-IEEE-FAST-NEXT: st.param::func.b32 [func_retval0], %r2;
+; SM90-IEEE-FAST-NEXT: ret;
+;
+; SM90-PRESERVESIGN-STRICT-LABEL: fadd_f32_generic(
+; SM90-PRESERVESIGN-STRICT: {
+; SM90-PRESERVESIGN-STRICT-NEXT: .reg .pred %p<2>;
+; SM90-PRESERVESIGN-STRICT-NEXT: .reg .b32 %r<5>;
+; SM90-PRESERVESIGN-STRICT-NEXT: .reg .b64 %rd<2>;
+; SM90-PRESERVESIGN-STRICT-EMPTY:
+; SM90-PRESERVESIGN-STRICT-NEXT: // %bb.0:
+; SM90-PRESERVESIGN-STRICT-NEXT: ld.param::func.b32 %r2, [fadd_f32_generic_param_1];
+; SM90-PRESERVESIGN-STRICT-NEXT: ld.param::func.b64 %rd1, [fadd_f32_generic_param_0];
+; SM90-PRESERVESIGN-STRICT-NEXT: ld.relaxed.sys.b32 %r4, [%rd1];
+; SM90-PRESERVESIGN-STRICT-NEXT: $L__BB1_1: // %atomicrmw.start
+; SM90-PRESERVESIGN-STRICT-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-PRESERVESIGN-STRICT-NEXT: add.rn.ftz.f32 %r3, %r4, %r2;
+; SM90-PRESERVESIGN-STRICT-NEXT: atom.relaxed.sys.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM90-PRESERVESIGN-STRICT-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM90-PRESERVESIGN-STRICT-NEXT: mov.b32 %r4, %r1;
+; SM90-PRESERVESIGN-STRICT-NEXT: @%p1 bra $L__BB1_1;
+; SM90-PRESERVESIGN-STRICT-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-PRESERVESIGN-STRICT-NEXT: st.param::func.b32 [func_retval0], %r1;
+; SM90-PRESERVESIGN-STRICT-NEXT: ret;
+;
+; SM90-PRESERVESIGN-LEGACY-LABEL: fadd_f32_generic(
+; SM90-PRESERVESIGN-LEGACY: {
+; SM90-PRESERVESIGN-LEGACY-NEXT: .reg .b32 %r<3>;
+; SM90-PRESERVESIGN-LEGACY-NEXT: .reg .b64 %rd<2>;
+; SM90-PRESERVESIGN-LEGACY-EMPTY:
+; SM90-PRESERVESIGN-LEGACY-NEXT: // %bb.0:
+; SM90-PRESERVESIGN-LEGACY-NEXT: ld.param::func.b64 %rd1, [fadd_f32_generic_param_0];
+; SM90-PRESERVESIGN-LEGACY-NEXT: ld.param::func.b32 %r1, [fadd_f32_generic_param_1];
+; SM90-PRESERVESIGN-LEGACY-NEXT: atom.relaxed.sys.add.f32 %r2, [%rd1], %r1;
+; SM90-PRESERVESIGN-LEGACY-NEXT: st.param::func.b32 [func_retval0], %r2;
+; SM90-PRESERVESIGN-LEGACY-NEXT: ret;
+;
+; SM90-PRESERVESIGN-FAST-LABEL: fadd_f32_generic(
+; SM90-PRESERVESIGN-FAST: {
+; SM90-PRESERVESIGN-FAST-NEXT: .reg .b32 %r<3>;
+; SM90-PRESERVESIGN-FAST-NEXT: .reg .b64 %rd<2>;
+; SM90-PRESERVESIGN-FAST-EMPTY:
+; SM90-PRESERVESIGN-FAST-NEXT: // %bb.0:
+; SM90-PRESERVESIGN-FAST-NEXT: ld.param::func.b64 %rd1, [fadd_f32_generic_param_0];
+; SM90-PRESERVESIGN-FAST-NEXT: ld.param::func.b32 %r1, [fadd_f32_generic_param_1];
+; SM90-PRESERVESIGN-FAST-NEXT: atom.relaxed.sys.add.f32 %r2, [%rd1], %r1;
+; SM90-PRESERVESIGN-FAST-NEXT: st.param::func.b32 [func_retval0], %r2;
+; SM90-PRESERVESIGN-FAST-NEXT: ret;
+ %r = atomicrmw fadd ptr %addr, float %val monotonic
+ ret float %r
+}
+
+; shared f32: atom.add.f32 does not flush denormals here, so it conforms only
+; when the function is in IEEE mode.
+define float @fadd_f32_shared(ptr addrspace(3) %addr, float %val) {
+; SM70-IEEE-STRICT-LABEL: fadd_f32_shared(
+; SM70-IEEE-STRICT: {
+; SM70-IEEE-STRICT-NEXT: .reg .b32 %r<3>;
+; SM70-IEEE-STRICT-NEXT: .reg .b64 %rd<2>;
+; SM70-IEEE-STRICT-EMPTY:
+; SM70-IEEE-STRICT-NEXT: // %bb.0:
+; SM70-IEEE-STRICT-NEXT: ld.param::func.b64 %rd1, [fadd_f32_shared_param_0];
+; SM70-IEEE-STRICT-NEXT: ld.param::func.b32 %r1, [fadd_f32_shared_param_1];
+; SM70-IEEE-STRICT-NEXT: atom.relaxed.sys.shared.add.f32 %r2, [%rd1], %r1;
+; SM70-IEEE-STRICT-NEXT: st.param::func.b32 [func_retval0], %r2;
+; SM70-IEEE-STRICT-NEXT: ret;
+;
+; SM70-IEEE-LEGACY-LABEL: fadd_f32_shared(
+; SM70-IEEE-LEGACY: {
+; SM70-IEEE-LEGACY-NEXT: .reg .b32 %r<3>;
+; SM70-IEEE-LEGACY-NEXT: .reg .b64 %rd<2>;
+; SM70-IEEE-LEGACY-EMPTY:
+; SM70-IEEE-LEGACY-NEXT: // %bb.0:
+; SM70-IEEE-LEGACY-NEXT: ld.param::func.b64 %rd1, [fadd_f32_shared_param_0];
+; SM70-IEEE-LEGACY-NEXT: ld.param::func.b32 %r1, [fadd_f32_shared_param_1];
+; SM70-IEEE-LEGACY-NEXT: atom.relaxed.sys.shared.add.f32 %r2, [%rd1], %r1;
+; SM70-IEEE-LEGACY-NEXT: st.param::func.b32 [func_retval0], %r2;
+; SM70-IEEE-LEGACY-NEXT: ret;
+;
+; SM70-IEEE-FAST-LABEL: fadd_f32_shared(
+; SM70-IEEE-FAST: {
+; SM70-IEEE-FAST-NEXT: .reg .b32 %r<3>;
+; SM70-IEEE-FAST-NEXT: .reg .b64 %rd<2>;
+; SM70-IEEE-FAST-EMPTY:
+; SM70-IEEE-FAST-NEXT: // %bb.0:
+; SM70-IEEE-FAST-NEXT: ld.param::func.b64 %rd1, [fadd_f32_shared_param_0];
+; SM70-IEEE-FAST-NEXT: ld.param::func.b32 %r1, [fadd_f32_shared_param_1];
+; SM70-IEEE-FAST-NEXT: atom.relaxed.sys.shared.add.f32 %r2, [%rd1], %r1;
+; SM70-IEEE-FAST-NEXT: st.param::func.b32 [func_retval0], %r2;
+; SM70-IEEE-FAST-NEXT: ret;
+;
+; SM70-PRESERVESIGN-STRICT-LABEL: fadd_f32_shared(
+; SM70-PRESERVESIGN-STRICT: {
+; SM70-PRESERVESIGN-STRICT-NEXT: .reg .pred %p<2>;
+; SM70-PRESERVESIGN-STRICT-NEXT: .reg .b32 %r<5>;
+; SM70-PRESERVESIGN-STRICT-NEXT: .reg .b64 %rd<2>;
+; SM70-PRESERVESIGN-STRICT-EMPTY:
+; SM70-PRESERVESIGN-STRICT-NEXT: // %bb.0:
+; SM70-PRESERVESIGN-STRICT-NEXT: ld.param::func.b32 %r2, [fadd_f32_shared_param_1];
+; SM70-PRESERVESIGN-STRICT-NEXT: ld.param::func.b64 %rd1, [fadd_f32_shared_param_0];
+; SM70-PRESERVESIGN-STRICT-NEXT: ld.relaxed.sys.shared.b32 %r4, [%rd1];
+; SM70-PRESERVESIGN-STRICT-NEXT: $L__BB2_1: // %atomicrmw.start
+; SM70-PRESERVESIGN-STRICT-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-PRESERVESIGN-STRICT-NEXT: add.rn.ftz.f32 %r3, %r4, %r2;
+; SM70-PRESERVESIGN-STRICT-NEXT: atom.relaxed.sys.shared.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM70-PRESERVESIGN-STRICT-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM70-PRESERVESIGN-STRICT-NEXT: mov.b32 %r4, %r1;
+; SM70-PRESERVESIGN-STRICT-NEXT: @%p1 bra $L__BB2_1;
+; SM70-PRESERVESIGN-STRICT-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-PRESERVESIGN-STRICT-NEXT: st.param::func.b32 [func_retval0], %r1;
+; SM70-PRESERVESIGN-STRICT-NEXT: ret;
+;
+; SM70-PRESERVESIGN-LEGACY-LABEL: fadd_f32_shared(
+; SM70-PRESERVESIGN-LEGACY: {
+; SM70-PRESERVESIGN-LEGACY-NEXT: .reg .b32 %r<3>;
+; SM70-PRESERVESIGN-LEGACY-NEXT: .reg .b64 %rd<2>;
+; SM70-PRESERVESIGN-LEGACY-EMPTY:
+; SM70-PRESERVESIGN-LEGACY-NEXT: // %bb.0:
+; SM70-PRESERVESIGN-LEGACY-NEXT: ld.param::func.b64 %rd1, [fadd_f32_shared_param_0];
+; SM70-PRESERVESIGN-LEGACY-NEXT: ld.param::func.b32 %r1, [fadd_f32_shared_param_1];
+; SM70-PRESERVESIGN-LEGACY-NEXT: atom.relaxed.sys.shared.add.f32 %r2, [%rd1], %r1;
+; SM70-PRESERVESIGN-LEGACY-NEXT: st.param::func.b32 [func_retval0], %r2;
+; SM70-PRESERVESIGN-LEGACY-NEXT: ret;
+;
+; SM70-PRESERVESIGN-FAST-LABEL: fadd_f32_shared(
+; SM70-PRESERVESIGN-FAST: {
+; SM70-PRESERVESIGN-FAST-NEXT: .reg .b32 %r<3>;
+; SM70-PRESERVESIGN-FAST-NEXT: .reg .b64 %rd<2>;
+; SM70-PRESERVESIGN-FAST-EMPTY:
+; SM70-PRESERVESIGN-FAST-NEXT: // %bb.0:
+; SM70-PRESERVESIGN-FAST-NEXT: ld.param::func.b64 %rd1, [fadd_f32_shared_param_0];
+; SM70-PRESERVESIGN-FAST-NEXT: ld.param::func.b32 %r1, [fadd_f32_shared_param_1];
+; SM70-PRESERVESIGN-FAST-NEXT: atom.relaxed.sys.shared.add.f32 %r2, [%rd1], %r1;
+; SM70-PRESERVESIGN-FAST-NEXT: st.param::func.b32 [func_retval0], %r2;
+; SM70-PRESERVESIGN-FAST-NEXT: ret;
+;
+; SM90-IEEE-STRICT-LABEL: fadd_f32_shared(
+; SM90-IEEE-STRICT: {
+; SM90-IEEE-STRICT-NEXT: .reg .b32 %r<3>;
+; SM90-IEEE-STRICT-NEXT: .reg .b64 %rd<2>;
+; SM90-IEEE-STRICT-EMPTY:
+; SM90-IEEE-STRICT-NEXT: // %bb.0:
+; SM90-IEEE-STRICT-NEXT: ld.param::func.b64 %rd1, [fadd_f32_shared_param_0];
+; SM90-IEEE-STRICT-NEXT: ld.param::func.b32 %r1, [fadd_f32_shared_param_1];
+; SM90-IEEE-STRICT-NEXT: atom.relaxed.sys.shared.add.noftz.f32 %r2, [%rd1], %r1;
+; SM90-IEEE-STRICT-NEXT: st.param::func.b32 [func_retval0], %r2;
+; SM90-IEEE-STRICT-NEXT: ret;
+;
+; SM90-IEEE-LEGACY-LABEL: fadd_f32_shared(
+; SM90-IEEE-LEGACY: {
+; SM90-IEEE-LEGACY-NEXT: .reg .b32 %r<3>;
+; SM90-IEEE-LEGACY-NEXT: .reg .b64 %rd<2>;
+; SM90-IEEE-LEGACY-EMPTY:
+; SM90-IEEE-LEGACY-NEXT: // %bb.0:
+; SM90-IEEE-LEGACY-NEXT: ld.param::func.b64 %rd1, [fadd_f32_shared_param_0];
+; SM90-IEEE-LEGACY-NEXT: ld.param::func.b32 %r1, [fadd_f32_shared_param_1];
+; SM90-IEEE-LEGACY-NEXT: atom.relaxed.sys.shared.add.f32 %r2, [%rd1], %r1;
+; SM90-IEEE-LEGACY-NEXT: st.param::func.b32 [func_retval0], %r2;
+; SM90-IEEE-LEGACY-NEXT: ret;
+;
+; SM90-IEEE-FAST-LABEL: fadd_f32_shared(
+; SM90-IEEE-FAST: {
+; SM90-IEEE-FAST-NEXT: .reg .b32 %r<3>;
+; SM90-IEEE-FAST-NEXT: .reg .b64 %rd<2>;
+; SM90-IEEE-FAST-EMPTY:
+; SM90-IEEE-FAST-NEXT: // %bb.0:
+; SM90-IEEE-FAST-NEXT: ld.param::func.b64 %rd1, [fadd_f32_shared_param_0];
+; SM90-IEEE-FAST-NEXT: ld.param::func.b32 %r1, [fadd_f32_shared_param_1];
+; SM90-IEEE-FAST-NEXT: atom.relaxed.sys.shared.add.noftz.f32 %r2, [%rd1], %r1;
+; SM90-IEEE-FAST-NEXT: st.param::func.b32 [func_retval0], %r2;
+; SM90-IEEE-FAST-NEXT: ret;
+;
+; SM90-PRESERVESIGN-STRICT-LABEL: fadd_f32_shared(
+; SM90-PRESERVESIGN-STRICT: {
+; SM90-PRESERVESIGN-STRICT-NEXT: .reg .pred %p<2>;
+; SM90-PRESERVESIGN-STRICT-NEXT: .reg .b32 %r<5>;
+; SM90-PRESERVESIGN-STRICT-NEXT: .reg .b64 %rd<2>;
+; SM90-PRESERVESIGN-STRICT-EMPTY:
+; SM90-PRESERVESIGN-STRICT-NEXT: // %bb.0:
+; SM90-PRESERVESIGN-STRICT-NEXT: ld.param::func.b32 %r2, [fadd_f32_shared_param_1];
+; SM90-PRESERVESIGN-STRICT-NEXT: ld.param::func.b64 %rd1, [fadd_f32_shared_param_0];
+; SM90-PRESERVESIGN-STRICT-NEXT: ld.relaxed.sys.shared.b32 %r4, [%rd1];
+; SM90-PRESERVESIGN-STRICT-NEXT: $L__BB2_1: // %atomicrmw.start
+; SM90-PRESERVESIGN-STRICT-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-PRESERVESIGN-STRICT-NEXT: add.rn.ftz.f32 %r3, %r4, %r2;
+; SM90-PRESERVESIGN-STRICT-NEXT: atom.relaxed.sys.shared.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM90-PRESERVESIGN-STRICT-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM90-PRESERVESIGN-STRICT-NEXT: mov.b32 %r4, %r1;
+; SM90-PRESERVESIGN-STRICT-NEXT: @%p1 bra $L__BB2_1;
+; SM90-PRESERVESIGN-STRICT-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-PRESERVESIGN-STRICT-NEXT: st.param::func.b32 [func_retval0], %r1;
+; SM90-PRESERVESIGN-STRICT-NEXT: ret;
+;
+; SM90-PRESERVESIGN-LEGACY-LABEL: fadd_f32_shared(
+; SM90-PRESERVESIGN-LEGACY: {
+; SM90-PRESERVESIGN-LEGACY-NEXT: .reg .b32 %r<3>;
+; SM90-PRESERVESIGN-LEGACY-NEXT: .reg .b64 %rd<2>;
+; SM90-PRESERVESIGN-LEGACY-EMPTY:
+; SM90-PRESERVESIGN-LEGACY-NEXT: // %bb.0:
+; SM90-PRESERVESIGN-LEGACY-NEXT: ld.param::func.b64 %rd1, [fadd_f32_shared_param_0];
+; SM90-PRESERVESIGN-LEGACY-NEXT: ld.param::func.b32 %r1, [fadd_f32_shared_param_1];
+; SM90-PRESERVESIGN-LEGACY-NEXT: atom.relaxed.sys.shared.add.f32 %r2, [%rd1], %r1;
+; SM90-PRESERVESIGN-LEGACY-NEXT: st.param::func.b32 [func_retval0], %r2;
+; SM90-PRESERVESIGN-LEGACY-NEXT: ret;
+;
+; SM90-PRESERVESIGN-FAST-LABEL: fadd_f32_shared(
+; SM90-PRESERVESIGN-FAST: {
+; SM90-PRESERVESIGN-FAST-NEXT: .reg .b32 %r<3>;
+; SM90-PRESERVESIGN-FAST-NEXT: .reg .b64 %rd<2>;
+; SM90-PRESERVESIGN-FAST-EMPTY:
+; SM90-PRESERVESIGN-FAST-NEXT: // %bb.0:
+; SM90-PRESERVESIGN-FAST-NEXT: ld.param::func.b64 %rd1, [fadd_f32_shared_param_0];
+; SM90-PRESERVESIGN-FAST-NEXT: ld.param::func.b32 %r1, [fadd_f32_shared_param_1];
+; SM90-PRESERVESIGN-FAST-NEXT: atom.relaxed.sys.shared.add.f32 %r2, [%rd1], %r1;
+; SM90-PRESERVESIGN-FAST-NEXT: st.param::func.b32 [func_retval0], %r2;
+; SM90-PRESERVESIGN-FAST-NEXT: ret;
+ %r = atomicrmw fadd ptr addrspace(3) %addr, float %val monotonic
+ ret float %r
+}
+
+; f16: atom.add.noftz.f16 never flushes denormals, so it conforms only when the
+; function is not in FTZ mode for f16.
+define half @fadd_f16_global(ptr addrspace(1) %addr, half %val) {
+; SM70-IEEE-STRICT-LABEL: fadd_f16_global(
+; SM70-IEEE-STRICT: {
+; SM70-IEEE-STRICT-NEXT: .reg .b16 %rs<3>;
+; SM70-IEEE-STRICT-NEXT: .reg .b64 %rd<2>;
+; SM70-IEEE-STRICT-EMPTY:
+; SM70-IEEE-STRICT-NEXT: // %bb.0:
+; SM70-IEEE-STRICT-NEXT: ld.param::func.b64 %rd1, [fadd_f16_global_param_0];
+; SM70-IEEE-STRICT-NEXT: ld.param::func.b16 %rs1, [fadd_f16_global_param_1];
+; SM70-IEEE-STRICT-NEXT: atom.relaxed.sys.global.add.noftz.f16 %rs2, [%rd1], %rs1;
+; SM70-IEEE-STRICT-NEXT: st.param::func.b16 [func_retval0], %rs2;
+; SM70-IEEE-STRICT-NEXT: ret;
+;
+; SM70-IEEE-LEGACY-LABEL: fadd_f16_global(
+; SM70-IEEE-LEGACY: {
+; SM70-IEEE-LEGACY-NEXT: .reg .b16 %rs<3>;
+; SM70-IEEE-LEGACY-NEXT: .reg .b64 %rd<2>;
+; SM70-IEEE-LEGACY-EMPTY:
+; SM70-IEEE-LEGACY-NEXT: // %bb.0:
+; SM70-IEEE-LEGACY-NEXT: ld.param::func.b64 %rd1, [fadd_f16_global_param_0];
+; SM70-IEEE-LEGACY-NEXT: ld.param::func.b16 %rs1, [fadd_f16_global_param_1];
+; SM70-IEEE-LEGACY-NEXT: atom.relaxed.sys.global.add.noftz.f16 %rs2, [%rd1], %rs1;
+; SM70-IEEE-LEGACY-NEXT: st.param::func.b16 [func_retval0], %rs2;
+; SM70-IEEE-LEGACY-NEXT: ret;
+;
+; SM70-IEEE-FAST-LABEL: fadd_f16_global(
+; SM70-IEEE-FAST: {
+; SM70-IEEE-FAST-NEXT: .reg .b16 %rs<3>;
+; SM70-IEEE-FAST-NEXT: .reg .b64 %rd<2>;
+; SM70-IEEE-FAST-EMPTY:
+; SM70-IEEE-FAST-NEXT: // %bb.0:
+; SM70-IEEE-FAST-NEXT: ld.param::func.b64 %rd1, [fadd_f16_global_param_0];
+; SM70-IEEE-FAST-NEXT: ld.param::func.b16 %rs1, [fadd_f16_global_param_1];
+; SM70-IEEE-FAST-NEXT: atom.relaxed.sys.global.add.noftz.f16 %rs2, [%rd1], %rs1;
+; SM70-IEEE-FAST-NEXT: st.param::func.b16 [func_retval0], %rs2;
+; SM70-IEEE-FAST-NEXT: ret;
+;
+; SM70-PRESERVESIGN-STRICT-LABEL: fadd_f16_global(
+; SM70-PRESERVESIGN-STRICT: {
+; SM70-PRESERVESIGN-STRICT-NEXT: .reg .pred %p<2>;
+; SM70-PRESERVESIGN-STRICT-NEXT: .reg .b16 %rs<4>;
+; SM70-PRESERVESIGN-STRICT-NEXT: .reg .b32 %r<15>;
+; SM70-PRESERVESIGN-STRICT-NEXT: .reg .b64 %rd<3>;
+; SM70-PRESERVESIGN-STRICT-EMPTY:
+; SM70-PRESERVESIGN-STRICT-NEXT: // %bb.0:
+; SM70-PRESERVESIGN-STRICT-NEXT: ld.param::func.b16 %rs1, [fadd_f16_global_param_1];
+; SM70-PRESERVESIGN-STRICT-NEXT: ld.param::func.b64 %rd2, [fadd_f16_global_param_0];
+; SM70-PRESERVESIGN-STRICT-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-PRESERVESIGN-STRICT-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-PRESERVESIGN-STRICT-NEXT: and.b32 %r5, %r4, 3;
+; SM70-PRESERVESIGN-STRICT-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-PRESERVESIGN-STRICT-NEXT: mov.b32 %r6, 65535;
+; SM70-PRESERVESIGN-STRICT-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-PRESERVESIGN-STRICT-NEXT: not.b32 %r2, %r7;
+; SM70-PRESERVESIGN-STRICT-NEXT: ld.relaxed.sys.global.b32 %r14, [%rd1];
+; SM70-PRESERVESIGN-STRICT-NEXT: $L__BB3_1: // %atomicrmw.start
+; SM70-PRESERVESIGN-STRICT-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-PRESERVESIGN-STRICT-NEXT: shr.u32 %r8, %r14, %r1;
+; SM70-PRESERVESIGN-STRICT-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM70-PRESERVESIGN-STRICT-NEXT: add.rn.ftz.f16 %rs3, %rs2, %rs1;
+; SM70-PRESERVESIGN-STRICT-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM70-PRESERVESIGN-STRICT-NEXT: shl.b32 %r10, %r9, %r1;
+; SM70-PRESERVESIGN-STRICT-NEXT: and.b32 %r11, %r14, %r2;
+; SM70-PRESERVESIGN-STRICT-NEXT: or.b32 %r12, %r11, %r10;
+; SM70-PRESERVESIGN-STRICT-NEXT: atom.relaxed.sys.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM70-PRESERVESIGN-STRICT-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM70-PRESERVESIGN-STRICT-NEXT: mov.b32 %r14, %r3;
+; SM70-PRESERVESIGN-STRICT-NEXT: @%p1 bra $L__BB3_1;
+; SM70-PRESERVESIGN-STRICT-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-PRESERVESIGN-STRICT-NEXT: shr.u32 %r13, %r3, %r1;
+; SM70-PRESERVESIGN-STRICT-NEXT: st.param::func.b16 [func_retval0], %r13;
+; SM70-PRESERVESIGN-STRICT-NEXT: ret;
+;
+; SM70-PRESERVESIGN-LEGACY-LABEL: fadd_f16_global(
+; SM70-PRESERVESIGN-LEGACY: {
+; SM70-PRESERVESIGN-LEGACY-NEXT: .reg .b16 %rs<3>;
+; SM70-PRESERVESIGN-LEGACY-NEXT: .reg .b64 %rd<2>;
+; SM70-PRESERVESIGN-LEGACY-EMPTY:
+; SM70-PRESERVESIGN-LEGACY-NEXT: // %bb.0:
+; SM70-PRESERVESIGN-LEGACY-NEXT: ld.param::func.b64 %rd1, [fadd_f16_global_param_0];
+; SM70-PRESERVESIGN-LEGACY-NEXT: ld.param::func.b16 %rs1, [fadd_f16_global_param_1];
+; SM70-PRESERVESIGN-LEGACY-NEXT: atom.relaxed.sys.global.add.noftz.f16 %rs2, [%rd1], %rs1;
+; SM70-PRESERVESIGN-LEGACY-NEXT: st.param::func.b16 [func_retval0], %rs2;
+; SM70-PRESERVESIGN-LEGACY-NEXT: ret;
+;
+; SM70-PRESERVESIGN-FAST-LABEL: fadd_f16_global(
+; SM70-PRESERVESIGN-FAST: {
+; SM70-PRESERVESIGN-FAST-NEXT: .reg .b16 %rs<3>;
+; SM70-PRESERVESIGN-FAST-NEXT: .reg .b64 %rd<2>;
+; SM70-PRESERVESIGN-FAST-EMPTY:
+; SM70-PRESERVESIGN-FAST-NEXT: // %bb.0:
+; SM70-PRESERVESIGN-FAST-NEXT: ld.param::func.b64 %rd1, [fadd_f16_global_param_0];
+; SM70-PRESERVESIGN-FAST-NEXT: ld.param::func.b16 %rs1, [fadd_f16_global_param_1];
+; SM70-PRESERVESIGN-FAST-NEXT: atom.relaxed.sys.global.add.noftz.f16 %rs2, [%rd1], %rs1;
+; SM70-PRESERVESIGN-FAST-NEXT: st.param::func.b16 [func_retval0], %rs2;
+; SM70-PRESERVESIGN-FAST-NEXT: ret;
+;
+; SM90-IEEE-STRICT-LABEL: fadd_f16_global(
+; SM90-IEEE-STRICT: {
+; SM90-IEEE-STRICT-NEXT: .reg .b16 %rs<3>;
+; SM90-IEEE-STRICT-NEXT: .reg .b64 %rd<2>;
+; SM90-IEEE-STRICT-EMPTY:
+; SM90-IEEE-STRICT-NEXT: // %bb.0:
+; SM90-IEEE-STRICT-NEXT: ld.param::func.b64 %rd1, [fadd_f16_global_param_0];
+; SM90-IEEE-STRICT-NEXT: ld.param::func.b16 %rs1, [fadd_f16_global_param_1];
+; SM90-IEEE-STRICT-NEXT: atom.relaxed.sys.global.add.noftz.f16 %rs2, [%rd1], %rs1;
+; SM90-IEEE-STRICT-NEXT: st.param::func.b16 [func_retval0], %rs2;
+; SM90-IEEE-STRICT-NEXT: ret;
+;
+; SM90-IEEE-LEGACY-LABEL: fadd_f16_global(
+; SM90-IEEE-LEGACY: {
+; SM90-IEEE-LEGACY-NEXT: .reg .b16 %rs<3>;
+; SM90-IEEE-LEGACY-NEXT: .reg .b64 %rd<2>;
+; SM90-IEEE-LEGACY-EMPTY:
+; SM90-IEEE-LEGACY-NEXT: // %bb.0:
+; SM90-IEEE-LEGACY-NEXT: ld.param::func.b64 %rd1, [fadd_f16_global_param_0];
+; SM90-IEEE-LEGACY-NEXT: ld.param::func.b16 %rs1, [fadd_f16_global_param_1];
+; SM90-IEEE-LEGACY-NEXT: atom.relaxed.sys.global.add.noftz.f16 %rs2, [%rd1], %rs1;
+; SM90-IEEE-LEGACY-NEXT: st.param::func.b16 [func_retval0], %rs2;
+; SM90-IEEE-LEGACY-NEXT: ret;
+;
+; SM90-IEEE-FAST-LABEL: fadd_f16_global(
+; SM90-IEEE-FAST: {
+; SM90-IEEE-FAST-NEXT: .reg .b16 %rs<3>;
+; SM90-IEEE-FAST-NEXT: .reg .b64 %rd<2>;
+; SM90-IEEE-FAST-EMPTY:
+; SM90-IEEE-FAST-NEXT: // %bb.0:
+; SM90-IEEE-FAST-NEXT: ld.param::func.b64 %rd1, [fadd_f16_global_param_0];
+; SM90-IEEE-FAST-NEXT: ld.param::func.b16 %rs1, [fadd_f16_global_param_1];
+; SM90-IEEE-FAST-NEXT: atom.relaxed.sys.global.add.noftz.f16 %rs2, [%rd1], %rs1;
+; SM90-IEEE-FAST-NEXT: st.param::func.b16 [func_retval0], %rs2;
+; SM90-IEEE-FAST-NEXT: ret;
+;
+; SM90-PRESERVESIGN-STRICT-LABEL: fadd_f16_global(
+; SM90-PRESERVESIGN-STRICT: {
+; SM90-PRESERVESIGN-STRICT-NEXT: .reg .pred %p<2>;
+; SM90-PRESERVESIGN-STRICT-NEXT: .reg .b16 %rs<4>;
+; SM90-PRESERVESIGN-STRICT-NEXT: .reg .b32 %r<15>;
+; SM90-PRESERVESIGN-STRICT-NEXT: .reg .b64 %rd<3>;
+; SM90-PRESERVESIGN-STRICT-EMPTY:
+; SM90-PRESERVESIGN-STRICT-NEXT: // %bb.0:
+; SM90-PRESERVESIGN-STRICT-NEXT: ld.param::func.b16 %rs1, [fadd_f16_global_param_1];
+; SM90-PRESERVESIGN-STRICT-NEXT: ld.param::func.b64 %rd2, [fadd_f16_global_param_0];
+; SM90-PRESERVESIGN-STRICT-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-PRESERVESIGN-STRICT-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM90-PRESERVESIGN-STRICT-NEXT: and.b32 %r5, %r4, 3;
+; SM90-PRESERVESIGN-STRICT-NEXT: shl.b32 %r1, %r5, 3;
+; SM90-PRESERVESIGN-STRICT-NEXT: mov.b32 %r6, 65535;
+; SM90-PRESERVESIGN-STRICT-NEXT: shl.b32 %r7, %r6, %r1;
+; SM90-PRESERVESIGN-STRICT-NEXT: not.b32 %r2, %r7;
+; SM90-PRESERVESIGN-STRICT-NEXT: ld.relaxed.sys.global.b32 %r14, [%rd1];
+; SM90-PRESERVESIGN-STRICT-NEXT: $L__BB3_1: // %atomicrmw.start
+; SM90-PRESERVESIGN-STRICT-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-PRESERVESIGN-STRICT-NEXT: shr.u32 %r8, %r14, %r1;
+; SM90-PRESERVESIGN-STRICT-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM90-PRESERVESIGN-STRICT-NEXT: add.rn.ftz.f16 %rs3, %rs2, %rs1;
+; SM90-PRESERVESIGN-STRICT-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM90-PRESERVESIGN-STRICT-NEXT: shl.b32 %r10, %r9, %r1;
+; SM90-PRESERVESIGN-STRICT-NEXT: and.b32 %r11, %r14, %r2;
+; SM90-PRESERVESIGN-STRICT-NEXT: or.b32 %r12, %r11, %r10;
+; SM90-PRESERVESIGN-STRICT-NEXT: atom.relaxed.sys.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM90-PRESERVESIGN-STRICT-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM90-PRESERVESIGN-STRICT-NEXT: mov.b32 %r14, %r3;
+; SM90-PRESERVESIGN-STRICT-NEXT: @%p1 bra $L__BB3_1;
+; SM90-PRESERVESIGN-STRICT-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-PRESERVESIGN-STRICT-NEXT: shr.u32 %r13, %r3, %r1;
+; SM90-PRESERVESIGN-STRICT-NEXT: st.param::func.b16 [func_retval0], %r13;
+; SM90-PRESERVESIGN-STRICT-NEXT: ret;
+;
+; SM90-PRESERVESIGN-LEGACY-LABEL: fadd_f16_global(
+; SM90-PRESERVESIGN-LEGACY: {
+; SM90-PRESERVESIGN-LEGACY-NEXT: .reg .b16 %rs<3>;
+; SM90-PRESERVESIGN-LEGACY-NEXT: .reg .b64 %rd<2>;
+; SM90-PRESERVESIGN-LEGACY-EMPTY:
+; SM90-PRESERVESIGN-LEGACY-NEXT: // %bb.0:
+; SM90-PRESERVESIGN-LEGACY-NEXT: ld.param::func.b64 %rd1, [fadd_f16_global_param_0];
+; SM90-PRESERVESIGN-LEGACY-NEXT: ld.param::func.b16 %rs1, [fadd_f16_global_param_1];
+; SM90-PRESERVESIGN-LEGACY-NEXT: atom.relaxed.sys.global.add.noftz.f16 %rs2, [%rd1], %rs1;
+; SM90-PRESERVESIGN-LEGACY-NEXT: st.param::func.b16 [func_retval0], %rs2;
+; SM90-PRESERVESIGN-LEGACY-NEXT: ret;
+;
+; SM90-PRESERVESIGN-FAST-LABEL: fadd_f16_global(
+; SM90-PRESERVESIGN-FAST: {
+; SM90-PRESERVESIGN-FAST-NEXT: .reg .b16 %rs<3>;
+; SM90-PRESERVESIGN-FAST-NEXT: .reg .b64 %rd<2>;
+; SM90-PRESERVESIGN-FAST-EMPTY:
+; SM90-PRESERVESIGN-FAST-NEXT: // %bb.0:
+; SM90-PRESERVESIGN-FAST-NEXT: ld.param::func.b64 %rd1, [fadd_f16_global_param_0];
+; SM90-PRESERVESIGN-FAST-NEXT: ld.param::func.b16 %rs1, [fadd_f16_global_param_1];
+; SM90-PRESERVESIGN-FAST-NEXT: atom.relaxed.sys.global.add.noftz.f16 %rs2, [%rd1], %rs1;
+; SM90-PRESERVESIGN-FAST-NEXT: st.param::func.b16 [func_retval0], %rs2;
+; SM90-PRESERVESIGN-FAST-NEXT: ret;
+ %r = atomicrmw fadd ptr addrspace(1) %addr, half %val monotonic
+ ret half %r
+}
+
+define half @fadd_f16_generic(ptr %addr, half %val) {
+; SM70-IEEE-STRICT-LABEL: fadd_f16_generic(
+; SM70-IEEE-STRICT: {
+; SM70-IEEE-STRICT-NEXT: .reg .b16 %rs<3>;
+; SM70-IEEE-STRICT-NEXT: .reg .b64 %rd<2>;
+; SM70-IEEE-STRICT-EMPTY:
+; SM70-IEEE-STRICT-NEXT: // %bb.0:
+; SM70-IEEE-STRICT-NEXT: ld.param::func.b64 %rd1, [fadd_f16_generic_param_0];
+; SM70-IEEE-STRICT-NEXT: ld.param::func.b16 %rs1, [fadd_f16_generic_param_1];
+; SM70-IEEE-STRICT-NEXT: atom.relaxed.sys.add.noftz.f16 %rs2, [%rd1], %rs1;
+; SM70-IEEE-STRICT-NEXT: st.param::func.b16 [func_retval0], %rs2;
+; SM70-IEEE-STRICT-NEXT: ret;
+;
+; SM70-IEEE-LEGACY-LABEL: fadd_f16_generic(
+; SM70-IEEE-LEGACY: {
+; SM70-IEEE-LEGACY-NEXT: .reg .b16 %rs<3>;
+; SM70-IEEE-LEGACY-NEXT: .reg .b64 %rd<2>;
+; SM70-IEEE-LEGACY-EMPTY:
+; SM70-IEEE-LEGACY-NEXT: // %bb.0:
+; SM70-IEEE-LEGACY-NEXT: ld.param::func.b64 %rd1, [fadd_f16_generic_param_0];
+; SM70-IEEE-LEGACY-NEXT: ld.param::func.b16 %rs1, [fadd_f16_generic_param_1];
+; SM70-IEEE-LEGACY-NEXT: atom.relaxed.sys.add.noftz.f16 %rs2, [%rd1], %rs1;
+; SM70-IEEE-LEGACY-NEXT: st.param::func.b16 [func_retval0], %rs2;
+; SM70-IEEE-LEGACY-NEXT: ret;
+;
+; SM70-IEEE-FAST-LABEL: fadd_f16_generic(
+; SM70-IEEE-FAST: {
+; SM70-IEEE-FAST-NEXT: .reg .b16 %rs<3>;
+; SM70-IEEE-FAST-NEXT: .reg .b64 %rd<2>;
+; SM70-IEEE-FAST-EMPTY:
+; SM70-IEEE-FAST-NEXT: // %bb.0:
+; SM70-IEEE-FAST-NEXT: ld.param::func.b64 %rd1, [fadd_f16_generic_param_0];
+; SM70-IEEE-FAST-NEXT: ld.param::func.b16 %rs1, [fadd_f16_generic_param_1];
+; SM70-IEEE-FAST-NEXT: atom.relaxed.sys.add.noftz.f16 %rs2, [%rd1], %rs1;
+; SM70-IEEE-FAST-NEXT: st.param::func.b16 [func_retval0], %rs2;
+; SM70-IEEE-FAST-NEXT: ret;
+;
+; SM70-PRESERVESIGN-STRICT-LABEL: fadd_f16_generic(
+; SM70-PRESERVESIGN-STRICT: {
+; SM70-PRESERVESIGN-STRICT-NEXT: .reg .pred %p<2>;
+; SM70-PRESERVESIGN-STRICT-NEXT: .reg .b16 %rs<4>;
+; SM70-PRESERVESIGN-STRICT-NEXT: .reg .b32 %r<15>;
+; SM70-PRESERVESIGN-STRICT-NEXT: .reg .b64 %rd<3>;
+; SM70-PRESERVESIGN-STRICT-EMPTY:
+; SM70-PRESERVESIGN-STRICT-NEXT: // %bb.0:
+; SM70-PRESERVESIGN-STRICT-NEXT: ld.param::func.b16 %rs1, [fadd_f16_generic_param_1];
+; SM70-PRESERVESIGN-STRICT-NEXT: ld.param::func.b64 %rd2, [fadd_f16_generic_param_0];
+; SM70-PRESERVESIGN-STRICT-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-PRESERVESIGN-STRICT-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-PRESERVESIGN-STRICT-NEXT: and.b32 %r5, %r4, 3;
+; SM70-PRESERVESIGN-STRICT-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-PRESERVESIGN-STRICT-NEXT: mov.b32 %r6, 65535;
+; SM70-PRESERVESIGN-STRICT-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-PRESERVESIGN-STRICT-NEXT: not.b32 %r2, %r7;
+; SM70-PRESERVESIGN-STRICT-NEXT: ld.relaxed.sys.b32 %r14, [%rd1];
+; SM70-PRESERVESIGN-STRICT-NEXT: $L__BB4_1: // %atomicrmw.start
+; SM70-PRESERVESIGN-STRICT-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-PRESERVESIGN-STRICT-NEXT: shr.u32 %r8, %r14, %r1;
+; SM70-PRESERVESIGN-STRICT-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM70-PRESERVESIGN-STRICT-NEXT: add.rn.ftz.f16 %rs3, %rs2, %rs1;
+; SM70-PRESERVESIGN-STRICT-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM70-PRESERVESIGN-STRICT-NEXT: shl.b32 %r10, %r9, %r1;
+; SM70-PRESERVESIGN-STRICT-NEXT: and.b32 %r11, %r14, %r2;
+; SM70-PRESERVESIGN-STRICT-NEXT: or.b32 %r12, %r11, %r10;
+; SM70-PRESERVESIGN-STRICT-NEXT: atom.relaxed.sys.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM70-PRESERVESIGN-STRICT-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM70-PRESERVESIGN-STRICT-NEXT: mov.b32 %r14, %r3;
+; SM70-PRESERVESIGN-STRICT-NEXT: @%p1 bra $L__BB4_1;
+; SM70-PRESERVESIGN-STRICT-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-PRESERVESIGN-STRICT-NEXT: shr.u32 %r13, %r3, %r1;
+; SM70-PRESERVESIGN-STRICT-NEXT: st.param::func.b16 [func_retval0], %r13;
+; SM70-PRESERVESIGN-STRICT-NEXT: ret;
+;
+; SM70-PRESERVESIGN-LEGACY-LABEL: fadd_f16_generic(
+; SM70-PRESERVESIGN-LEGACY: {
+; SM70-PRESERVESIGN-LEGACY-NEXT: .reg .b16 %rs<3>;
+; SM70-PRESERVESIGN-LEGACY-NEXT: .reg .b64 %rd<2>;
+; SM70-PRESERVESIGN-LEGACY-EMPTY:
+; SM70-PRESERVESIGN-LEGACY-NEXT: // %bb.0:
+; SM70-PRESERVESIGN-LEGACY-NEXT: ld.param::func.b64 %rd1, [fadd_f16_generic_param_0];
+; SM70-PRESERVESIGN-LEGACY-NEXT: ld.param::func.b16 %rs1, [fadd_f16_generic_param_1];
+; SM70-PRESERVESIGN-LEGACY-NEXT: atom.relaxed.sys.add.noftz.f16 %rs2, [%rd1], %rs1;
+; SM70-PRESERVESIGN-LEGACY-NEXT: st.param::func.b16 [func_retval0], %rs2;
+; SM70-PRESERVESIGN-LEGACY-NEXT: ret;
+;
+; SM70-PRESERVESIGN-FAST-LABEL: fadd_f16_generic(
+; SM70-PRESERVESIGN-FAST: {
+; SM70-PRESERVESIGN-FAST-NEXT: .reg .b16 %rs<3>;
+; SM70-PRESERVESIGN-FAST-NEXT: .reg .b64 %rd<2>;
+; SM70-PRESERVESIGN-FAST-EMPTY:
+; SM70-PRESERVESIGN-FAST-NEXT: // %bb.0:
+; SM70-PRESERVESIGN-FAST-NEXT: ld.param::func.b64 %rd1, [fadd_f16_generic_param_0];
+; SM70-PRESERVESIGN-FAST-NEXT: ld.param::func.b16 %rs1, [fadd_f16_generic_param_1];
+; SM70-PRESERVESIGN-FAST-NEXT: atom.relaxed.sys.add.noftz.f16 %rs2, [%rd1], %rs1;
+; SM70-PRESERVESIGN-FAST-NEXT: st.param::func.b16 [func_retval0], %rs2;
+; SM70-PRESERVESIGN-FAST-NEXT: ret;
+;
+; SM90-IEEE-STRICT-LABEL: fadd_f16_generic(
+; SM90-IEEE-STRICT: {
+; SM90-IEEE-STRICT-NEXT: .reg .b16 %rs<3>;
+; SM90-IEEE-STRICT-NEXT: .reg .b64 %rd<2>;
+; SM90-IEEE-STRICT-EMPTY:
+; SM90-IEEE-STRICT-NEXT: // %bb.0:
+; SM90-IEEE-STRICT-NEXT: ld.param::func.b64 %rd1, [fadd_f16_generic_param_0];
+; SM90-IEEE-STRICT-NEXT: ld.param::func.b16 %rs1, [fadd_f16_generic_param_1];
+; SM90-IEEE-STRICT-NEXT: atom.relaxed.sys.add.noftz.f16 %rs2, [%rd1], %rs1;
+; SM90-IEEE-STRICT-NEXT: st.param::func.b16 [func_retval0], %rs2;
+; SM90-IEEE-STRICT-NEXT: ret;
+;
+; SM90-IEEE-LEGACY-LABEL: fadd_f16_generic(
+; SM90-IEEE-LEGACY: {
+; SM90-IEEE-LEGACY-NEXT: .reg .b16 %rs<3>;
+; SM90-IEEE-LEGACY-NEXT: .reg .b64 %rd<2>;
+; SM90-IEEE-LEGACY-EMPTY:
+; SM90-IEEE-LEGACY-NEXT: // %bb.0:
+; SM90-IEEE-LEGACY-NEXT: ld.param::func.b64 %rd1, [fadd_f16_generic_param_0];
+; SM90-IEEE-LEGACY-NEXT: ld.param::func.b16 %rs1, [fadd_f16_generic_param_1];
+; SM90-IEEE-LEGACY-NEXT: atom.relaxed.sys.add.noftz.f16 %rs2, [%rd1], %rs1;
+; SM90-IEEE-LEGACY-NEXT: st.param::func.b16 [func_retval0], %rs2;
+; SM90-IEEE-LEGACY-NEXT: ret;
+;
+; SM90-IEEE-FAST-LABEL: fadd_f16_generic(
+; SM90-IEEE-FAST: {
+; SM90-IEEE-FAST-NEXT: .reg .b16 %rs<3>;
+; SM90-IEEE-FAST-NEXT: .reg .b64 %rd<2>;
+; SM90-IEEE-FAST-EMPTY:
+; SM90-IEEE-FAST-NEXT: // %bb.0:
+; SM90-IEEE-FAST-NEXT: ld.param::func.b64 %rd1, [fadd_f16_generic_param_0];
+; SM90-IEEE-FAST-NEXT: ld.param::func.b16 %rs1, [fadd_f16_generic_param_1];
+; SM90-IEEE-FAST-NEXT: atom.relaxed.sys.add.noftz.f16 %rs2, [%rd1], %rs1;
+; SM90-IEEE-FAST-NEXT: st.param::func.b16 [func_retval0], %rs2;
+; SM90-IEEE-FAST-NEXT: ret;
+;
+; SM90-PRESERVESIGN-STRICT-LABEL: fadd_f16_generic(
+; SM90-PRESERVESIGN-STRICT: {
+; SM90-PRESERVESIGN-STRICT-NEXT: .reg .pred %p<2>;
+; SM90-PRESERVESIGN-STRICT-NEXT: .reg .b16 %rs<4>;
+; SM90-PRESERVESIGN-STRICT-NEXT: .reg .b32 %r<15>;
+; SM90-PRESERVESIGN-STRICT-NEXT: .reg .b64 %rd<3>;
+; SM90-PRESERVESIGN-STRICT-EMPTY:
+; SM90-PRESERVESIGN-STRICT-NEXT: // %bb.0:
+; SM90-PRESERVESIGN-STRICT-NEXT: ld.param::func.b16 %rs1, [fadd_f16_generic_param_1];
+; SM90-PRESERVESIGN-STRICT-NEXT: ld.param::func.b64 %rd2, [fadd_f16_generic_param_0];
+; SM90-PRESERVESIGN-STRICT-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-PRESERVESIGN-STRICT-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM90-PRESERVESIGN-STRICT-NEXT: and.b32 %r5, %r4, 3;
+; SM90-PRESERVESIGN-STRICT-NEXT: shl.b32 %r1, %r5, 3;
+; SM90-PRESERVESIGN-STRICT-NEXT: mov.b32 %r6, 65535;
+; SM90-PRESERVESIGN-STRICT-NEXT: shl.b32 %r7, %r6, %r1;
+; SM90-PRESERVESIGN-STRICT-NEXT: not.b32 %r2, %r7;
+; SM90-PRESERVESIGN-STRICT-NEXT: ld.relaxed.sys.b32 %r14, [%rd1];
+; SM90-PRESERVESIGN-STRICT-NEXT: $L__BB4_1: // %atomicrmw.start
+; SM90-PRESERVESIGN-STRICT-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-PRESERVESIGN-STRICT-NEXT: shr.u32 %r8, %r14, %r1;
+; SM90-PRESERVESIGN-STRICT-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM90-PRESERVESIGN-STRICT-NEXT: add.rn.ftz.f16 %rs3, %rs2, %rs1;
+; SM90-PRESERVESIGN-STRICT-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM90-PRESERVESIGN-STRICT-NEXT: shl.b32 %r10, %r9, %r1;
+; SM90-PRESERVESIGN-STRICT-NEXT: and.b32 %r11, %r14, %r2;
+; SM90-PRESERVESIGN-STRICT-NEXT: or.b32 %r12, %r11, %r10;
+; SM90-PRESERVESIGN-STRICT-NEXT: atom.relaxed.sys.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM90-PRESERVESIGN-STRICT-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM90-PRESERVESIGN-STRICT-NEXT: mov.b32 %r14, %r3;
+; SM90-PRESERVESIGN-STRICT-NEXT: @%p1 bra $L__BB4_1;
+; SM90-PRESERVESIGN-STRICT-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-PRESERVESIGN-STRICT-NEXT: shr.u32 %r13, %r3, %r1;
+; SM90-PRESERVESIGN-STRICT-NEXT: st.param::func.b16 [func_retval0], %r13;
+; SM90-PRESERVESIGN-STRICT-NEXT: ret;
+;
+; SM90-PRESERVESIGN-LEGACY-LABEL: fadd_f16_generic(
+; SM90-PRESERVESIGN-LEGACY: {
+; SM90-PRESERVESIGN-LEGACY-NEXT: .reg .b16 %rs<3>;
+; SM90-PRESERVESIGN-LEGACY-NEXT: .reg .b64 %rd<2>;
+; SM90-PRESERVESIGN-LEGACY-EMPTY:
+; SM90-PRESERVESIGN-LEGACY-NEXT: // %bb.0:
+; SM90-PRESERVESIGN-LEGACY-NEXT: ld.param::func.b64 %rd1, [fadd_f16_generic_param_0];
+; SM90-PRESERVESIGN-LEGACY-NEXT: ld.param::func.b16 %rs1, [fadd_f16_generic_param_1];
+; SM90-PRESERVESIGN-LEGACY-NEXT: atom.relaxed.sys.add.noftz.f16 %rs2, [%rd1], %rs1;
+; SM90-PRESERVESIGN-LEGACY-NEXT: st.param::func.b16 [func_retval0], %rs2;
+; SM90-PRESERVESIGN-LEGACY-NEXT: ret;
+;
+; SM90-PRESERVESIGN-FAST-LABEL: fadd_f16_generic(
+; SM90-PRESERVESIGN-FAST: {
+; SM90-PRESERVESIGN-FAST-NEXT: .reg .b16 %rs<3>;
+; SM90-PRESERVESIGN-FAST-NEXT: .reg .b64 %rd<2>;
+; SM90-PRESERVESIGN-FAST-EMPTY:
+; SM90-PRESERVESIGN-FAST-NEXT: // %bb.0:
+; SM90-PRESERVESIGN-FAST-NEXT: ld.param::func.b64 %rd1, [fadd_f16_generic_param_0];
+; SM90-PRESERVESIGN-FAST-NEXT: ld.param::func.b16 %rs1, [fadd_f16_generic_param_1];
+; SM90-PRESERVESIGN-FAST-NEXT: atom.relaxed.sys.add.noftz.f16 %rs2, [%rd1], %rs1;
+; SM90-PRESERVESIGN-FAST-NEXT: st.param::func.b16 [func_retval0], %rs2;
+; SM90-PRESERVESIGN-FAST-NEXT: ret;
+ %r = atomicrmw fadd ptr %addr, half %val monotonic
+ ret half %r
+}
+
+define half @fadd_f16_shared(ptr addrspace(3) %addr, half %val) {
+; SM70-IEEE-STRICT-LABEL: fadd_f16_shared(
+; SM70-IEEE-STRICT: {
+; SM70-IEEE-STRICT-NEXT: .reg .b16 %rs<3>;
+; SM70-IEEE-STRICT-NEXT: .reg .b64 %rd<2>;
+; SM70-IEEE-STRICT-EMPTY:
+; SM70-IEEE-STRICT-NEXT: // %bb.0:
+; SM70-IEEE-STRICT-NEXT: ld.param::func.b64 %rd1, [fadd_f16_shared_param_0];
+; SM70-IEEE-STRICT-NEXT: ld.param::func.b16 %rs1, [fadd_f16_shared_param_1];
+; SM70-IEEE-STRICT-NEXT: atom.relaxed.sys.shared.add.noftz.f16 %rs2, [%rd1], %rs1;
+; SM70-IEEE-STRICT-NEXT: st.param::func.b16 [func_retval0], %rs2;
+; SM70-IEEE-STRICT-NEXT: ret;
+;
+; SM70-IEEE-LEGACY-LABEL: fadd_f16_shared(
+; SM70-IEEE-LEGACY: {
+; SM70-IEEE-LEGACY-NEXT: .reg .b16 %rs<3>;
+; SM70-IEEE-LEGACY-NEXT: .reg .b64 %rd<2>;
+; SM70-IEEE-LEGACY-EMPTY:
+; SM70-IEEE-LEGACY-NEXT: // %bb.0:
+; SM70-IEEE-LEGACY-NEXT: ld.param::func.b64 %rd1, [fadd_f16_shared_param_0];
+; SM70-IEEE-LEGACY-NEXT: ld.param::func.b16 %rs1, [fadd_f16_shared_param_1];
+; SM70-IEEE-LEGACY-NEXT: atom.relaxed.sys.shared.add.noftz.f16 %rs2, [%rd1], %rs1;
+; SM70-IEEE-LEGACY-NEXT: st.param::func.b16 [func_retval0], %rs2;
+; SM70-IEEE-LEGACY-NEXT: ret;
+;
+; SM70-IEEE-FAST-LABEL: fadd_f16_shared(
+; SM70-IEEE-FAST: {
+; SM70-IEEE-FAST-NEXT: .reg .b16 %rs<3>;
+; SM70-IEEE-FAST-NEXT: .reg .b64 %rd<2>;
+; SM70-IEEE-FAST-EMPTY:
+; SM70-IEEE-FAST-NEXT: // %bb.0:
+; SM70-IEEE-FAST-NEXT: ld.param::func.b64 %rd1, [fadd_f16_shared_param_0];
+; SM70-IEEE-FAST-NEXT: ld.param::func.b16 %rs1, [fadd_f16_shared_param_1];
+; SM70-IEEE-FAST-NEXT: atom.relaxed.sys.shared.add.noftz.f16 %rs2, [%rd1], %rs1;
+; SM70-IEEE-FAST-NEXT: st.param::func.b16 [func_retval0], %rs2;
+; SM70-IEEE-FAST-NEXT: ret;
+;
+; SM70-PRESERVESIGN-STRICT-LABEL: fadd_f16_shared(
+; SM70-PRESERVESIGN-STRICT: {
+; SM70-PRESERVESIGN-STRICT-NEXT: .reg .pred %p<2>;
+; SM70-PRESERVESIGN-STRICT-NEXT: .reg .b16 %rs<4>;
+; SM70-PRESERVESIGN-STRICT-NEXT: .reg .b32 %r<15>;
+; SM70-PRESERVESIGN-STRICT-NEXT: .reg .b64 %rd<3>;
+; SM70-PRESERVESIGN-STRICT-EMPTY:
+; SM70-PRESERVESIGN-STRICT-NEXT: // %bb.0:
+; SM70-PRESERVESIGN-STRICT-NEXT: ld.param::func.b16 %rs1, [fadd_f16_shared_param_1];
+; SM70-PRESERVESIGN-STRICT-NEXT: ld.param::func.b64 %rd2, [fadd_f16_shared_param_0];
+; SM70-PRESERVESIGN-STRICT-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-PRESERVESIGN-STRICT-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-PRESERVESIGN-STRICT-NEXT: and.b32 %r5, %r4, 3;
+; SM70-PRESERVESIGN-STRICT-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-PRESERVESIGN-STRICT-NEXT: mov.b32 %r6, 65535;
+; SM70-PRESERVESIGN-STRICT-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-PRESERVESIGN-STRICT-NEXT: not.b32 %r2, %r7;
+; SM70-PRESERVESIGN-STRICT-NEXT: ld.relaxed.sys.shared.b32 %r14, [%rd1];
+; SM70-PRESERVESIGN-STRICT-NEXT: $L__BB5_1: // %atomicrmw.start
+; SM70-PRESERVESIGN-STRICT-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-PRESERVESIGN-STRICT-NEXT: shr.u32 %r8, %r14, %r1;
+; SM70-PRESERVESIGN-STRICT-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM70-PRESERVESIGN-STRICT-NEXT: add.rn.ftz.f16 %rs3, %rs2, %rs1;
+; SM70-PRESERVESIGN-STRICT-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM70-PRESERVESIGN-STRICT-NEXT: shl.b32 %r10, %r9, %r1;
+; SM70-PRESERVESIGN-STRICT-NEXT: and.b32 %r11, %r14, %r2;
+; SM70-PRESERVESIGN-STRICT-NEXT: or.b32 %r12, %r11, %r10;
+; SM70-PRESERVESIGN-STRICT-NEXT: atom.relaxed.sys.shared.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM70-PRESERVESIGN-STRICT-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM70-PRESERVESIGN-STRICT-NEXT: mov.b32 %r14, %r3;
+; SM70-PRESERVESIGN-STRICT-NEXT: @%p1 bra $L__BB5_1;
+; SM70-PRESERVESIGN-STRICT-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-PRESERVESIGN-STRICT-NEXT: shr.u32 %r13, %r3, %r1;
+; SM70-PRESERVESIGN-STRICT-NEXT: st.param::func.b16 [func_retval0], %r13;
+; SM70-PRESERVESIGN-STRICT-NEXT: ret;
+;
+; SM70-PRESERVESIGN-LEGACY-LABEL: fadd_f16_shared(
+; SM70-PRESERVESIGN-LEGACY: {
+; SM70-PRESERVESIGN-LEGACY-NEXT: .reg .b16 %rs<3>;
+; SM70-PRESERVESIGN-LEGACY-NEXT: .reg .b64 %rd<2>;
+; SM70-PRESERVESIGN-LEGACY-EMPTY:
+; SM70-PRESERVESIGN-LEGACY-NEXT: // %bb.0:
+; SM70-PRESERVESIGN-LEGACY-NEXT: ld.param::func.b64 %rd1, [fadd_f16_shared_param_0];
+; SM70-PRESERVESIGN-LEGACY-NEXT: ld.param::func.b16 %rs1, [fadd_f16_shared_param_1];
+; SM70-PRESERVESIGN-LEGACY-NEXT: atom.relaxed.sys.shared.add.noftz.f16 %rs2, [%rd1], %rs1;
+; SM70-PRESERVESIGN-LEGACY-NEXT: st.param::func.b16 [func_retval0], %rs2;
+; SM70-PRESERVESIGN-LEGACY-NEXT: ret;
+;
+; SM70-PRESERVESIGN-FAST-LABEL: fadd_f16_shared(
+; SM70-PRESERVESIGN-FAST: {
+; SM70-PRESERVESIGN-FAST-NEXT: .reg .b16 %rs<3>;
+; SM70-PRESERVESIGN-FAST-NEXT: .reg .b64 %rd<2>;
+; SM70-PRESERVESIGN-FAST-EMPTY:
+; SM70-PRESERVESIGN-FAST-NEXT: // %bb.0:
+; SM70-PRESERVESIGN-FAST-NEXT: ld.param::func.b64 %rd1, [fadd_f16_shared_param_0];
+; SM70-PRESERVESIGN-FAST-NEXT: ld.param::func.b16 %rs1, [fadd_f16_shared_param_1];
+; SM70-PRESERVESIGN-FAST-NEXT: atom.relaxed.sys.shared.add.noftz.f16 %rs2, [%rd1], %rs1;
+; SM70-PRESERVESIGN-FAST-NEXT: st.param::func.b16 [func_retval0], %rs2;
+; SM70-PRESERVESIGN-FAST-NEXT: ret;
+;
+; SM90-IEEE-STRICT-LABEL: fadd_f16_shared(
+; SM90-IEEE-STRICT: {
+; SM90-IEEE-STRICT-NEXT: .reg .b16 %rs<3>;
+; SM90-IEEE-STRICT-NEXT: .reg .b64 %rd<2>;
+; SM90-IEEE-STRICT-EMPTY:
+; SM90-IEEE-STRICT-NEXT: // %bb.0:
+; SM90-IEEE-STRICT-NEXT: ld.param::func.b64 %rd1, [fadd_f16_shared_param_0];
+; SM90-IEEE-STRICT-NEXT: ld.param::func.b16 %rs1, [fadd_f16_shared_param_1];
+; SM90-IEEE-STRICT-NEXT: atom.relaxed.sys.shared.add.noftz.f16 %rs2, [%rd1], %rs1;
+; SM90-IEEE-STRICT-NEXT: st.param::func.b16 [func_retval0], %rs2;
+; SM90-IEEE-STRICT-NEXT: ret;
+;
+; SM90-IEEE-LEGACY-LABEL: fadd_f16_shared(
+; SM90-IEEE-LEGACY: {
+; SM90-IEEE-LEGACY-NEXT: .reg .b16 %rs<3>;
+; SM90-IEEE-LEGACY-NEXT: .reg .b64 %rd<2>;
+; SM90-IEEE-LEGACY-EMPTY:
+; SM90-IEEE-LEGACY-NEXT: // %bb.0:
+; SM90-IEEE-LEGACY-NEXT: ld.param::func.b64 %rd1, [fadd_f16_shared_param_0];
+; SM90-IEEE-LEGACY-NEXT: ld.param::func.b16 %rs1, [fadd_f16_shared_param_1];
+; SM90-IEEE-LEGACY-NEXT: atom.relaxed.sys.shared.add.noftz.f16 %rs2, [%rd1], %rs1;
+; SM90-IEEE-LEGACY-NEXT: st.param::func.b16 [func_retval0], %rs2;
+; SM90-IEEE-LEGACY-NEXT: ret;
+;
+; SM90-IEEE-FAST-LABEL: fadd_f16_shared(
+; SM90-IEEE-FAST: {
+; SM90-IEEE-FAST-NEXT: .reg .b16 %rs<3>;
+; SM90-IEEE-FAST-NEXT: .reg .b64 %rd<2>;
+; SM90-IEEE-FAST-EMPTY:
+; SM90-IEEE-FAST-NEXT: // %bb.0:
+; SM90-IEEE-FAST-NEXT: ld.param::func.b64 %rd1, [fadd_f16_shared_param_0];
+; SM90-IEEE-FAST-NEXT: ld.param::func.b16 %rs1, [fadd_f16_shared_param_1];
+; SM90-IEEE-FAST-NEXT: atom.relaxed.sys.shared.add.noftz.f16 %rs2, [%rd1], %rs1;
+; SM90-IEEE-FAST-NEXT: st.param::func.b16 [func_retval0], %rs2;
+; SM90-IEEE-FAST-NEXT: ret;
+;
+; SM90-PRESERVESIGN-STRICT-LABEL: fadd_f16_shared(
+; SM90-PRESERVESIGN-STRICT: {
+; SM90-PRESERVESIGN-STRICT-NEXT: .reg .pred %p<2>;
+; SM90-PRESERVESIGN-STRICT-NEXT: .reg .b16 %rs<4>;
+; SM90-PRESERVESIGN-STRICT-NEXT: .reg .b32 %r<15>;
+; SM90-PRESERVESIGN-STRICT-NEXT: .reg .b64 %rd<3>;
+; SM90-PRESERVESIGN-STRICT-EMPTY:
+; SM90-PRESERVESIGN-STRICT-NEXT: // %bb.0:
+; SM90-PRESERVESIGN-STRICT-NEXT: ld.param::func.b16 %rs1, [fadd_f16_shared_param_1];
+; SM90-PRESERVESIGN-STRICT-NEXT: ld.param::func.b64 %rd2, [fadd_f16_shared_param_0];
+; SM90-PRESERVESIGN-STRICT-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-PRESERVESIGN-STRICT-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM90-PRESERVESIGN-STRICT-NEXT: and.b32 %r5, %r4, 3;
+; SM90-PRESERVESIGN-STRICT-NEXT: shl.b32 %r1, %r5, 3;
+; SM90-PRESERVESIGN-STRICT-NEXT: mov.b32 %r6, 65535;
+; SM90-PRESERVESIGN-STRICT-NEXT: shl.b32 %r7, %r6, %r1;
+; SM90-PRESERVESIGN-STRICT-NEXT: not.b32 %r2, %r7;
+; SM90-PRESERVESIGN-STRICT-NEXT: ld.relaxed.sys.shared.b32 %r14, [%rd1];
+; SM90-PRESERVESIGN-STRICT-NEXT: $L__BB5_1: // %atomicrmw.start
+; SM90-PRESERVESIGN-STRICT-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-PRESERVESIGN-STRICT-NEXT: shr.u32 %r8, %r14, %r1;
+; SM90-PRESERVESIGN-STRICT-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM90-PRESERVESIGN-STRICT-NEXT: add.rn.ftz.f16 %rs3, %rs2, %rs1;
+; SM90-PRESERVESIGN-STRICT-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM90-PRESERVESIGN-STRICT-NEXT: shl.b32 %r10, %r9, %r1;
+; SM90-PRESERVESIGN-STRICT-NEXT: and.b32 %r11, %r14, %r2;
+; SM90-PRESERVESIGN-STRICT-NEXT: or.b32 %r12, %r11, %r10;
+; SM90-PRESERVESIGN-STRICT-NEXT: atom.relaxed.sys.shared.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM90-PRESERVESIGN-STRICT-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM90-PRESERVESIGN-STRICT-NEXT: mov.b32 %r14, %r3;
+; SM90-PRESERVESIGN-STRICT-NEXT: @%p1 bra $L__BB5_1;
+; SM90-PRESERVESIGN-STRICT-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-PRESERVESIGN-STRICT-NEXT: shr.u32 %r13, %r3, %r1;
+; SM90-PRESERVESIGN-STRICT-NEXT: st.param::func.b16 [func_retval0], %r13;
+; SM90-PRESERVESIGN-STRICT-NEXT: ret;
+;
+; SM90-PRESERVESIGN-LEGACY-LABEL: fadd_f16_shared(
+; SM90-PRESERVESIGN-LEGACY: {
+; SM90-PRESERVESIGN-LEGACY-NEXT: .reg .b16 %rs<3>;
+; SM90-PRESERVESIGN-LEGACY-NEXT: .reg .b64 %rd<2>;
+; SM90-PRESERVESIGN-LEGACY-EMPTY:
+; SM90-PRESERVESIGN-LEGACY-NEXT: // %bb.0:
+; SM90-PRESERVESIGN-LEGACY-NEXT: ld.param::func.b64 %rd1, [fadd_f16_shared_param_0];
+; SM90-PRESERVESIGN-LEGACY-NEXT: ld.param::func.b16 %rs1, [fadd_f16_shared_param_1];
+; SM90-PRESERVESIGN-LEGACY-NEXT: atom.relaxed.sys.shared.add.noftz.f16 %rs2, [%rd1], %rs1;
+; SM90-PRESERVESIGN-LEGACY-NEXT: st.param::func.b16 [func_retval0], %rs2;
+; SM90-PRESERVESIGN-LEGACY-NEXT: ret;
+;
+; SM90-PRESERVESIGN-FAST-LABEL: fadd_f16_shared(
+; SM90-PRESERVESIGN-FAST: {
+; SM90-PRESERVESIGN-FAST-NEXT: .reg .b16 %rs<3>;
+; SM90-PRESERVESIGN-FAST-NEXT: .reg .b64 %rd<2>;
+; SM90-PRESERVESIGN-FAST-EMPTY:
+; SM90-PRESERVESIGN-FAST-NEXT: // %bb.0:
+; SM90-PRESERVESIGN-FAST-NEXT: ld.param::func.b64 %rd1, [fadd_f16_shared_param_0];
+; SM90-PRESERVESIGN-FAST-NEXT: ld.param::func.b16 %rs1, [fadd_f16_shared_param_1];
+; SM90-PRESERVESIGN-FAST-NEXT: atom.relaxed.sys.shared.add.noftz.f16 %rs2, [%rd1], %rs1;
+; SM90-PRESERVESIGN-FAST-NEXT: st.param::func.b16 [func_retval0], %rs2;
+; SM90-PRESERVESIGN-FAST-NEXT: ret;
+ %r = atomicrmw fadd ptr addrspace(3) %addr, half %val monotonic
+ ret half %r
+}
+
+; bf16: PTX has no flushing bf16 add at all, so a CAS loop would behave
+; identically to the native atomic and we always take the atomic.
+define bfloat @fadd_bf16_global(ptr addrspace(1) %addr, bfloat %val) {
+; SM70-IEEE-STRICT-LABEL: fadd_bf16_global(
+; SM70-IEEE-STRICT: {
+; SM70-IEEE-STRICT-NEXT: .reg .pred %p<3>;
+; SM70-IEEE-STRICT-NEXT: .reg .b16 %rs<2>;
+; SM70-IEEE-STRICT-NEXT: .reg .b32 %r<24>;
+; SM70-IEEE-STRICT-NEXT: .reg .b64 %rd<3>;
+; SM70-IEEE-STRICT-EMPTY:
+; SM70-IEEE-STRICT-NEXT: // %bb.0:
+; SM70-IEEE-STRICT-NEXT: ld.param::func.b16 %rs1, [fadd_bf16_global_param_1];
+; SM70-IEEE-STRICT-NEXT: ld.param::func.b64 %rd2, [fadd_bf16_global_param_0];
+; SM70-IEEE-STRICT-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-IEEE-STRICT-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-IEEE-STRICT-NEXT: and.b32 %r5, %r4, 3;
+; SM70-IEEE-STRICT-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-IEEE-STRICT-NEXT: mov.b32 %r6, 65535;
+; SM70-IEEE-STRICT-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-IEEE-STRICT-NEXT: not.b32 %r2, %r7;
+; SM70-IEEE-STRICT-NEXT: ld.relaxed.sys.global.b32 %r23, [%rd1];
+; SM70-IEEE-STRICT-NEXT: cvt.u32.u16 %r10, %rs1;
+; SM70-IEEE-STRICT-NEXT: shl.b32 %r11, %r10, 16;
+; SM70-IEEE-STRICT-NEXT: $L__BB6_1: // %atomicrmw.start
+; SM70-IEEE-STRICT-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-IEEE-STRICT-NEXT: shr.u32 %r8, %r23, %r1;
+; SM70-IEEE-STRICT-NEXT: shl.b32 %r9, %r8, 16;
+; SM70-IEEE-STRICT-NEXT: add.rn.f32 %r12, %r9, %r11;
+; SM70-IEEE-STRICT-NEXT: bfe.u32 %r13, %r12, 16, 1;
+; SM70-IEEE-STRICT-NEXT: add.s32 %r14, %r13, %r12;
+; SM70-IEEE-STRICT-NEXT: add.s32 %r15, %r14, 32767;
+; SM70-IEEE-STRICT-NEXT: setp.nan.f32 %p1, %r12, %r12;
+; SM70-IEEE-STRICT-NEXT: or.b32 %r16, %r12, 4194304;
+; SM70-IEEE-STRICT-NEXT: selp.b32 %r17, %r16, %r15, %p1;
+; SM70-IEEE-STRICT-NEXT: shr.u32 %r18, %r17, 16;
+; SM70-IEEE-STRICT-NEXT: shl.b32 %r19, %r18, %r1;
+; SM70-IEEE-STRICT-NEXT: and.b32 %r20, %r23, %r2;
+; SM70-IEEE-STRICT-NEXT: or.b32 %r21, %r20, %r19;
+; SM70-IEEE-STRICT-NEXT: atom.relaxed.sys.global.cas.b32 %r3, [%rd1], %r23, %r21;
+; SM70-IEEE-STRICT-NEXT: setp.ne.b32 %p2, %r3, %r23;
+; SM70-IEEE-STRICT-NEXT: mov.b32 %r23, %r3;
+; SM70-IEEE-STRICT-NEXT: @%p2 bra $L__BB6_1;
+; SM70-IEEE-STRICT-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-IEEE-STRICT-NEXT: shr.u32 %r22, %r3, %r1;
+; SM70-IEEE-STRICT-NEXT: st.param::func.b16 [func_retval0], %r22;
+; SM70-IEEE-STRICT-NEXT: ret;
+;
+; SM70-IEEE-LEGACY-LABEL: fadd_bf16_global(
+; SM70-IEEE-LEGACY: {
+; SM70-IEEE-LEGACY-NEXT: .reg .pred %p<3>;
+; SM70-IEEE-LEGACY-NEXT: .reg .b16 %rs<2>;
+; SM70-IEEE-LEGACY-NEXT: .reg .b32 %r<24>;
+; SM70-IEEE-LEGACY-NEXT: .reg .b64 %rd<3>;
+; SM70-IEEE-LEGACY-EMPTY:
+; SM70-IEEE-LEGACY-NEXT: // %bb.0:
+; SM70-IEEE-LEGACY-NEXT: ld.param::func.b16 %rs1, [fadd_bf16_global_param_1];
+; SM70-IEEE-LEGACY-NEXT: ld.param::func.b64 %rd2, [fadd_bf16_global_param_0];
+; SM70-IEEE-LEGACY-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-IEEE-LEGACY-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-IEEE-LEGACY-NEXT: and.b32 %r5, %r4, 3;
+; SM70-IEEE-LEGACY-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-IEEE-LEGACY-NEXT: mov.b32 %r6, 65535;
+; SM70-IEEE-LEGACY-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-IEEE-LEGACY-NEXT: not.b32 %r2, %r7;
+; SM70-IEEE-LEGACY-NEXT: ld.relaxed.sys.global.b32 %r23, [%rd1];
+; SM70-IEEE-LEGACY-NEXT: cvt.u32.u16 %r10, %rs1;
+; SM70-IEEE-LEGACY-NEXT: shl.b32 %r11, %r10, 16;
+; SM70-IEEE-LEGACY-NEXT: $L__BB6_1: // %atomicrmw.start
+; SM70-IEEE-LEGACY-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-IEEE-LEGACY-NEXT: shr.u32 %r8, %r23, %r1;
+; SM70-IEEE-LEGACY-NEXT: shl.b32 %r9, %r8, 16;
+; SM70-IEEE-LEGACY-NEXT: add.rn.f32 %r12, %r9, %r11;
+; SM70-IEEE-LEGACY-NEXT: bfe.u32 %r13, %r12, 16, 1;
+; SM70-IEEE-LEGACY-NEXT: add.s32 %r14, %r13, %r12;
+; SM70-IEEE-LEGACY-NEXT: add.s32 %r15, %r14, 32767;
+; SM70-IEEE-LEGACY-NEXT: setp.nan.f32 %p1, %r12, %r12;
+; SM70-IEEE-LEGACY-NEXT: or.b32 %r16, %r12, 4194304;
+; SM70-IEEE-LEGACY-NEXT: selp.b32 %r17, %r16, %r15, %p1;
+; SM70-IEEE-LEGACY-NEXT: shr.u32 %r18, %r17, 16;
+; SM70-IEEE-LEGACY-NEXT: shl.b32 %r19, %r18, %r1;
+; SM70-IEEE-LEGACY-NEXT: and.b32 %r20, %r23, %r2;
+; SM70-IEEE-LEGACY-NEXT: or.b32 %r21, %r20, %r19;
+; SM70-IEEE-LEGACY-NEXT: atom.relaxed.sys.global.cas.b32 %r3, [%rd1], %r23, %r21;
+; SM70-IEEE-LEGACY-NEXT: setp.ne.b32 %p2, %r3, %r23;
+; SM70-IEEE-LEGACY-NEXT: mov.b32 %r23, %r3;
+; SM70-IEEE-LEGACY-NEXT: @%p2 bra $L__BB6_1;
+; SM70-IEEE-LEGACY-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-IEEE-LEGACY-NEXT: shr.u32 %r22, %r3, %r1;
+; SM70-IEEE-LEGACY-NEXT: st.param::func.b16 [func_retval0], %r22;
+; SM70-IEEE-LEGACY-NEXT: ret;
+;
+; SM70-IEEE-FAST-LABEL: fadd_bf16_global(
+; SM70-IEEE-FAST: {
+; SM70-IEEE-FAST-NEXT: .reg .pred %p<3>;
+; SM70-IEEE-FAST-NEXT: .reg .b16 %rs<2>;
+; SM70-IEEE-FAST-NEXT: .reg .b32 %r<24>;
+; SM70-IEEE-FAST-NEXT: .reg .b64 %rd<3>;
+; SM70-IEEE-FAST-EMPTY:
+; SM70-IEEE-FAST-NEXT: // %bb.0:
+; SM70-IEEE-FAST-NEXT: ld.param::func.b16 %rs1, [fadd_bf16_global_param_1];
+; SM70-IEEE-FAST-NEXT: ld.param::func.b64 %rd2, [fadd_bf16_global_param_0];
+; SM70-IEEE-FAST-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-IEEE-FAST-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-IEEE-FAST-NEXT: and.b32 %r5, %r4, 3;
+; SM70-IEEE-FAST-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-IEEE-FAST-NEXT: mov.b32 %r6, 65535;
+; SM70-IEEE-FAST-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-IEEE-FAST-NEXT: not.b32 %r2, %r7;
+; SM70-IEEE-FAST-NEXT: ld.relaxed.sys.global.b32 %r23, [%rd1];
+; SM70-IEEE-FAST-NEXT: cvt.u32.u16 %r10, %rs1;
+; SM70-IEEE-FAST-NEXT: shl.b32 %r11, %r10, 16;
+; SM70-IEEE-FAST-NEXT: $L__BB6_1: // %atomicrmw.start
+; SM70-IEEE-FAST-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-IEEE-FAST-NEXT: shr.u32 %r8, %r23, %r1;
+; SM70-IEEE-FAST-NEXT: shl.b32 %r9, %r8, 16;
+; SM70-IEEE-FAST-NEXT: add.rn.f32 %r12, %r9, %r11;
+; SM70-IEEE-FAST-NEXT: bfe.u32 %r13, %r12, 16, 1;
+; SM70-IEEE-FAST-NEXT: add.s32 %r14, %r13, %r12;
+; SM70-IEEE-FAST-NEXT: add.s32 %r15, %r14, 32767;
+; SM70-IEEE-FAST-NEXT: setp.nan.f32 %p1, %r12, %r12;
+; SM70-IEEE-FAST-NEXT: or.b32 %r16, %r12, 4194304;
+; SM70-IEEE-FAST-NEXT: selp.b32 %r17, %r16, %r15, %p1;
+; SM70-IEEE-FAST-NEXT: shr.u32 %r18, %r17, 16;
+; SM70-IEEE-FAST-NEXT: shl.b32 %r19, %r18, %r1;
+; SM70-IEEE-FAST-NEXT: and.b32 %r20, %r23, %r2;
+; SM70-IEEE-FAST-NEXT: or.b32 %r21, %r20, %r19;
+; SM70-IEEE-FAST-NEXT: atom.relaxed.sys.global.cas.b32 %r3, [%rd1], %r23, %r21;
+; SM70-IEEE-FAST-NEXT: setp.ne.b32 %p2, %r3, %r23;
+; SM70-IEEE-FAST-NEXT: mov.b32 %r23, %r3;
+; SM70-IEEE-FAST-NEXT: @%p2 bra $L__BB6_1;
+; SM70-IEEE-FAST-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-IEEE-FAST-NEXT: shr.u32 %r22, %r3, %r1;
+; SM70-IEEE-FAST-NEXT: st.param::func.b16 [func_retval0], %r22;
+; SM70-IEEE-FAST-NEXT: ret;
+;
+; SM70-PRESERVESIGN-STRICT-LABEL: fadd_bf16_global(
+; SM70-PRESERVESIGN-STRICT: {
+; SM70-PRESERVESIGN-STRICT-NEXT: .reg .pred %p<3>;
+; SM70-PRESERVESIGN-STRICT-NEXT: .reg .b16 %rs<2>;
+; SM70-PRESERVESIGN-STRICT-NEXT: .reg .b32 %r<24>;
+; SM70-PRESERVESIGN-STRICT-NEXT: .reg .b64 %rd<3>;
+; SM70-PRESERVESIGN-STRICT-EMPTY:
+; SM70-PRESERVESIGN-STRICT-NEXT: // %bb.0:
+; SM70-PRESERVESIGN-STRICT-NEXT: ld.param::func.b16 %rs1, [fadd_bf16_global_param_1];
+; SM70-PRESERVESIGN-STRICT-NEXT: ld.param::func.b64 %rd2, [fadd_bf16_global_param_0];
+; SM70-PRESERVESIGN-STRICT-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-PRESERVESIGN-STRICT-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-PRESERVESIGN-STRICT-NEXT: and.b32 %r5, %r4, 3;
+; SM70-PRESERVESIGN-STRICT-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-PRESERVESIGN-STRICT-NEXT: mov.b32 %r6, 65535;
+; SM70-PRESERVESIGN-STRICT-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-PRESERVESIGN-STRICT-NEXT: not.b32 %r2, %r7;
+; SM70-PRESERVESIGN-STRICT-NEXT: ld.relaxed.sys.global.b32 %r23, [%rd1];
+; SM70-PRESERVESIGN-STRICT-NEXT: cvt.u32.u16 %r10, %rs1;
+; SM70-PRESERVESIGN-STRICT-NEXT: shl.b32 %r11, %r10, 16;
+; SM70-PRESERVESIGN-STRICT-NEXT: $L__BB6_1: // %atomicrmw.start
+; SM70-PRESERVESIGN-STRICT-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-PRESERVESIGN-STRICT-NEXT: shr.u32 %r8, %r23, %r1;
+; SM70-PRESERVESIGN-STRICT-NEXT: shl.b32 %r9, %r8, 16;
+; SM70-PRESERVESIGN-STRICT-NEXT: add.rn.ftz.f32 %r12, %r9, %r11;
+; SM70-PRESERVESIGN-STRICT-NEXT: bfe.u32 %r13, %r12, 16, 1;
+; SM70-PRESERVESIGN-STRICT-NEXT: add.s32 %r14, %r13, %r12;
+; SM70-PRESERVESIGN-STRICT-NEXT: add.s32 %r15, %r14, 32767;
+; SM70-PRESERVESIGN-STRICT-NEXT: setp.nan.ftz.f32 %p1, %r12, %r12;
+; SM70-PRESERVESIGN-STRICT-NEXT: or.b32 %r16, %r12, 4194304;
+; SM70-PRESERVESIGN-STRICT-NEXT: selp.b32 %r17, %r16, %r15, %p1;
+; SM70-PRESERVESIGN-STRICT-NEXT: shr.u32 %r18, %r17, 16;
+; SM70-PRESERVESIGN-STRICT-NEXT: shl.b32 %r19, %r18, %r1;
+; SM70-PRESERVESIGN-STRICT-NEXT: and.b32 %r20, %r23, %r2;
+; SM70-PRESERVESIGN-STRICT-NEXT: or.b32 %r21, %r20, %r19;
+; SM70-PRESERVESIGN-STRICT-NEXT: atom.relaxed.sys.global.cas.b32 %r3, [%rd1], %r23, %r21;
+; SM70-PRESERVESIGN-STRICT-NEXT: setp.ne.b32 %p2, %r3, %r23;
+; SM70-PRESERVESIGN-STRICT-NEXT: mov.b32 %r23, %r3;
+; SM70-PRESERVESIGN-STRICT-NEXT: @%p2 bra $L__BB6_1;
+; SM70-PRESERVESIGN-STRICT-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-PRESERVESIGN-STRICT-NEXT: shr.u32 %r22, %r3, %r1;
+; SM70-PRESERVESIGN-STRICT-NEXT: st.param::func.b16 [func_retval0], %r22;
+; SM70-PRESERVESIGN-STRICT-NEXT: ret;
+;
+; SM70-PRESERVESIGN-LEGACY-LABEL: fadd_bf16_global(
+; SM70-PRESERVESIGN-LEGACY: {
+; SM70-PRESERVESIGN-LEGACY-NEXT: .reg .pred %p<3>;
+; SM70-PRESERVESIGN-LEGACY-NEXT: .reg .b16 %rs<2>;
+; SM70-PRESERVESIGN-LEGACY-NEXT: .reg .b32 %r<24>;
+; SM70-PRESERVESIGN-LEGACY-NEXT: .reg .b64 %rd<3>;
+; SM70-PRESERVESIGN-LEGACY-EMPTY:
+; SM70-PRESERVESIGN-LEGACY-NEXT: // %bb.0:
+; SM70-PRESERVESIGN-LEGACY-NEXT: ld.param::func.b16 %rs1, [fadd_bf16_global_param_1];
+; SM70-PRESERVESIGN-LEGACY-NEXT: ld.param::func.b64 %rd2, [fadd_bf16_global_param_0];
+; SM70-PRESERVESIGN-LEGACY-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-PRESERVESIGN-LEGACY-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-PRESERVESIGN-LEGACY-NEXT: and.b32 %r5, %r4, 3;
+; SM70-PRESERVESIGN-LEGACY-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-PRESERVESIGN-LEGACY-NEXT: mov.b32 %r6, 65535;
+; SM70-PRESERVESIGN-LEGACY-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-PRESERVESIGN-LEGACY-NEXT: not.b32 %r2, %r7;
+; SM70-PRESERVESIGN-LEGACY-NEXT: ld.relaxed.sys.global.b32 %r23, [%rd1];
+; SM70-PRESERVESIGN-LEGACY-NEXT: cvt.u32.u16 %r10, %rs1;
+; SM70-PRESERVESIGN-LEGACY-NEXT: shl.b32 %r11, %r10, 16;
+; SM70-PRESERVESIGN-LEGACY-NEXT: $L__BB6_1: // %atomicrmw.start
+; SM70-PRESERVESIGN-LEGACY-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-PRESERVESIGN-LEGACY-NEXT: shr.u32 %r8, %r23, %r1;
+; SM70-PRESERVESIGN-LEGACY-NEXT: shl.b32 %r9, %r8, 16;
+; SM70-PRESERVESIGN-LEGACY-NEXT: add.rn.ftz.f32 %r12, %r9, %r11;
+; SM70-PRESERVESIGN-LEGACY-NEXT: bfe.u32 %r13, %r12, 16, 1;
+; SM70-PRESERVESIGN-LEGACY-NEXT: add.s32 %r14, %r13, %r12;
+; SM70-PRESERVESIGN-LEGACY-NEXT: add.s32 %r15, %r14, 32767;
+; SM70-PRESERVESIGN-LEGACY-NEXT: setp.nan.ftz.f32 %p1, %r12, %r12;
+; SM70-PRESERVESIGN-LEGACY-NEXT: or.b32 %r16, %r12, 4194304;
+; SM70-PRESERVESIGN-LEGACY-NEXT: selp.b32 %r17, %r16, %r15, %p1;
+; SM70-PRESERVESIGN-LEGACY-NEXT: shr.u32 %r18, %r17, 16;
+; SM70-PRESERVESIGN-LEGACY-NEXT: shl.b32 %r19, %r18, %r1;
+; SM70-PRESERVESIGN-LEGACY-NEXT: and.b32 %r20, %r23, %r2;
+; SM70-PRESERVESIGN-LEGACY-NEXT: or.b32 %r21, %r20, %r19;
+; SM70-PRESERVESIGN-LEGACY-NEXT: atom.relaxed.sys.global.cas.b32 %r3, [%rd1], %r23, %r21;
+; SM70-PRESERVESIGN-LEGACY-NEXT: setp.ne.b32 %p2, %r3, %r23;
+; SM70-PRESERVESIGN-LEGACY-NEXT: mov.b32 %r23, %r3;
+; SM70-PRESERVESIGN-LEGACY-NEXT: @%p2 bra $L__BB6_1;
+; SM70-PRESERVESIGN-LEGACY-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-PRESERVESIGN-LEGACY-NEXT: shr.u32 %r22, %r3, %r1;
+; SM70-PRESERVESIGN-LEGACY-NEXT: st.param::func.b16 [func_retval0], %r22;
+; SM70-PRESERVESIGN-LEGACY-NEXT: ret;
+;
+; SM70-PRESERVESIGN-FAST-LABEL: fadd_bf16_global(
+; SM70-PRESERVESIGN-FAST: {
+; SM70-PRESERVESIGN-FAST-NEXT: .reg .pred %p<3>;
+; SM70-PRESERVESIGN-FAST-NEXT: .reg .b16 %rs<2>;
+; SM70-PRESERVESIGN-FAST-NEXT: .reg .b32 %r<24>;
+; SM70-PRESERVESIGN-FAST-NEXT: .reg .b64 %rd<3>;
+; SM70-PRESERVESIGN-FAST-EMPTY:
+; SM70-PRESERVESIGN-FAST-NEXT: // %bb.0:
+; SM70-PRESERVESIGN-FAST-NEXT: ld.param::func.b16 %rs1, [fadd_bf16_global_param_1];
+; SM70-PRESERVESIGN-FAST-NEXT: ld.param::func.b64 %rd2, [fadd_bf16_global_param_0];
+; SM70-PRESERVESIGN-FAST-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-PRESERVESIGN-FAST-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-PRESERVESIGN-FAST-NEXT: and.b32 %r5, %r4, 3;
+; SM70-PRESERVESIGN-FAST-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-PRESERVESIGN-FAST-NEXT: mov.b32 %r6, 65535;
+; SM70-PRESERVESIGN-FAST-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-PRESERVESIGN-FAST-NEXT: not.b32 %r2, %r7;
+; SM70-PRESERVESIGN-FAST-NEXT: ld.relaxed.sys.global.b32 %r23, [%rd1];
+; SM70-PRESERVESIGN-FAST-NEXT: cvt.u32.u16 %r10, %rs1;
+; SM70-PRESERVESIGN-FAST-NEXT: shl.b32 %r11, %r10, 16;
+; SM70-PRESERVESIGN-FAST-NEXT: $L__BB6_1: // %atomicrmw.start
+; SM70-PRESERVESIGN-FAST-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-PRESERVESIGN-FAST-NEXT: shr.u32 %r8, %r23, %r1;
+; SM70-PRESERVESIGN-FAST-NEXT: shl.b32 %r9, %r8, 16;
+; SM70-PRESERVESIGN-FAST-NEXT: add.rn.ftz.f32 %r12, %r9, %r11;
+; SM70-PRESERVESIGN-FAST-NEXT: bfe.u32 %r13, %r12, 16, 1;
+; SM70-PRESERVESIGN-FAST-NEXT: add.s32 %r14, %r13, %r12;
+; SM70-PRESERVESIGN-FAST-NEXT: add.s32 %r15, %r14, 32767;
+; SM70-PRESERVESIGN-FAST-NEXT: setp.nan.ftz.f32 %p1, %r12, %r12;
+; SM70-PRESERVESIGN-FAST-NEXT: or.b32 %r16, %r12, 4194304;
+; SM70-PRESERVESIGN-FAST-NEXT: selp.b32 %r17, %r16, %r15, %p1;
+; SM70-PRESERVESIGN-FAST-NEXT: shr.u32 %r18, %r17, 16;
+; SM70-PRESERVESIGN-FAST-NEXT: shl.b32 %r19, %r18, %r1;
+; SM70-PRESERVESIGN-FAST-NEXT: and.b32 %r20, %r23, %r2;
+; SM70-PRESERVESIGN-FAST-NEXT: or.b32 %r21, %r20, %r19;
+; SM70-PRESERVESIGN-FAST-NEXT: atom.relaxed.sys.global.cas.b32 %r3, [%rd1], %r23, %r21;
+; SM70-PRESERVESIGN-FAST-NEXT: setp.ne.b32 %p2, %r3, %r23;
+; SM70-PRESERVESIGN-FAST-NEXT: mov.b32 %r23, %r3;
+; SM70-PRESERVESIGN-FAST-NEXT: @%p2 bra $L__BB6_1;
+; SM70-PRESERVESIGN-FAST-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-PRESERVESIGN-FAST-NEXT: shr.u32 %r22, %r3, %r1;
+; SM70-PRESERVESIGN-FAST-NEXT: st.param::func.b16 [func_retval0], %r22;
+; SM70-PRESERVESIGN-FAST-NEXT: ret;
+;
+; SM90-IEEE-STRICT-LABEL: fadd_bf16_global(
+; SM90-IEEE-STRICT: {
+; SM90-IEEE-STRICT-NEXT: .reg .b16 %rs<3>;
+; SM90-IEEE-STRICT-NEXT: .reg .b64 %rd<2>;
+; SM90-IEEE-STRICT-EMPTY:
+; SM90-IEEE-STRICT-NEXT: // %bb.0:
+; SM90-IEEE-STRICT-NEXT: ld.param::func.b64 %rd1, [fadd_bf16_global_param_0];
+; SM90-IEEE-STRICT-NEXT: ld.param::func.b16 %rs1, [fadd_bf16_global_param_1];
+; SM90-IEEE-STRICT-NEXT: atom.relaxed.sys.global.add.noftz.bf16 %rs2, [%rd1], %rs1;
+; SM90-IEEE-STRICT-NEXT: st.param::func.b16 [func_retval0], %rs2;
+; SM90-IEEE-STRICT-NEXT: ret;
+;
+; SM90-IEEE-LEGACY-LABEL: fadd_bf16_global(
+; SM90-IEEE-LEGACY: {
+; SM90-IEEE-LEGACY-NEXT: .reg .b16 %rs<3>;
+; SM90-IEEE-LEGACY-NEXT: .reg .b64 %rd<2>;
+; SM90-IEEE-LEGACY-EMPTY:
+; SM90-IEEE-LEGACY-NEXT: // %bb.0:
+; SM90-IEEE-LEGACY-NEXT: ld.param::func.b64 %rd1, [fadd_bf16_global_param_0];
+; SM90-IEEE-LEGACY-NEXT: ld.param::func.b16 %rs1, [fadd_bf16_global_param_1];
+; SM90-IEEE-LEGACY-NEXT: atom.relaxed.sys.global.add.noftz.bf16 %rs2, [%rd1], %rs1;
+; SM90-IEEE-LEGACY-NEXT: st.param::func.b16 [func_retval0], %rs2;
+; SM90-IEEE-LEGACY-NEXT: ret;
+;
+; SM90-IEEE-FAST-LABEL: fadd_bf16_global(
+; SM90-IEEE-FAST: {
+; SM90-IEEE-FAST-NEXT: .reg .b16 %rs<3>;
+; SM90-IEEE-FAST-NEXT: .reg .b64 %rd<2>;
+; SM90-IEEE-FAST-EMPTY:
+; SM90-IEEE-FAST-NEXT: // %bb.0:
+; SM90-IEEE-FAST-NEXT: ld.param::func.b64 %rd1, [fadd_bf16_global_param_0];
+; SM90-IEEE-FAST-NEXT: ld.param::func.b16 %rs1, [fadd_bf16_global_param_1];
+; SM90-IEEE-FAST-NEXT: atom.relaxed.sys.global.add.noftz.bf16 %rs2, [%rd1], %rs1;
+; SM90-IEEE-FAST-NEXT: st.param::func.b16 [func_retval0], %rs2;
+; SM90-IEEE-FAST-NEXT: ret;
+;
+; SM90-PRESERVESIGN-STRICT-LABEL: fadd_bf16_global(
+; SM90-PRESERVESIGN-STRICT: {
+; SM90-PRESERVESIGN-STRICT-NEXT: .reg .b16 %rs<3>;
+; SM90-PRESERVESIGN-STRICT-NEXT: .reg .b64 %rd<2>;
+; SM90-PRESERVESIGN-STRICT-EMPTY:
+; SM90-PRESERVESIGN-STRICT-NEXT: // %bb.0:
+; SM90-PRESERVESIGN-STRICT-NEXT: ld.param::func.b64 %rd1, [fadd_bf16_global_param_0];
+; SM90-PRESERVESIGN-STRICT-NEXT: ld.param::func.b16 %rs1, [fadd_bf16_global_param_1];
+; SM90-PRESERVESIGN-STRICT-NEXT: atom.relaxed.sys.global.add.noftz.bf16 %rs2, [%rd1], %rs1;
+; SM90-PRESERVESIGN-STRICT-NEXT: st.param::func.b16 [func_retval0], %rs2;
+; SM90-PRESERVESIGN-STRICT-NEXT: ret;
+;
+; SM90-PRESERVESIGN-LEGACY-LABEL: fadd_bf16_global(
+; SM90-PRESERVESIGN-LEGACY: {
+; SM90-PRESERVESIGN-LEGACY-NEXT: .reg .b16 %rs<3>;
+; SM90-PRESERVESIGN-LEGACY-NEXT: .reg .b64 %rd<2>;
+; SM90-PRESERVESIGN-LEGACY-EMPTY:
+; SM90-PRESERVESIGN-LEGACY-NEXT: // %bb.0:
+; SM90-PRESERVESIGN-LEGACY-NEXT: ld.param::func.b64 %rd1, [fadd_bf16_global_param_0];
+; SM90-PRESERVESIGN-LEGACY-NEXT: ld.param::func.b16 %rs1, [fadd_bf16_global_param_1];
+; SM90-PRESERVESIGN-LEGACY-NEXT: atom.relaxed.sys.global.add.noftz.bf16 %rs2, [%rd1], %rs1;
+; SM90-PRESERVESIGN-LEGACY-NEXT: st.param::func.b16 [func_retval0], %rs2;
+; SM90-PRESERVESIGN-LEGACY-NEXT: ret;
+;
+; SM90-PRESERVESIGN-FAST-LABEL: fadd_bf16_global(
+; SM90-PRESERVESIGN-FAST: {
+; SM90-PRESERVESIGN-FAST-NEXT: .reg .b16 %rs<3>;
+; SM90-PRESERVESIGN-FAST-NEXT: .reg .b64 %rd<2>;
+; SM90-PRESERVESIGN-FAST-EMPTY:
+; SM90-PRESERVESIGN-FAST-NEXT: // %bb.0:
+; SM90-PRESERVESIGN-FAST-NEXT: ld.param::func.b64 %rd1, [fadd_bf16_global_param_0];
+; SM90-PRESERVESIGN-FAST-NEXT: ld.param::func.b16 %rs1, [fadd_bf16_global_param_1];
+; SM90-PRESERVESIGN-FAST-NEXT: atom.relaxed.sys.global.add.noftz.bf16 %rs2, [%rd1], %rs1;
+; SM90-PRESERVESIGN-FAST-NEXT: st.param::func.b16 [func_retval0], %rs2;
+; SM90-PRESERVESIGN-FAST-NEXT: ret;
+ %r = atomicrmw fadd ptr addrspace(1) %addr, bfloat %val monotonic
+ ret bfloat %r
+}
+
+define bfloat @fadd_bf16_generic(ptr %addr, bfloat %val) {
+; SM70-IEEE-STRICT-LABEL: fadd_bf16_generic(
+; SM70-IEEE-STRICT: {
+; SM70-IEEE-STRICT-NEXT: .reg .pred %p<3>;
+; SM70-IEEE-STRICT-NEXT: .reg .b16 %rs<2>;
+; SM70-IEEE-STRICT-NEXT: .reg .b32 %r<24>;
+; SM70-IEEE-STRICT-NEXT: .reg .b64 %rd<3>;
+; SM70-IEEE-STRICT-EMPTY:
+; SM70-IEEE-STRICT-NEXT: // %bb.0:
+; SM70-IEEE-STRICT-NEXT: ld.param::func.b16 %rs1, [fadd_bf16_generic_param_1];
+; SM70-IEEE-STRICT-NEXT: ld.param::func.b64 %rd2, [fadd_bf16_generic_param_0];
+; SM70-IEEE-STRICT-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-IEEE-STRICT-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-IEEE-STRICT-NEXT: and.b32 %r5, %r4, 3;
+; SM70-IEEE-STRICT-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-IEEE-STRICT-NEXT: mov.b32 %r6, 65535;
+; SM70-IEEE-STRICT-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-IEEE-STRICT-NEXT: not.b32 %r2, %r7;
+; SM70-IEEE-STRICT-NEXT: ld.relaxed.sys.b32 %r23, [%rd1];
+; SM70-IEEE-STRICT-NEXT: cvt.u32.u16 %r10, %rs1;
+; SM70-IEEE-STRICT-NEXT: shl.b32 %r11, %r10, 16;
+; SM70-IEEE-STRICT-NEXT: $L__BB7_1: // %atomicrmw.start
+; SM70-IEEE-STRICT-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-IEEE-STRICT-NEXT: shr.u32 %r8, %r23, %r1;
+; SM70-IEEE-STRICT-NEXT: shl.b32 %r9, %r8, 16;
+; SM70-IEEE-STRICT-NEXT: add.rn.f32 %r12, %r9, %r11;
+; SM70-IEEE-STRICT-NEXT: bfe.u32 %r13, %r12, 16, 1;
+; SM70-IEEE-STRICT-NEXT: add.s32 %r14, %r13, %r12;
+; SM70-IEEE-STRICT-NEXT: add.s32 %r15, %r14, 32767;
+; SM70-IEEE-STRICT-NEXT: setp.nan.f32 %p1, %r12, %r12;
+; SM70-IEEE-STRICT-NEXT: or.b32 %r16, %r12, 4194304;
+; SM70-IEEE-STRICT-NEXT: selp.b32 %r17, %r16, %r15, %p1;
+; SM70-IEEE-STRICT-NEXT: shr.u32 %r18, %r17, 16;
+; SM70-IEEE-STRICT-NEXT: shl.b32 %r19, %r18, %r1;
+; SM70-IEEE-STRICT-NEXT: and.b32 %r20, %r23, %r2;
+; SM70-IEEE-STRICT-NEXT: or.b32 %r21, %r20, %r19;
+; SM70-IEEE-STRICT-NEXT: atom.relaxed.sys.cas.b32 %r3, [%rd1], %r23, %r21;
+; SM70-IEEE-STRICT-NEXT: setp.ne.b32 %p2, %r3, %r23;
+; SM70-IEEE-STRICT-NEXT: mov.b32 %r23, %r3;
+; SM70-IEEE-STRICT-NEXT: @%p2 bra $L__BB7_1;
+; SM70-IEEE-STRICT-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-IEEE-STRICT-NEXT: shr.u32 %r22, %r3, %r1;
+; SM70-IEEE-STRICT-NEXT: st.param::func.b16 [func_retval0], %r22;
+; SM70-IEEE-STRICT-NEXT: ret;
+;
+; SM70-IEEE-LEGACY-LABEL: fadd_bf16_generic(
+; SM70-IEEE-LEGACY: {
+; SM70-IEEE-LEGACY-NEXT: .reg .pred %p<3>;
+; SM70-IEEE-LEGACY-NEXT: .reg .b16 %rs<2>;
+; SM70-IEEE-LEGACY-NEXT: .reg .b32 %r<24>;
+; SM70-IEEE-LEGACY-NEXT: .reg .b64 %rd<3>;
+; SM70-IEEE-LEGACY-EMPTY:
+; SM70-IEEE-LEGACY-NEXT: // %bb.0:
+; SM70-IEEE-LEGACY-NEXT: ld.param::func.b16 %rs1, [fadd_bf16_generic_param_1];
+; SM70-IEEE-LEGACY-NEXT: ld.param::func.b64 %rd2, [fadd_bf16_generic_param_0];
+; SM70-IEEE-LEGACY-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-IEEE-LEGACY-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-IEEE-LEGACY-NEXT: and.b32 %r5, %r4, 3;
+; SM70-IEEE-LEGACY-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-IEEE-LEGACY-NEXT: mov.b32 %r6, 65535;
+; SM70-IEEE-LEGACY-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-IEEE-LEGACY-NEXT: not.b32 %r2, %r7;
+; SM70-IEEE-LEGACY-NEXT: ld.relaxed.sys.b32 %r23, [%rd1];
+; SM70-IEEE-LEGACY-NEXT: cvt.u32.u16 %r10, %rs1;
+; SM70-IEEE-LEGACY-NEXT: shl.b32 %r11, %r10, 16;
+; SM70-IEEE-LEGACY-NEXT: $L__BB7_1: // %atomicrmw.start
+; SM70-IEEE-LEGACY-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-IEEE-LEGACY-NEXT: shr.u32 %r8, %r23, %r1;
+; SM70-IEEE-LEGACY-NEXT: shl.b32 %r9, %r8, 16;
+; SM70-IEEE-LEGACY-NEXT: add.rn.f32 %r12, %r9, %r11;
+; SM70-IEEE-LEGACY-NEXT: bfe.u32 %r13, %r12, 16, 1;
+; SM70-IEEE-LEGACY-NEXT: add.s32 %r14, %r13, %r12;
+; SM70-IEEE-LEGACY-NEXT: add.s32 %r15, %r14, 32767;
+; SM70-IEEE-LEGACY-NEXT: setp.nan.f32 %p1, %r12, %r12;
+; SM70-IEEE-LEGACY-NEXT: or.b32 %r16, %r12, 4194304;
+; SM70-IEEE-LEGACY-NEXT: selp.b32 %r17, %r16, %r15, %p1;
+; SM70-IEEE-LEGACY-NEXT: shr.u32 %r18, %r17, 16;
+; SM70-IEEE-LEGACY-NEXT: shl.b32 %r19, %r18, %r1;
+; SM70-IEEE-LEGACY-NEXT: and.b32 %r20, %r23, %r2;
+; SM70-IEEE-LEGACY-NEXT: or.b32 %r21, %r20, %r19;
+; SM70-IEEE-LEGACY-NEXT: atom.relaxed.sys.cas.b32 %r3, [%rd1], %r23, %r21;
+; SM70-IEEE-LEGACY-NEXT: setp.ne.b32 %p2, %r3, %r23;
+; SM70-IEEE-LEGACY-NEXT: mov.b32 %r23, %r3;
+; SM70-IEEE-LEGACY-NEXT: @%p2 bra $L__BB7_1;
+; SM70-IEEE-LEGACY-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-IEEE-LEGACY-NEXT: shr.u32 %r22, %r3, %r1;
+; SM70-IEEE-LEGACY-NEXT: st.param::func.b16 [func_retval0], %r22;
+; SM70-IEEE-LEGACY-NEXT: ret;
+;
+; SM70-IEEE-FAST-LABEL: fadd_bf16_generic(
+; SM70-IEEE-FAST: {
+; SM70-IEEE-FAST-NEXT: .reg .pred %p<3>;
+; SM70-IEEE-FAST-NEXT: .reg .b16 %rs<2>;
+; SM70-IEEE-FAST-NEXT: .reg .b32 %r<24>;
+; SM70-IEEE-FAST-NEXT: .reg .b64 %rd<3>;
+; SM70-IEEE-FAST-EMPTY:
+; SM70-IEEE-FAST-NEXT: // %bb.0:
+; SM70-IEEE-FAST-NEXT: ld.param::func.b16 %rs1, [fadd_bf16_generic_param_1];
+; SM70-IEEE-FAST-NEXT: ld.param::func.b64 %rd2, [fadd_bf16_generic_param_0];
+; SM70-IEEE-FAST-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-IEEE-FAST-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-IEEE-FAST-NEXT: and.b32 %r5, %r4, 3;
+; SM70-IEEE-FAST-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-IEEE-FAST-NEXT: mov.b32 %r6, 65535;
+; SM70-IEEE-FAST-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-IEEE-FAST-NEXT: not.b32 %r2, %r7;
+; SM70-IEEE-FAST-NEXT: ld.relaxed.sys.b32 %r23, [%rd1];
+; SM70-IEEE-FAST-NEXT: cvt.u32.u16 %r10, %rs1;
+; SM70-IEEE-FAST-NEXT: shl.b32 %r11, %r10, 16;
+; SM70-IEEE-FAST-NEXT: $L__BB7_1: // %atomicrmw.start
+; SM70-IEEE-FAST-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-IEEE-FAST-NEXT: shr.u32 %r8, %r23, %r1;
+; SM70-IEEE-FAST-NEXT: shl.b32 %r9, %r8, 16;
+; SM70-IEEE-FAST-NEXT: add.rn.f32 %r12, %r9, %r11;
+; SM70-IEEE-FAST-NEXT: bfe.u32 %r13, %r12, 16, 1;
+; SM70-IEEE-FAST-NEXT: add.s32 %r14, %r13, %r12;
+; SM70-IEEE-FAST-NEXT: add.s32 %r15, %r14, 32767;
+; SM70-IEEE-FAST-NEXT: setp.nan.f32 %p1, %r12, %r12;
+; SM70-IEEE-FAST-NEXT: or.b32 %r16, %r12, 4194304;
+; SM70-IEEE-FAST-NEXT: selp.b32 %r17, %r16, %r15, %p1;
+; SM70-IEEE-FAST-NEXT: shr.u32 %r18, %r17, 16;
+; SM70-IEEE-FAST-NEXT: shl.b32 %r19, %r18, %r1;
+; SM70-IEEE-FAST-NEXT: and.b32 %r20, %r23, %r2;
+; SM70-IEEE-FAST-NEXT: or.b32 %r21, %r20, %r19;
+; SM70-IEEE-FAST-NEXT: atom.relaxed.sys.cas.b32 %r3, [%rd1], %r23, %r21;
+; SM70-IEEE-FAST-NEXT: setp.ne.b32 %p2, %r3, %r23;
+; SM70-IEEE-FAST-NEXT: mov.b32 %r23, %r3;
+; SM70-IEEE-FAST-NEXT: @%p2 bra $L__BB7_1;
+; SM70-IEEE-FAST-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-IEEE-FAST-NEXT: shr.u32 %r22, %r3, %r1;
+; SM70-IEEE-FAST-NEXT: st.param::func.b16 [func_retval0], %r22;
+; SM70-IEEE-FAST-NEXT: ret;
+;
+; SM70-PRESERVESIGN-STRICT-LABEL: fadd_bf16_generic(
+; SM70-PRESERVESIGN-STRICT: {
+; SM70-PRESERVESIGN-STRICT-NEXT: .reg .pred %p<3>;
+; SM70-PRESERVESIGN-STRICT-NEXT: .reg .b16 %rs<2>;
+; SM70-PRESERVESIGN-STRICT-NEXT: .reg .b32 %r<24>;
+; SM70-PRESERVESIGN-STRICT-NEXT: .reg .b64 %rd<3>;
+; SM70-PRESERVESIGN-STRICT-EMPTY:
+; SM70-PRESERVESIGN-STRICT-NEXT: // %bb.0:
+; SM70-PRESERVESIGN-STRICT-NEXT: ld.param::func.b16 %rs1, [fadd_bf16_generic_param_1];
+; SM70-PRESERVESIGN-STRICT-NEXT: ld.param::func.b64 %rd2, [fadd_bf16_generic_param_0];
+; SM70-PRESERVESIGN-STRICT-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-PRESERVESIGN-STRICT-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-PRESERVESIGN-STRICT-NEXT: and.b32 %r5, %r4, 3;
+; SM70-PRESERVESIGN-STRICT-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-PRESERVESIGN-STRICT-NEXT: mov.b32 %r6, 65535;
+; SM70-PRESERVESIGN-STRICT-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-PRESERVESIGN-STRICT-NEXT: not.b32 %r2, %r7;
+; SM70-PRESERVESIGN-STRICT-NEXT: ld.relaxed.sys.b32 %r23, [%rd1];
+; SM70-PRESERVESIGN-STRICT-NEXT: cvt.u32.u16 %r10, %rs1;
+; SM70-PRESERVESIGN-STRICT-NEXT: shl.b32 %r11, %r10, 16;
+; SM70-PRESERVESIGN-STRICT-NEXT: $L__BB7_1: // %atomicrmw.start
+; SM70-PRESERVESIGN-STRICT-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-PRESERVESIGN-STRICT-NEXT: shr.u32 %r8, %r23, %r1;
+; SM70-PRESERVESIGN-STRICT-NEXT: shl.b32 %r9, %r8, 16;
+; SM70-PRESERVESIGN-STRICT-NEXT: add.rn.ftz.f32 %r12, %r9, %r11;
+; SM70-PRESERVESIGN-STRICT-NEXT: bfe.u32 %r13, %r12, 16, 1;
+; SM70-PRESERVESIGN-STRICT-NEXT: add.s32 %r14, %r13, %r12;
+; SM70-PRESERVESIGN-STRICT-NEXT: add.s32 %r15, %r14, 32767;
+; SM70-PRESERVESIGN-STRICT-NEXT: setp.nan.ftz.f32 %p1, %r12, %r12;
+; SM70-PRESERVESIGN-STRICT-NEXT: or.b32 %r16, %r12, 4194304;
+; SM70-PRESERVESIGN-STRICT-NEXT: selp.b32 %r17, %r16, %r15, %p1;
+; SM70-PRESERVESIGN-STRICT-NEXT: shr.u32 %r18, %r17, 16;
+; SM70-PRESERVESIGN-STRICT-NEXT: shl.b32 %r19, %r18, %r1;
+; SM70-PRESERVESIGN-STRICT-NEXT: and.b32 %r20, %r23, %r2;
+; SM70-PRESERVESIGN-STRICT-NEXT: or.b32 %r21, %r20, %r19;
+; SM70-PRESERVESIGN-STRICT-NEXT: atom.relaxed.sys.cas.b32 %r3, [%rd1], %r23, %r21;
+; SM70-PRESERVESIGN-STRICT-NEXT: setp.ne.b32 %p2, %r3, %r23;
+; SM70-PRESERVESIGN-STRICT-NEXT: mov.b32 %r23, %r3;
+; SM70-PRESERVESIGN-STRICT-NEXT: @%p2 bra $L__BB7_1;
+; SM70-PRESERVESIGN-STRICT-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-PRESERVESIGN-STRICT-NEXT: shr.u32 %r22, %r3, %r1;
+; SM70-PRESERVESIGN-STRICT-NEXT: st.param::func.b16 [func_retval0], %r22;
+; SM70-PRESERVESIGN-STRICT-NEXT: ret;
+;
+; SM70-PRESERVESIGN-LEGACY-LABEL: fadd_bf16_generic(
+; SM70-PRESERVESIGN-LEGACY: {
+; SM70-PRESERVESIGN-LEGACY-NEXT: .reg .pred %p<3>;
+; SM70-PRESERVESIGN-LEGACY-NEXT: .reg .b16 %rs<2>;
+; SM70-PRESERVESIGN-LEGACY-NEXT: .reg .b32 %r<24>;
+; SM70-PRESERVESIGN-LEGACY-NEXT: .reg .b64 %rd<3>;
+; SM70-PRESERVESIGN-LEGACY-EMPTY:
+; SM70-PRESERVESIGN-LEGACY-NEXT: // %bb.0:
+; SM70-PRESERVESIGN-LEGACY-NEXT: ld.param::func.b16 %rs1, [fadd_bf16_generic_param_1];
+; SM70-PRESERVESIGN-LEGACY-NEXT: ld.param::func.b64 %rd2, [fadd_bf16_generic_param_0];
+; SM70-PRESERVESIGN-LEGACY-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-PRESERVESIGN-LEGACY-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-PRESERVESIGN-LEGACY-NEXT: and.b32 %r5, %r4, 3;
+; SM70-PRESERVESIGN-LEGACY-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-PRESERVESIGN-LEGACY-NEXT: mov.b32 %r6, 65535;
+; SM70-PRESERVESIGN-LEGACY-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-PRESERVESIGN-LEGACY-NEXT: not.b32 %r2, %r7;
+; SM70-PRESERVESIGN-LEGACY-NEXT: ld.relaxed.sys.b32 %r23, [%rd1];
+; SM70-PRESERVESIGN-LEGACY-NEXT: cvt.u32.u16 %r10, %rs1;
+; SM70-PRESERVESIGN-LEGACY-NEXT: shl.b32 %r11, %r10, 16;
+; SM70-PRESERVESIGN-LEGACY-NEXT: $L__BB7_1: // %atomicrmw.start
+; SM70-PRESERVESIGN-LEGACY-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-PRESERVESIGN-LEGACY-NEXT: shr.u32 %r8, %r23, %r1;
+; SM70-PRESERVESIGN-LEGACY-NEXT: shl.b32 %r9, %r8, 16;
+; SM70-PRESERVESIGN-LEGACY-NEXT: add.rn.ftz.f32 %r12, %r9, %r11;
+; SM70-PRESERVESIGN-LEGACY-NEXT: bfe.u32 %r13, %r12, 16, 1;
+; SM70-PRESERVESIGN-LEGACY-NEXT: add.s32 %r14, %r13, %r12;
+; SM70-PRESERVESIGN-LEGACY-NEXT: add.s32 %r15, %r14, 32767;
+; SM70-PRESERVESIGN-LEGACY-NEXT: setp.nan.ftz.f32 %p1, %r12, %r12;
+; SM70-PRESERVESIGN-LEGACY-NEXT: or.b32 %r16, %r12, 4194304;
+; SM70-PRESERVESIGN-LEGACY-NEXT: selp.b32 %r17, %r16, %r15, %p1;
+; SM70-PRESERVESIGN-LEGACY-NEXT: shr.u32 %r18, %r17, 16;
+; SM70-PRESERVESIGN-LEGACY-NEXT: shl.b32 %r19, %r18, %r1;
+; SM70-PRESERVESIGN-LEGACY-NEXT: and.b32 %r20, %r23, %r2;
+; SM70-PRESERVESIGN-LEGACY-NEXT: or.b32 %r21, %r20, %r19;
+; SM70-PRESERVESIGN-LEGACY-NEXT: atom.relaxed.sys.cas.b32 %r3, [%rd1], %r23, %r21;
+; SM70-PRESERVESIGN-LEGACY-NEXT: setp.ne.b32 %p2, %r3, %r23;
+; SM70-PRESERVESIGN-LEGACY-NEXT: mov.b32 %r23, %r3;
+; SM70-PRESERVESIGN-LEGACY-NEXT: @%p2 bra $L__BB7_1;
+; SM70-PRESERVESIGN-LEGACY-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-PRESERVESIGN-LEGACY-NEXT: shr.u32 %r22, %r3, %r1;
+; SM70-PRESERVESIGN-LEGACY-NEXT: st.param::func.b16 [func_retval0], %r22;
+; SM70-PRESERVESIGN-LEGACY-NEXT: ret;
+;
+; SM70-PRESERVESIGN-FAST-LABEL: fadd_bf16_generic(
+; SM70-PRESERVESIGN-FAST: {
+; SM70-PRESERVESIGN-FAST-NEXT: .reg .pred %p<3>;
+; SM70-PRESERVESIGN-FAST-NEXT: .reg .b16 %rs<2>;
+; SM70-PRESERVESIGN-FAST-NEXT: .reg .b32 %r<24>;
+; SM70-PRESERVESIGN-FAST-NEXT: .reg .b64 %rd<3>;
+; SM70-PRESERVESIGN-FAST-EMPTY:
+; SM70-PRESERVESIGN-FAST-NEXT: // %bb.0:
+; SM70-PRESERVESIGN-FAST-NEXT: ld.param::func.b16 %rs1, [fadd_bf16_generic_param_1];
+; SM70-PRESERVESIGN-FAST-NEXT: ld.param::func.b64 %rd2, [fadd_bf16_generic_param_0];
+; SM70-PRESERVESIGN-FAST-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-PRESERVESIGN-FAST-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-PRESERVESIGN-FAST-NEXT: and.b32 %r5, %r4, 3;
+; SM70-PRESERVESIGN-FAST-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-PRESERVESIGN-FAST-NEXT: mov.b32 %r6, 65535;
+; SM70-PRESERVESIGN-FAST-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-PRESERVESIGN-FAST-NEXT: not.b32 %r2, %r7;
+; SM70-PRESERVESIGN-FAST-NEXT: ld.relaxed.sys.b32 %r23, [%rd1];
+; SM70-PRESERVESIGN-FAST-NEXT: cvt.u32.u16 %r10, %rs1;
+; SM70-PRESERVESIGN-FAST-NEXT: shl.b32 %r11, %r10, 16;
+; SM70-PRESERVESIGN-FAST-NEXT: $L__BB7_1: // %atomicrmw.start
+; SM70-PRESERVESIGN-FAST-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-PRESERVESIGN-FAST-NEXT: shr.u32 %r8, %r23, %r1;
+; SM70-PRESERVESIGN-FAST-NEXT: shl.b32 %r9, %r8, 16;
+; SM70-PRESERVESIGN-FAST-NEXT: add.rn.ftz.f32 %r12, %r9, %r11;
+; SM70-PRESERVESIGN-FAST-NEXT: bfe.u32 %r13, %r12, 16, 1;
+; SM70-PRESERVESIGN-FAST-NEXT: add.s32 %r14, %r13, %r12;
+; SM70-PRESERVESIGN-FAST-NEXT: add.s32 %r15, %r14, 32767;
+; SM70-PRESERVESIGN-FAST-NEXT: setp.nan.ftz.f32 %p1, %r12, %r12;
+; SM70-PRESERVESIGN-FAST-NEXT: or.b32 %r16, %r12, 4194304;
+; SM70-PRESERVESIGN-FAST-NEXT: selp.b32 %r17, %r16, %r15, %p1;
+; SM70-PRESERVESIGN-FAST-NEXT: shr.u32 %r18, %r17, 16;
+; SM70-PRESERVESIGN-FAST-NEXT: shl.b32 %r19, %r18, %r1;
+; SM70-PRESERVESIGN-FAST-NEXT: and.b32 %r20, %r23, %r2;
+; SM70-PRESERVESIGN-FAST-NEXT: or.b32 %r21, %r20, %r19;
+; SM70-PRESERVESIGN-FAST-NEXT: atom.relaxed.sys.cas.b32 %r3, [%rd1], %r23, %r21;
+; SM70-PRESERVESIGN-FAST-NEXT: setp.ne.b32 %p2, %r3, %r23;
+; SM70-PRESERVESIGN-FAST-NEXT: mov.b32 %r23, %r3;
+; SM70-PRESERVESIGN-FAST-NEXT: @%p2 bra $L__BB7_1;
+; SM70-PRESERVESIGN-FAST-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-PRESERVESIGN-FAST-NEXT: shr.u32 %r22, %r3, %r1;
+; SM70-PRESERVESIGN-FAST-NEXT: st.param::func.b16 [func_retval0], %r22;
+; SM70-PRESERVESIGN-FAST-NEXT: ret;
+;
+; SM90-IEEE-STRICT-LABEL: fadd_bf16_generic(
+; SM90-IEEE-STRICT: {
+; SM90-IEEE-STRICT-NEXT: .reg .b16 %rs<3>;
+; SM90-IEEE-STRICT-NEXT: .reg .b64 %rd<2>;
+; SM90-IEEE-STRICT-EMPTY:
+; SM90-IEEE-STRICT-NEXT: // %bb.0:
+; SM90-IEEE-STRICT-NEXT: ld.param::func.b64 %rd1, [fadd_bf16_generic_param_0];
+; SM90-IEEE-STRICT-NEXT: ld.param::func.b16 %rs1, [fadd_bf16_generic_param_1];
+; SM90-IEEE-STRICT-NEXT: atom.relaxed.sys.add.noftz.bf16 %rs2, [%rd1], %rs1;
+; SM90-IEEE-STRICT-NEXT: st.param::func.b16 [func_retval0], %rs2;
+; SM90-IEEE-STRICT-NEXT: ret;
+;
+; SM90-IEEE-LEGACY-LABEL: fadd_bf16_generic(
+; SM90-IEEE-LEGACY: {
+; SM90-IEEE-LEGACY-NEXT: .reg .b16 %rs<3>;
+; SM90-IEEE-LEGACY-NEXT: .reg .b64 %rd<2>;
+; SM90-IEEE-LEGACY-EMPTY:
+; SM90-IEEE-LEGACY-NEXT: // %bb.0:
+; SM90-IEEE-LEGACY-NEXT: ld.param::func.b64 %rd1, [fadd_bf16_generic_param_0];
+; SM90-IEEE-LEGACY-NEXT: ld.param::func.b16 %rs1, [fadd_bf16_generic_param_1];
+; SM90-IEEE-LEGACY-NEXT: atom.relaxed.sys.add.noftz.bf16 %rs2, [%rd1], %rs1;
+; SM90-IEEE-LEGACY-NEXT: st.param::func.b16 [func_retval0], %rs2;
+; SM90-IEEE-LEGACY-NEXT: ret;
+;
+; SM90-IEEE-FAST-LABEL: fadd_bf16_generic(
+; SM90-IEEE-FAST: {
+; SM90-IEEE-FAST-NEXT: .reg .b16 %rs<3>;
+; SM90-IEEE-FAST-NEXT: .reg .b64 %rd<2>;
+; SM90-IEEE-FAST-EMPTY:
+; SM90-IEEE-FAST-NEXT: // %bb.0:
+; SM90-IEEE-FAST-NEXT: ld.param::func.b64 %rd1, [fadd_bf16_generic_param_0];
+; SM90-IEEE-FAST-NEXT: ld.param::func.b16 %rs1, [fadd_bf16_generic_param_1];
+; SM90-IEEE-FAST-NEXT: atom.relaxed.sys.add.noftz.bf16 %rs2, [%rd1], %rs1;
+; SM90-IEEE-FAST-NEXT: st.param::func.b16 [func_retval0], %rs2;
+; SM90-IEEE-FAST-NEXT: ret;
+;
+; SM90-PRESERVESIGN-STRICT-LABEL: fadd_bf16_generic(
+; SM90-PRESERVESIGN-STRICT: {
+; SM90-PRESERVESIGN-STRICT-NEXT: .reg .b16 %rs<3>;
+; SM90-PRESERVESIGN-STRICT-NEXT: .reg .b64 %rd<2>;
+; SM90-PRESERVESIGN-STRICT-EMPTY:
+; SM90-PRESERVESIGN-STRICT-NEXT: // %bb.0:
+; SM90-PRESERVESIGN-STRICT-NEXT: ld.param::func.b64 %rd1, [fadd_bf16_generic_param_0];
+; SM90-PRESERVESIGN-STRICT-NEXT: ld.param::func.b16 %rs1, [fadd_bf16_generic_param_1];
+; SM90-PRESERVESIGN-STRICT-NEXT: atom.relaxed.sys.add.noftz.bf16 %rs2, [%rd1], %rs1;
+; SM90-PRESERVESIGN-STRICT-NEXT: st.param::func.b16 [func_retval0], %rs2;
+; SM90-PRESERVESIGN-STRICT-NEXT: ret;
+;
+; SM90-PRESERVESIGN-LEGACY-LABEL: fadd_bf16_generic(
+; SM90-PRESERVESIGN-LEGACY: {
+; SM90-PRESERVESIGN-LEGACY-NEXT: .reg .b16 %rs<3>;
+; SM90-PRESERVESIGN-LEGACY-NEXT: .reg .b64 %rd<2>;
+; SM90-PRESERVESIGN-LEGACY-EMPTY:
+; SM90-PRESERVESIGN-LEGACY-NEXT: // %bb.0:
+; SM90-PRESERVESIGN-LEGACY-NEXT: ld.param::func.b64 %rd1, [fadd_bf16_generic_param_0];
+; SM90-PRESERVESIGN-LEGACY-NEXT: ld.param::func.b16 %rs1, [fadd_bf16_generic_param_1];
+; SM90-PRESERVESIGN-LEGACY-NEXT: atom.relaxed.sys.add.noftz.bf16 %rs2, [%rd1], %rs1;
+; SM90-PRESERVESIGN-LEGACY-NEXT: st.param::func.b16 [func_retval0], %rs2;
+; SM90-PRESERVESIGN-LEGACY-NEXT: ret;
+;
+; SM90-PRESERVESIGN-FAST-LABEL: fadd_bf16_generic(
+; SM90-PRESERVESIGN-FAST: {
+; SM90-PRESERVESIGN-FAST-NEXT: .reg .b16 %rs<3>;
+; SM90-PRESERVESIGN-FAST-NEXT: .reg .b64 %rd<2>;
+; SM90-PRESERVESIGN-FAST-EMPTY:
+; SM90-PRESERVESIGN-FAST-NEXT: // %bb.0:
+; SM90-PRESERVESIGN-FAST-NEXT: ld.param::func.b64 %rd1, [fadd_bf16_generic_param_0];
+; SM90-PRESERVESIGN-FAST-NEXT: ld.param::func.b16 %rs1, [fadd_bf16_generic_param_1];
+; SM90-PRESERVESIGN-FAST-NEXT: atom.relaxed.sys.add.noftz.bf16 %rs2, [%rd1], %rs1;
+; SM90-PRESERVESIGN-FAST-NEXT: st.param::func.b16 [func_retval0], %rs2;
+; SM90-PRESERVESIGN-FAST-NEXT: ret;
+ %r = atomicrmw fadd ptr %addr, bfloat %val monotonic
+ ret bfloat %r
+}
+
+define bfloat @fadd_bf16_shared(ptr addrspace(3) %addr, bfloat %val) {
+; SM70-IEEE-STRICT-LABEL: fadd_bf16_shared(
+; SM70-IEEE-STRICT: {
+; SM70-IEEE-STRICT-NEXT: .reg .pred %p<3>;
+; SM70-IEEE-STRICT-NEXT: .reg .b16 %rs<2>;
+; SM70-IEEE-STRICT-NEXT: .reg .b32 %r<24>;
+; SM70-IEEE-STRICT-NEXT: .reg .b64 %rd<3>;
+; SM70-IEEE-STRICT-EMPTY:
+; SM70-IEEE-STRICT-NEXT: // %bb.0:
+; SM70-IEEE-STRICT-NEXT: ld.param::func.b16 %rs1, [fadd_bf16_shared_param_1];
+; SM70-IEEE-STRICT-NEXT: ld.param::func.b64 %rd2, [fadd_bf16_shared_param_0];
+; SM70-IEEE-STRICT-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-IEEE-STRICT-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-IEEE-STRICT-NEXT: and.b32 %r5, %r4, 3;
+; SM70-IEEE-STRICT-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-IEEE-STRICT-NEXT: mov.b32 %r6, 65535;
+; SM70-IEEE-STRICT-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-IEEE-STRICT-NEXT: not.b32 %r2, %r7;
+; SM70-IEEE-STRICT-NEXT: ld.relaxed.sys.shared.b32 %r23, [%rd1];
+; SM70-IEEE-STRICT-NEXT: cvt.u32.u16 %r10, %rs1;
+; SM70-IEEE-STRICT-NEXT: shl.b32 %r11, %r10, 16;
+; SM70-IEEE-STRICT-NEXT: $L__BB8_1: // %atomicrmw.start
+; SM70-IEEE-STRICT-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-IEEE-STRICT-NEXT: shr.u32 %r8, %r23, %r1;
+; SM70-IEEE-STRICT-NEXT: shl.b32 %r9, %r8, 16;
+; SM70-IEEE-STRICT-NEXT: add.rn.f32 %r12, %r9, %r11;
+; SM70-IEEE-STRICT-NEXT: bfe.u32 %r13, %r12, 16, 1;
+; SM70-IEEE-STRICT-NEXT: add.s32 %r14, %r13, %r12;
+; SM70-IEEE-STRICT-NEXT: add.s32 %r15, %r14, 32767;
+; SM70-IEEE-STRICT-NEXT: setp.nan.f32 %p1, %r12, %r12;
+; SM70-IEEE-STRICT-NEXT: or.b32 %r16, %r12, 4194304;
+; SM70-IEEE-STRICT-NEXT: selp.b32 %r17, %r16, %r15, %p1;
+; SM70-IEEE-STRICT-NEXT: shr.u32 %r18, %r17, 16;
+; SM70-IEEE-STRICT-NEXT: shl.b32 %r19, %r18, %r1;
+; SM70-IEEE-STRICT-NEXT: and.b32 %r20, %r23, %r2;
+; SM70-IEEE-STRICT-NEXT: or.b32 %r21, %r20, %r19;
+; SM70-IEEE-STRICT-NEXT: atom.relaxed.sys.shared.cas.b32 %r3, [%rd1], %r23, %r21;
+; SM70-IEEE-STRICT-NEXT: setp.ne.b32 %p2, %r3, %r23;
+; SM70-IEEE-STRICT-NEXT: mov.b32 %r23, %r3;
+; SM70-IEEE-STRICT-NEXT: @%p2 bra $L__BB8_1;
+; SM70-IEEE-STRICT-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-IEEE-STRICT-NEXT: shr.u32 %r22, %r3, %r1;
+; SM70-IEEE-STRICT-NEXT: st.param::func.b16 [func_retval0], %r22;
+; SM70-IEEE-STRICT-NEXT: ret;
+;
+; SM70-IEEE-LEGACY-LABEL: fadd_bf16_shared(
+; SM70-IEEE-LEGACY: {
+; SM70-IEEE-LEGACY-NEXT: .reg .pred %p<3>;
+; SM70-IEEE-LEGACY-NEXT: .reg .b16 %rs<2>;
+; SM70-IEEE-LEGACY-NEXT: .reg .b32 %r<24>;
+; SM70-IEEE-LEGACY-NEXT: .reg .b64 %rd<3>;
+; SM70-IEEE-LEGACY-EMPTY:
+; SM70-IEEE-LEGACY-NEXT: // %bb.0:
+; SM70-IEEE-LEGACY-NEXT: ld.param::func.b16 %rs1, [fadd_bf16_shared_param_1];
+; SM70-IEEE-LEGACY-NEXT: ld.param::func.b64 %rd2, [fadd_bf16_shared_param_0];
+; SM70-IEEE-LEGACY-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-IEEE-LEGACY-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-IEEE-LEGACY-NEXT: and.b32 %r5, %r4, 3;
+; SM70-IEEE-LEGACY-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-IEEE-LEGACY-NEXT: mov.b32 %r6, 65535;
+; SM70-IEEE-LEGACY-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-IEEE-LEGACY-NEXT: not.b32 %r2, %r7;
+; SM70-IEEE-LEGACY-NEXT: ld.relaxed.sys.shared.b32 %r23, [%rd1];
+; SM70-IEEE-LEGACY-NEXT: cvt.u32.u16 %r10, %rs1;
+; SM70-IEEE-LEGACY-NEXT: shl.b32 %r11, %r10, 16;
+; SM70-IEEE-LEGACY-NEXT: $L__BB8_1: // %atomicrmw.start
+; SM70-IEEE-LEGACY-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-IEEE-LEGACY-NEXT: shr.u32 %r8, %r23, %r1;
+; SM70-IEEE-LEGACY-NEXT: shl.b32 %r9, %r8, 16;
+; SM70-IEEE-LEGACY-NEXT: add.rn.f32 %r12, %r9, %r11;
+; SM70-IEEE-LEGACY-NEXT: bfe.u32 %r13, %r12, 16, 1;
+; SM70-IEEE-LEGACY-NEXT: add.s32 %r14, %r13, %r12;
+; SM70-IEEE-LEGACY-NEXT: add.s32 %r15, %r14, 32767;
+; SM70-IEEE-LEGACY-NEXT: setp.nan.f32 %p1, %r12, %r12;
+; SM70-IEEE-LEGACY-NEXT: or.b32 %r16, %r12, 4194304;
+; SM70-IEEE-LEGACY-NEXT: selp.b32 %r17, %r16, %r15, %p1;
+; SM70-IEEE-LEGACY-NEXT: shr.u32 %r18, %r17, 16;
+; SM70-IEEE-LEGACY-NEXT: shl.b32 %r19, %r18, %r1;
+; SM70-IEEE-LEGACY-NEXT: and.b32 %r20, %r23, %r2;
+; SM70-IEEE-LEGACY-NEXT: or.b32 %r21, %r20, %r19;
+; SM70-IEEE-LEGACY-NEXT: atom.relaxed.sys.shared.cas.b32 %r3, [%rd1], %r23, %r21;
+; SM70-IEEE-LEGACY-NEXT: setp.ne.b32 %p2, %r3, %r23;
+; SM70-IEEE-LEGACY-NEXT: mov.b32 %r23, %r3;
+; SM70-IEEE-LEGACY-NEXT: @%p2 bra $L__BB8_1;
+; SM70-IEEE-LEGACY-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-IEEE-LEGACY-NEXT: shr.u32 %r22, %r3, %r1;
+; SM70-IEEE-LEGACY-NEXT: st.param::func.b16 [func_retval0], %r22;
+; SM70-IEEE-LEGACY-NEXT: ret;
+;
+; SM70-IEEE-FAST-LABEL: fadd_bf16_shared(
+; SM70-IEEE-FAST: {
+; SM70-IEEE-FAST-NEXT: .reg .pred %p<3>;
+; SM70-IEEE-FAST-NEXT: .reg .b16 %rs<2>;
+; SM70-IEEE-FAST-NEXT: .reg .b32 %r<24>;
+; SM70-IEEE-FAST-NEXT: .reg .b64 %rd<3>;
+; SM70-IEEE-FAST-EMPTY:
+; SM70-IEEE-FAST-NEXT: // %bb.0:
+; SM70-IEEE-FAST-NEXT: ld.param::func.b16 %rs1, [fadd_bf16_shared_param_1];
+; SM70-IEEE-FAST-NEXT: ld.param::func.b64 %rd2, [fadd_bf16_shared_param_0];
+; SM70-IEEE-FAST-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-IEEE-FAST-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-IEEE-FAST-NEXT: and.b32 %r5, %r4, 3;
+; SM70-IEEE-FAST-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-IEEE-FAST-NEXT: mov.b32 %r6, 65535;
+; SM70-IEEE-FAST-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-IEEE-FAST-NEXT: not.b32 %r2, %r7;
+; SM70-IEEE-FAST-NEXT: ld.relaxed.sys.shared.b32 %r23, [%rd1];
+; SM70-IEEE-FAST-NEXT: cvt.u32.u16 %r10, %rs1;
+; SM70-IEEE-FAST-NEXT: shl.b32 %r11, %r10, 16;
+; SM70-IEEE-FAST-NEXT: $L__BB8_1: // %atomicrmw.start
+; SM70-IEEE-FAST-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-IEEE-FAST-NEXT: shr.u32 %r8, %r23, %r1;
+; SM70-IEEE-FAST-NEXT: shl.b32 %r9, %r8, 16;
+; SM70-IEEE-FAST-NEXT: add.rn.f32 %r12, %r9, %r11;
+; SM70-IEEE-FAST-NEXT: bfe.u32 %r13, %r12, 16, 1;
+; SM70-IEEE-FAST-NEXT: add.s32 %r14, %r13, %r12;
+; SM70-IEEE-FAST-NEXT: add.s32 %r15, %r14, 32767;
+; SM70-IEEE-FAST-NEXT: setp.nan.f32 %p1, %r12, %r12;
+; SM70-IEEE-FAST-NEXT: or.b32 %r16, %r12, 4194304;
+; SM70-IEEE-FAST-NEXT: selp.b32 %r17, %r16, %r15, %p1;
+; SM70-IEEE-FAST-NEXT: shr.u32 %r18, %r17, 16;
+; SM70-IEEE-FAST-NEXT: shl.b32 %r19, %r18, %r1;
+; SM70-IEEE-FAST-NEXT: and.b32 %r20, %r23, %r2;
+; SM70-IEEE-FAST-NEXT: or.b32 %r21, %r20, %r19;
+; SM70-IEEE-FAST-NEXT: atom.relaxed.sys.shared.cas.b32 %r3, [%rd1], %r23, %r21;
+; SM70-IEEE-FAST-NEXT: setp.ne.b32 %p2, %r3, %r23;
+; SM70-IEEE-FAST-NEXT: mov.b32 %r23, %r3;
+; SM70-IEEE-FAST-NEXT: @%p2 bra $L__BB8_1;
+; SM70-IEEE-FAST-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-IEEE-FAST-NEXT: shr.u32 %r22, %r3, %r1;
+; SM70-IEEE-FAST-NEXT: st.param::func.b16 [func_retval0], %r22;
+; SM70-IEEE-FAST-NEXT: ret;
+;
+; SM70-PRESERVESIGN-STRICT-LABEL: fadd_bf16_shared(
+; SM70-PRESERVESIGN-STRICT: {
+; SM70-PRESERVESIGN-STRICT-NEXT: .reg .pred %p<3>;
+; SM70-PRESERVESIGN-STRICT-NEXT: .reg .b16 %rs<2>;
+; SM70-PRESERVESIGN-STRICT-NEXT: .reg .b32 %r<24>;
+; SM70-PRESERVESIGN-STRICT-NEXT: .reg .b64 %rd<3>;
+; SM70-PRESERVESIGN-STRICT-EMPTY:
+; SM70-PRESERVESIGN-STRICT-NEXT: // %bb.0:
+; SM70-PRESERVESIGN-STRICT-NEXT: ld.param::func.b16 %rs1, [fadd_bf16_shared_param_1];
+; SM70-PRESERVESIGN-STRICT-NEXT: ld.param::func.b64 %rd2, [fadd_bf16_shared_param_0];
+; SM70-PRESERVESIGN-STRICT-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-PRESERVESIGN-STRICT-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-PRESERVESIGN-STRICT-NEXT: and.b32 %r5, %r4, 3;
+; SM70-PRESERVESIGN-STRICT-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-PRESERVESIGN-STRICT-NEXT: mov.b32 %r6, 65535;
+; SM70-PRESERVESIGN-STRICT-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-PRESERVESIGN-STRICT-NEXT: not.b32 %r2, %r7;
+; SM70-PRESERVESIGN-STRICT-NEXT: ld.relaxed.sys.shared.b32 %r23, [%rd1];
+; SM70-PRESERVESIGN-STRICT-NEXT: cvt.u32.u16 %r10, %rs1;
+; SM70-PRESERVESIGN-STRICT-NEXT: shl.b32 %r11, %r10, 16;
+; SM70-PRESERVESIGN-STRICT-NEXT: $L__BB8_1: // %atomicrmw.start
+; SM70-PRESERVESIGN-STRICT-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-PRESERVESIGN-STRICT-NEXT: shr.u32 %r8, %r23, %r1;
+; SM70-PRESERVESIGN-STRICT-NEXT: shl.b32 %r9, %r8, 16;
+; SM70-PRESERVESIGN-STRICT-NEXT: add.rn.ftz.f32 %r12, %r9, %r11;
+; SM70-PRESERVESIGN-STRICT-NEXT: bfe.u32 %r13, %r12, 16, 1;
+; SM70-PRESERVESIGN-STRICT-NEXT: add.s32 %r14, %r13, %r12;
+; SM70-PRESERVESIGN-STRICT-NEXT: add.s32 %r15, %r14, 32767;
+; SM70-PRESERVESIGN-STRICT-NEXT: setp.nan.ftz.f32 %p1, %r12, %r12;
+; SM70-PRESERVESIGN-STRICT-NEXT: or.b32 %r16, %r12, 4194304;
+; SM70-PRESERVESIGN-STRICT-NEXT: selp.b32 %r17, %r16, %r15, %p1;
+; SM70-PRESERVESIGN-STRICT-NEXT: shr.u32 %r18, %r17, 16;
+; SM70-PRESERVESIGN-STRICT-NEXT: shl.b32 %r19, %r18, %r1;
+; SM70-PRESERVESIGN-STRICT-NEXT: and.b32 %r20, %r23, %r2;
+; SM70-PRESERVESIGN-STRICT-NEXT: or.b32 %r21, %r20, %r19;
+; SM70-PRESERVESIGN-STRICT-NEXT: atom.relaxed.sys.shared.cas.b32 %r3, [%rd1], %r23, %r21;
+; SM70-PRESERVESIGN-STRICT-NEXT: setp.ne.b32 %p2, %r3, %r23;
+; SM70-PRESERVESIGN-STRICT-NEXT: mov.b32 %r23, %r3;
+; SM70-PRESERVESIGN-STRICT-NEXT: @%p2 bra $L__BB8_1;
+; SM70-PRESERVESIGN-STRICT-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-PRESERVESIGN-STRICT-NEXT: shr.u32 %r22, %r3, %r1;
+; SM70-PRESERVESIGN-STRICT-NEXT: st.param::func.b16 [func_retval0], %r22;
+; SM70-PRESERVESIGN-STRICT-NEXT: ret;
+;
+; SM70-PRESERVESIGN-LEGACY-LABEL: fadd_bf16_shared(
+; SM70-PRESERVESIGN-LEGACY: {
+; SM70-PRESERVESIGN-LEGACY-NEXT: .reg .pred %p<3>;
+; SM70-PRESERVESIGN-LEGACY-NEXT: .reg .b16 %rs<2>;
+; SM70-PRESERVESIGN-LEGACY-NEXT: .reg .b32 %r<24>;
+; SM70-PRESERVESIGN-LEGACY-NEXT: .reg .b64 %rd<3>;
+; SM70-PRESERVESIGN-LEGACY-EMPTY:
+; SM70-PRESERVESIGN-LEGACY-NEXT: // %bb.0:
+; SM70-PRESERVESIGN-LEGACY-NEXT: ld.param::func.b16 %rs1, [fadd_bf16_shared_param_1];
+; SM70-PRESERVESIGN-LEGACY-NEXT: ld.param::func.b64 %rd2, [fadd_bf16_shared_param_0];
+; SM70-PRESERVESIGN-LEGACY-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-PRESERVESIGN-LEGACY-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-PRESERVESIGN-LEGACY-NEXT: and.b32 %r5, %r4, 3;
+; SM70-PRESERVESIGN-LEGACY-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-PRESERVESIGN-LEGACY-NEXT: mov.b32 %r6, 65535;
+; SM70-PRESERVESIGN-LEGACY-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-PRESERVESIGN-LEGACY-NEXT: not.b32 %r2, %r7;
+; SM70-PRESERVESIGN-LEGACY-NEXT: ld.relaxed.sys.shared.b32 %r23, [%rd1];
+; SM70-PRESERVESIGN-LEGACY-NEXT: cvt.u32.u16 %r10, %rs1;
+; SM70-PRESERVESIGN-LEGACY-NEXT: shl.b32 %r11, %r10, 16;
+; SM70-PRESERVESIGN-LEGACY-NEXT: $L__BB8_1: // %atomicrmw.start
+; SM70-PRESERVESIGN-LEGACY-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-PRESERVESIGN-LEGACY-NEXT: shr.u32 %r8, %r23, %r1;
+; SM70-PRESERVESIGN-LEGACY-NEXT: shl.b32 %r9, %r8, 16;
+; SM70-PRESERVESIGN-LEGACY-NEXT: add.rn.ftz.f32 %r12, %r9, %r11;
+; SM70-PRESERVESIGN-LEGACY-NEXT: bfe.u32 %r13, %r12, 16, 1;
+; SM70-PRESERVESIGN-LEGACY-NEXT: add.s32 %r14, %r13, %r12;
+; SM70-PRESERVESIGN-LEGACY-NEXT: add.s32 %r15, %r14, 32767;
+; SM70-PRESERVESIGN-LEGACY-NEXT: setp.nan.ftz.f32 %p1, %r12, %r12;
+; SM70-PRESERVESIGN-LEGACY-NEXT: or.b32 %r16, %r12, 4194304;
+; SM70-PRESERVESIGN-LEGACY-NEXT: selp.b32 %r17, %r16, %r15, %p1;
+; SM70-PRESERVESIGN-LEGACY-NEXT: shr.u32 %r18, %r17, 16;
+; SM70-PRESERVESIGN-LEGACY-NEXT: shl.b32 %r19, %r18, %r1;
+; SM70-PRESERVESIGN-LEGACY-NEXT: and.b32 %r20, %r23, %r2;
+; SM70-PRESERVESIGN-LEGACY-NEXT: or.b32 %r21, %r20, %r19;
+; SM70-PRESERVESIGN-LEGACY-NEXT: atom.relaxed.sys.shared.cas.b32 %r3, [%rd1], %r23, %r21;
+; SM70-PRESERVESIGN-LEGACY-NEXT: setp.ne.b32 %p2, %r3, %r23;
+; SM70-PRESERVESIGN-LEGACY-NEXT: mov.b32 %r23, %r3;
+; SM70-PRESERVESIGN-LEGACY-NEXT: @%p2 bra $L__BB8_1;
+; SM70-PRESERVESIGN-LEGACY-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-PRESERVESIGN-LEGACY-NEXT: shr.u32 %r22, %r3, %r1;
+; SM70-PRESERVESIGN-LEGACY-NEXT: st.param::func.b16 [func_retval0], %r22;
+; SM70-PRESERVESIGN-LEGACY-NEXT: ret;
+;
+; SM70-PRESERVESIGN-FAST-LABEL: fadd_bf16_shared(
+; SM70-PRESERVESIGN-FAST: {
+; SM70-PRESERVESIGN-FAST-NEXT: .reg .pred %p<3>;
+; SM70-PRESERVESIGN-FAST-NEXT: .reg .b16 %rs<2>;
+; SM70-PRESERVESIGN-FAST-NEXT: .reg .b32 %r<24>;
+; SM70-PRESERVESIGN-FAST-NEXT: .reg .b64 %rd<3>;
+; SM70-PRESERVESIGN-FAST-EMPTY:
+; SM70-PRESERVESIGN-FAST-NEXT: // %bb.0:
+; SM70-PRESERVESIGN-FAST-NEXT: ld.param::func.b16 %rs1, [fadd_bf16_shared_param_1];
+; SM70-PRESERVESIGN-FAST-NEXT: ld.param::func.b64 %rd2, [fadd_bf16_shared_param_0];
+; SM70-PRESERVESIGN-FAST-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-PRESERVESIGN-FAST-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-PRESERVESIGN-FAST-NEXT: and.b32 %r5, %r4, 3;
+; SM70-PRESERVESIGN-FAST-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-PRESERVESIGN-FAST-NEXT: mov.b32 %r6, 65535;
+; SM70-PRESERVESIGN-FAST-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-PRESERVESIGN-FAST-NEXT: not.b32 %r2, %r7;
+; SM70-PRESERVESIGN-FAST-NEXT: ld.relaxed.sys.shared.b32 %r23, [%rd1];
+; SM70-PRESERVESIGN-FAST-NEXT: cvt.u32.u16 %r10, %rs1;
+; SM70-PRESERVESIGN-FAST-NEXT: shl.b32 %r11, %r10, 16;
+; SM70-PRESERVESIGN-FAST-NEXT: $L__BB8_1: // %atomicrmw.start
+; SM70-PRESERVESIGN-FAST-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-PRESERVESIGN-FAST-NEXT: shr.u32 %r8, %r23, %r1;
+; SM70-PRESERVESIGN-FAST-NEXT: shl.b32 %r9, %r8, 16;
+; SM70-PRESERVESIGN-FAST-NEXT: add.rn.ftz.f32 %r12, %r9, %r11;
+; SM70-PRESERVESIGN-FAST-NEXT: bfe.u32 %r13, %r12, 16, 1;
+; SM70-PRESERVESIGN-FAST-NEXT: add.s32 %r14, %r13, %r12;
+; SM70-PRESERVESIGN-FAST-NEXT: add.s32 %r15, %r14, 32767;
+; SM70-PRESERVESIGN-FAST-NEXT: setp.nan.ftz.f32 %p1, %r12, %r12;
+; SM70-PRESERVESIGN-FAST-NEXT: or.b32 %r16, %r12, 4194304;
+; SM70-PRESERVESIGN-FAST-NEXT: selp.b32 %r17, %r16, %r15, %p1;
+; SM70-PRESERVESIGN-FAST-NEXT: shr.u32 %r18, %r17, 16;
+; SM70-PRESERVESIGN-FAST-NEXT: shl.b32 %r19, %r18, %r1;
+; SM70-PRESERVESIGN-FAST-NEXT: and.b32 %r20, %r23, %r2;
+; SM70-PRESERVESIGN-FAST-NEXT: or.b32 %r21, %r20, %r19;
+; SM70-PRESERVESIGN-FAST-NEXT: atom.relaxed.sys.shared.cas.b32 %r3, [%rd1], %r23, %r21;
+; SM70-PRESERVESIGN-FAST-NEXT: setp.ne.b32 %p2, %r3, %r23;
+; SM70-PRESERVESIGN-FAST-NEXT: mov.b32 %r23, %r3;
+; SM70-PRESERVESIGN-FAST-NEXT: @%p2 bra $L__BB8_1;
+; SM70-PRESERVESIGN-FAST-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-PRESERVESIGN-FAST-NEXT: shr.u32 %r22, %r3, %r1;
+; SM70-PRESERVESIGN-FAST-NEXT: st.param::func.b16 [func_retval0], %r22;
+; SM70-PRESERVESIGN-FAST-NEXT: ret;
+;
+; SM90-IEEE-STRICT-LABEL: fadd_bf16_shared(
+; SM90-IEEE-STRICT: {
+; SM90-IEEE-STRICT-NEXT: .reg .b16 %rs<3>;
+; SM90-IEEE-STRICT-NEXT: .reg .b64 %rd<2>;
+; SM90-IEEE-STRICT-EMPTY:
+; SM90-IEEE-STRICT-NEXT: // %bb.0:
+; SM90-IEEE-STRICT-NEXT: ld.param::func.b64 %rd1, [fadd_bf16_shared_param_0];
+; SM90-IEEE-STRICT-NEXT: ld.param::func.b16 %rs1, [fadd_bf16_shared_param_1];
+; SM90-IEEE-STRICT-NEXT: atom.relaxed.sys.shared.add.noftz.bf16 %rs2, [%rd1], %rs1;
+; SM90-IEEE-STRICT-NEXT: st.param::func.b16 [func_retval0], %rs2;
+; SM90-IEEE-STRICT-NEXT: ret;
+;
+; SM90-IEEE-LEGACY-LABEL: fadd_bf16_shared(
+; SM90-IEEE-LEGACY: {
+; SM90-IEEE-LEGACY-NEXT: .reg .b16 %rs<3>;
+; SM90-IEEE-LEGACY-NEXT: .reg .b64 %rd<2>;
+; SM90-IEEE-LEGACY-EMPTY:
+; SM90-IEEE-LEGACY-NEXT: // %bb.0:
+; SM90-IEEE-LEGACY-NEXT: ld.param::func.b64 %rd1, [fadd_bf16_shared_param_0];
+; SM90-IEEE-LEGACY-NEXT: ld.param::func.b16 %rs1, [fadd_bf16_shared_param_1];
+; SM90-IEEE-LEGACY-NEXT: atom.relaxed.sys.shared.add.noftz.bf16 %rs2, [%rd1], %rs1;
+; SM90-IEEE-LEGACY-NEXT: st.param::func.b16 [func_retval0], %rs2;
+; SM90-IEEE-LEGACY-NEXT: ret;
+;
+; SM90-IEEE-FAST-LABEL: fadd_bf16_shared(
+; SM90-IEEE-FAST: {
+; SM90-IEEE-FAST-NEXT: .reg .b16 %rs<3>;
+; SM90-IEEE-FAST-NEXT: .reg .b64 %rd<2>;
+; SM90-IEEE-FAST-EMPTY:
+; SM90-IEEE-FAST-NEXT: // %bb.0:
+; SM90-IEEE-FAST-NEXT: ld.param::func.b64 %rd1, [fadd_bf16_shared_param_0];
+; SM90-IEEE-FAST-NEXT: ld.param::func.b16 %rs1, [fadd_bf16_shared_param_1];
+; SM90-IEEE-FAST-NEXT: atom.relaxed.sys.shared.add.noftz.bf16 %rs2, [%rd1], %rs1;
+; SM90-IEEE-FAST-NEXT: st.param::func.b16 [func_retval0], %rs2;
+; SM90-IEEE-FAST-NEXT: ret;
+;
+; SM90-PRESERVESIGN-STRICT-LABEL: fadd_bf16_shared(
+; SM90-PRESERVESIGN-STRICT: {
+; SM90-PRESERVESIGN-STRICT-NEXT: .reg .b16 %rs<3>;
+; SM90-PRESERVESIGN-STRICT-NEXT: .reg .b64 %rd<2>;
+; SM90-PRESERVESIGN-STRICT-EMPTY:
+; SM90-PRESERVESIGN-STRICT-NEXT: // %bb.0:
+; SM90-PRESERVESIGN-STRICT-NEXT: ld.param::func.b64 %rd1, [fadd_bf16_shared_param_0];
+; SM90-PRESERVESIGN-STRICT-NEXT: ld.param::func.b16 %rs1, [fadd_bf16_shared_param_1];
+; SM90-PRESERVESIGN-STRICT-NEXT: atom.relaxed.sys.shared.add.noftz.bf16 %rs2, [%rd1], %rs1;
+; SM90-PRESERVESIGN-STRICT-NEXT: st.param::func.b16 [func_retval0], %rs2;
+; SM90-PRESERVESIGN-STRICT-NEXT: ret;
+;
+; SM90-PRESERVESIGN-LEGACY-LABEL: fadd_bf16_shared(
+; SM90-PRESERVESIGN-LEGACY: {
+; SM90-PRESERVESIGN-LEGACY-NEXT: .reg .b16 %rs<3>;
+; SM90-PRESERVESIGN-LEGACY-NEXT: .reg .b64 %rd<2>;
+; SM90-PRESERVESIGN-LEGACY-EMPTY:
+; SM90-PRESERVESIGN-LEGACY-NEXT: // %bb.0:
+; SM90-PRESERVESIGN-LEGACY-NEXT: ld.param::func.b64 %rd1, [fadd_bf16_shared_param_0];
+; SM90-PRESERVESIGN-LEGACY-NEXT: ld.param::func.b16 %rs1, [fadd_bf16_shared_param_1];
+; SM90-PRESERVESIGN-LEGACY-NEXT: atom.relaxed.sys.shared.add.noftz.bf16 %rs2, [%rd1], %rs1;
+; SM90-PRESERVESIGN-LEGACY-NEXT: st.param::func.b16 [func_retval0], %rs2;
+; SM90-PRESERVESIGN-LEGACY-NEXT: ret;
+;
+; SM90-PRESERVESIGN-FAST-LABEL: fadd_bf16_shared(
+; SM90-PRESERVESIGN-FAST: {
+; SM90-PRESERVESIGN-FAST-NEXT: .reg .b16 %rs<3>;
+; SM90-PRESERVESIGN-FAST-NEXT: .reg .b64 %rd<2>;
+; SM90-PRESERVESIGN-FAST-EMPTY:
+; SM90-PRESERVESIGN-FAST-NEXT: // %bb.0:
+; SM90-PRESERVESIGN-FAST-NEXT: ld.param::func.b64 %rd1, [fadd_bf16_shared_param_0];
+; SM90-PRESERVESIGN-FAST-NEXT: ld.param::func.b16 %rs1, [fadd_bf16_shared_param_1];
+; SM90-PRESERVESIGN-FAST-NEXT: atom.relaxed.sys.shared.add.noftz.bf16 %rs2, [%rd1], %rs1;
+; SM90-PRESERVESIGN-FAST-NEXT: st.param::func.b16 [func_retval0], %rs2;
+; SM90-PRESERVESIGN-FAST-NEXT: ret;
+ %r = atomicrmw fadd ptr addrspace(3) %addr, bfloat %val monotonic
+ ret bfloat %r
+}
+
+; f64: likewise, neither atom.add.f64 nor add.rn.f64 has a flushing form.
+define double @fadd_f64_global(ptr addrspace(1) %addr, double %val) {
+; SM70-IEEE-STRICT-LABEL: fadd_f64_global(
+; SM70-IEEE-STRICT: {
+; SM70-IEEE-STRICT-NEXT: .reg .b64 %rd<4>;
+; SM70-IEEE-STRICT-EMPTY:
+; SM70-IEEE-STRICT-NEXT: // %bb.0:
+; SM70-IEEE-STRICT-NEXT: ld.param::func.b64 %rd1, [fadd_f64_global_param_0];
+; SM70-IEEE-STRICT-NEXT: ld.param::func.b64 %rd2, [fadd_f64_global_param_1];
+; SM70-IEEE-STRICT-NEXT: atom.relaxed.sys.global.add.f64 %rd3, [%rd1], %rd2;
+; SM70-IEEE-STRICT-NEXT: st.param::func.b64 [func_retval0], %rd3;
+; SM70-IEEE-STRICT-NEXT: ret;
+;
+; SM70-IEEE-LEGACY-LABEL: fadd_f64_global(
+; SM70-IEEE-LEGACY: {
+; SM70-IEEE-LEGACY-NEXT: .reg .b64 %rd<4>;
+; SM70-IEEE-LEGACY-EMPTY:
+; SM70-IEEE-LEGACY-NEXT: // %bb.0:
+; SM70-IEEE-LEGACY-NEXT: ld.param::func.b64 %rd1, [fadd_f64_global_param_0];
+; SM70-IEEE-LEGACY-NEXT: ld.param::func.b64 %rd2, [fadd_f64_global_param_1];
+; SM70-IEEE-LEGACY-NEXT: atom.relaxed.sys.global.add.f64 %rd3, [%rd1], %rd2;
+; SM70-IEEE-LEGACY-NEXT: st.param::func.b64 [func_retval0], %rd3;
+; SM70-IEEE-LEGACY-NEXT: ret;
+;
+; SM70-IEEE-FAST-LABEL: fadd_f64_global(
+; SM70-IEEE-FAST: {
+; SM70-IEEE-FAST-NEXT: .reg .b64 %rd<4>;
+; SM70-IEEE-FAST-EMPTY:
+; SM70-IEEE-FAST-NEXT: // %bb.0:
+; SM70-IEEE-FAST-NEXT: ld.param::func.b64 %rd1, [fadd_f64_global_param_0];
+; SM70-IEEE-FAST-NEXT: ld.param::func.b64 %rd2, [fadd_f64_global_param_1];
+; SM70-IEEE-FAST-NEXT: atom.relaxed.sys.global.add.f64 %rd3, [%rd1], %rd2;
+; SM70-IEEE-FAST-NEXT: st.param::func.b64 [func_retval0], %rd3;
+; SM70-IEEE-FAST-NEXT: ret;
+;
+; SM70-PRESERVESIGN-STRICT-LABEL: fadd_f64_global(
+; SM70-PRESERVESIGN-STRICT: {
+; SM70-PRESERVESIGN-STRICT-NEXT: .reg .b64 %rd<4>;
+; SM70-PRESERVESIGN-STRICT-EMPTY:
+; SM70-PRESERVESIGN-STRICT-NEXT: // %bb.0:
+; SM70-PRESERVESIGN-STRICT-NEXT: ld.param::func.b64 %rd1, [fadd_f64_global_param_0];
+; SM70-PRESERVESIGN-STRICT-NEXT: ld.param::func.b64 %rd2, [fadd_f64_global_param_1];
+; SM70-PRESERVESIGN-STRICT-NEXT: atom.relaxed.sys.global.add.f64 %rd3, [%rd1], %rd2;
+; SM70-PRESERVESIGN-STRICT-NEXT: st.param::func.b64 [func_retval0], %rd3;
+; SM70-PRESERVESIGN-STRICT-NEXT: ret;
+;
+; SM70-PRESERVESIGN-LEGACY-LABEL: fadd_f64_global(
+; SM70-PRESERVESIGN-LEGACY: {
+; SM70-PRESERVESIGN-LEGACY-NEXT: .reg .b64 %rd<4>;
+; SM70-PRESERVESIGN-LEGACY-EMPTY:
+; SM70-PRESERVESIGN-LEGACY-NEXT: // %bb.0:
+; SM70-PRESERVESIGN-LEGACY-NEXT: ld.param::func.b64 %rd1, [fadd_f64_global_param_0];
+; SM70-PRESERVESIGN-LEGACY-NEXT: ld.param::func.b64 %rd2, [fadd_f64_global_param_1];
+; SM70-PRESERVESIGN-LEGACY-NEXT: atom.relaxed.sys.global.add.f64 %rd3, [%rd1], %rd2;
+; SM70-PRESERVESIGN-LEGACY-NEXT: st.param::func.b64 [func_retval0], %rd3;
+; SM70-PRESERVESIGN-LEGACY-NEXT: ret;
+;
+; SM70-PRESERVESIGN-FAST-LABEL: fadd_f64_global(
+; SM70-PRESERVESIGN-FAST: {
+; SM70-PRESERVESIGN-FAST-NEXT: .reg .b64 %rd<4>;
+; SM70-PRESERVESIGN-FAST-EMPTY:
+; SM70-PRESERVESIGN-FAST-NEXT: // %bb.0:
+; SM70-PRESERVESIGN-FAST-NEXT: ld.param::func.b64 %rd1, [fadd_f64_global_param_0];
+; SM70-PRESERVESIGN-FAST-NEXT: ld.param::func.b64 %rd2, [fadd_f64_global_param_1];
+; SM70-PRESERVESIGN-FAST-NEXT: atom.relaxed.sys.global.add.f64 %rd3, [%rd1], %rd2;
+; SM70-PRESERVESIGN-FAST-NEXT: st.param::func.b64 [func_retval0], %rd3;
+; SM70-PRESERVESIGN-FAST-NEXT: ret;
+;
+; SM90-IEEE-STRICT-LABEL: fadd_f64_global(
+; SM90-IEEE-STRICT: {
+; SM90-IEEE-STRICT-NEXT: .reg .b64 %rd<4>;
+; SM90-IEEE-STRICT-EMPTY:
+; SM90-IEEE-STRICT-NEXT: // %bb.0:
+; SM90-IEEE-STRICT-NEXT: ld.param::func.b64 %rd1, [fadd_f64_global_param_0];
+; SM90-IEEE-STRICT-NEXT: ld.param::func.b64 %rd2, [fadd_f64_global_param_1];
+; SM90-IEEE-STRICT-NEXT: atom.relaxed.sys.global.add.f64 %rd3, [%rd1], %rd2;
+; SM90-IEEE-STRICT-NEXT: st.param::func.b64 [func_retval0], %rd3;
+; SM90-IEEE-STRICT-NEXT: ret;
+;
+; SM90-IEEE-LEGACY-LABEL: fadd_f64_global(
+; SM90-IEEE-LEGACY: {
+; SM90-IEEE-LEGACY-NEXT: .reg .b64 %rd<4>;
+; SM90-IEEE-LEGACY-EMPTY:
+; SM90-IEEE-LEGACY-NEXT: // %bb.0:
+; SM90-IEEE-LEGACY-NEXT: ld.param::func.b64 %rd1, [fadd_f64_global_param_0];
+; SM90-IEEE-LEGACY-NEXT: ld.param::func.b64 %rd2, [fadd_f64_global_param_1];
+; SM90-IEEE-LEGACY-NEXT: atom.relaxed.sys.global.add.f64 %rd3, [%rd1], %rd2;
+; SM90-IEEE-LEGACY-NEXT: st.param::func.b64 [func_retval0], %rd3;
+; SM90-IEEE-LEGACY-NEXT: ret;
+;
+; SM90-IEEE-FAST-LABEL: fadd_f64_global(
+; SM90-IEEE-FAST: {
+; SM90-IEEE-FAST-NEXT: .reg .b64 %rd<4>;
+; SM90-IEEE-FAST-EMPTY:
+; SM90-IEEE-FAST-NEXT: // %bb.0:
+; SM90-IEEE-FAST-NEXT: ld.param::func.b64 %rd1, [fadd_f64_global_param_0];
+; SM90-IEEE-FAST-NEXT: ld.param::func.b64 %rd2, [fadd_f64_global_param_1];
+; SM90-IEEE-FAST-NEXT: atom.relaxed.sys.global.add.f64 %rd3, [%rd1], %rd2;
+; SM90-IEEE-FAST-NEXT: st.param::func.b64 [func_retval0], %rd3;
+; SM90-IEEE-FAST-NEXT: ret;
+;
+; SM90-PRESERVESIGN-STRICT-LABEL: fadd_f64_global(
+; SM90-PRESERVESIGN-STRICT: {
+; SM90-PRESERVESIGN-STRICT-NEXT: .reg .b64 %rd<4>;
+; SM90-PRESERVESIGN-STRICT-EMPTY:
+; SM90-PRESERVESIGN-STRICT-NEXT: // %bb.0:
+; SM90-PRESERVESIGN-STRICT-NEXT: ld.param::func.b64 %rd1, [fadd_f64_global_param_0];
+; SM90-PRESERVESIGN-STRICT-NEXT: ld.param::func.b64 %rd2, [fadd_f64_global_param_1];
+; SM90-PRESERVESIGN-STRICT-NEXT: atom.relaxed.sys.global.add.f64 %rd3, [%rd1], %rd2;
+; SM90-PRESERVESIGN-STRICT-NEXT: st.param::func.b64 [func_retval0], %rd3;
+; SM90-PRESERVESIGN-STRICT-NEXT: ret;
+;
+; SM90-PRESERVESIGN-LEGACY-LABEL: fadd_f64_global(
+; SM90-PRESERVESIGN-LEGACY: {
+; SM90-PRESERVESIGN-LEGACY-NEXT: .reg .b64 %rd<4>;
+; SM90-PRESERVESIGN-LEGACY-EMPTY:
+; SM90-PRESERVESIGN-LEGACY-NEXT: // %bb.0:
+; SM90-PRESERVESIGN-LEGACY-NEXT: ld.param::func.b64 %rd1, [fadd_f64_global_param_0];
+; SM90-PRESERVESIGN-LEGACY-NEXT: ld.param::func.b64 %rd2, [fadd_f64_global_param_1];
+; SM90-PRESERVESIGN-LEGACY-NEXT: atom.relaxed.sys.global.add.f64 %rd3, [%rd1], %rd2;
+; SM90-PRESERVESIGN-LEGACY-NEXT: st.param::func.b64 [func_retval0], %rd3;
+; SM90-PRESERVESIGN-LEGACY-NEXT: ret;
+;
+; SM90-PRESERVESIGN-FAST-LABEL: fadd_f64_global(
+; SM90-PRESERVESIGN-FAST: {
+; SM90-PRESERVESIGN-FAST-NEXT: .reg .b64 %rd<4>;
+; SM90-PRESERVESIGN-FAST-EMPTY:
+; SM90-PRESERVESIGN-FAST-NEXT: // %bb.0:
+; SM90-PRESERVESIGN-FAST-NEXT: ld.param::func.b64 %rd1, [fadd_f64_global_param_0];
+; SM90-PRESERVESIGN-FAST-NEXT: ld.param::func.b64 %rd2, [fadd_f64_global_param_1];
+; SM90-PRESERVESIGN-FAST-NEXT: atom.relaxed.sys.global.add.f64 %rd3, [%rd1], %rd2;
+; SM90-PRESERVESIGN-FAST-NEXT: st.param::func.b64 [func_retval0], %rd3;
+; SM90-PRESERVESIGN-FAST-NEXT: ret;
+ %r = atomicrmw fadd ptr addrspace(1) %addr, double %val monotonic
+ ret double %r
+}
+
+define double @fadd_f64_generic(ptr %addr, double %val) {
+; SM70-IEEE-STRICT-LABEL: fadd_f64_generic(
+; SM70-IEEE-STRICT: {
+; SM70-IEEE-STRICT-NEXT: .reg .b64 %rd<4>;
+; SM70-IEEE-STRICT-EMPTY:
+; SM70-IEEE-STRICT-NEXT: // %bb.0:
+; SM70-IEEE-STRICT-NEXT: ld.param::func.b64 %rd1, [fadd_f64_generic_param_0];
+; SM70-IEEE-STRICT-NEXT: ld.param::func.b64 %rd2, [fadd_f64_generic_param_1];
+; SM70-IEEE-STRICT-NEXT: atom.relaxed.sys.add.f64 %rd3, [%rd1], %rd2;
+; SM70-IEEE-STRICT-NEXT: st.param::func.b64 [func_retval0], %rd3;
+; SM70-IEEE-STRICT-NEXT: ret;
+;
+; SM70-IEEE-LEGACY-LABEL: fadd_f64_generic(
+; SM70-IEEE-LEGACY: {
+; SM70-IEEE-LEGACY-NEXT: .reg .b64 %rd<4>;
+; SM70-IEEE-LEGACY-EMPTY:
+; SM70-IEEE-LEGACY-NEXT: // %bb.0:
+; SM70-IEEE-LEGACY-NEXT: ld.param::func.b64 %rd1, [fadd_f64_generic_param_0];
+; SM70-IEEE-LEGACY-NEXT: ld.param::func.b64 %rd2, [fadd_f64_generic_param_1];
+; SM70-IEEE-LEGACY-NEXT: atom.relaxed.sys.add.f64 %rd3, [%rd1], %rd2;
+; SM70-IEEE-LEGACY-NEXT: st.param::func.b64 [func_retval0], %rd3;
+; SM70-IEEE-LEGACY-NEXT: ret;
+;
+; SM70-IEEE-FAST-LABEL: fadd_f64_generic(
+; SM70-IEEE-FAST: {
+; SM70-IEEE-FAST-NEXT: .reg .b64 %rd<4>;
+; SM70-IEEE-FAST-EMPTY:
+; SM70-IEEE-FAST-NEXT: // %bb.0:
+; SM70-IEEE-FAST-NEXT: ld.param::func.b64 %rd1, [fadd_f64_generic_param_0];
+; SM70-IEEE-FAST-NEXT: ld.param::func.b64 %rd2, [fadd_f64_generic_param_1];
+; SM70-IEEE-FAST-NEXT: atom.relaxed.sys.add.f64 %rd3, [%rd1], %rd2;
+; SM70-IEEE-FAST-NEXT: st.param::func.b64 [func_retval0], %rd3;
+; SM70-IEEE-FAST-NEXT: ret;
+;
+; SM70-PRESERVESIGN-STRICT-LABEL: fadd_f64_generic(
+; SM70-PRESERVESIGN-STRICT: {
+; SM70-PRESERVESIGN-STRICT-NEXT: .reg .b64 %rd<4>;
+; SM70-PRESERVESIGN-STRICT-EMPTY:
+; SM70-PRESERVESIGN-STRICT-NEXT: // %bb.0:
+; SM70-PRESERVESIGN-STRICT-NEXT: ld.param::func.b64 %rd1, [fadd_f64_generic_param_0];
+; SM70-PRESERVESIGN-STRICT-NEXT: ld.param::func.b64 %rd2, [fadd_f64_generic_param_1];
+; SM70-PRESERVESIGN-STRICT-NEXT: atom.relaxed.sys.add.f64 %rd3, [%rd1], %rd2;
+; SM70-PRESERVESIGN-STRICT-NEXT: st.param::func.b64 [func_retval0], %rd3;
+; SM70-PRESERVESIGN-STRICT-NEXT: ret;
+;
+; SM70-PRESERVESIGN-LEGACY-LABEL: fadd_f64_generic(
+; SM70-PRESERVESIGN-LEGACY: {
+; SM70-PRESERVESIGN-LEGACY-NEXT: .reg .b64 %rd<4>;
+; SM70-PRESERVESIGN-LEGACY-EMPTY:
+; SM70-PRESERVESIGN-LEGACY-NEXT: // %bb.0:
+; SM70-PRESERVESIGN-LEGACY-NEXT: ld.param::func.b64 %rd1, [fadd_f64_generic_param_0];
+; SM70-PRESERVESIGN-LEGACY-NEXT: ld.param::func.b64 %rd2, [fadd_f64_generic_param_1];
+; SM70-PRESERVESIGN-LEGACY-NEXT: atom.relaxed.sys.add.f64 %rd3, [%rd1], %rd2;
+; SM70-PRESERVESIGN-LEGACY-NEXT: st.param::func.b64 [func_retval0], %rd3;
+; SM70-PRESERVESIGN-LEGACY-NEXT: ret;
+;
+; SM70-PRESERVESIGN-FAST-LABEL: fadd_f64_generic(
+; SM70-PRESERVESIGN-FAST: {
+; SM70-PRESERVESIGN-FAST-NEXT: .reg .b64 %rd<4>;
+; SM70-PRESERVESIGN-FAST-EMPTY:
+; SM70-PRESERVESIGN-FAST-NEXT: // %bb.0:
+; SM70-PRESERVESIGN-FAST-NEXT: ld.param::func.b64 %rd1, [fadd_f64_generic_param_0];
+; SM70-PRESERVESIGN-FAST-NEXT: ld.param::func.b64 %rd2, [fadd_f64_generic_param_1];
+; SM70-PRESERVESIGN-FAST-NEXT: atom.relaxed.sys.add.f64 %rd3, [%rd1], %rd2;
+; SM70-PRESERVESIGN-FAST-NEXT: st.param::func.b64 [func_retval0], %rd3;
+; SM70-PRESERVESIGN-FAST-NEXT: ret;
+;
+; SM90-IEEE-STRICT-LABEL: fadd_f64_generic(
+; SM90-IEEE-STRICT: {
+; SM90-IEEE-STRICT-NEXT: .reg .b64 %rd<4>;
+; SM90-IEEE-STRICT-EMPTY:
+; SM90-IEEE-STRICT-NEXT: // %bb.0:
+; SM90-IEEE-STRICT-NEXT: ld.param::func.b64 %rd1, [fadd_f64_generic_param_0];
+; SM90-IEEE-STRICT-NEXT: ld.param::func.b64 %rd2, [fadd_f64_generic_param_1];
+; SM90-IEEE-STRICT-NEXT: atom.relaxed.sys.add.f64 %rd3, [%rd1], %rd2;
+; SM90-IEEE-STRICT-NEXT: st.param::func.b64 [func_retval0], %rd3;
+; SM90-IEEE-STRICT-NEXT: ret;
+;
+; SM90-IEEE-LEGACY-LABEL: fadd_f64_generic(
+; SM90-IEEE-LEGACY: {
+; SM90-IEEE-LEGACY-NEXT: .reg .b64 %rd<4>;
+; SM90-IEEE-LEGACY-EMPTY:
+; SM90-IEEE-LEGACY-NEXT: // %bb.0:
+; SM90-IEEE-LEGACY-NEXT: ld.param::func.b64 %rd1, [fadd_f64_generic_param_0];
+; SM90-IEEE-LEGACY-NEXT: ld.param::func.b64 %rd2, [fadd_f64_generic_param_1];
+; SM90-IEEE-LEGACY-NEXT: atom.relaxed.sys.add.f64 %rd3, [%rd1], %rd2;
+; SM90-IEEE-LEGACY-NEXT: st.param::func.b64 [func_retval0], %rd3;
+; SM90-IEEE-LEGACY-NEXT: ret;
+;
+; SM90-IEEE-FAST-LABEL: fadd_f64_generic(
+; SM90-IEEE-FAST: {
+; SM90-IEEE-FAST-NEXT: .reg .b64 %rd<4>;
+; SM90-IEEE-FAST-EMPTY:
+; SM90-IEEE-FAST-NEXT: // %bb.0:
+; SM90-IEEE-FAST-NEXT: ld.param::func.b64 %rd1, [fadd_f64_generic_param_0];
+; SM90-IEEE-FAST-NEXT: ld.param::func.b64 %rd2, [fadd_f64_generic_param_1];
+; SM90-IEEE-FAST-NEXT: atom.relaxed.sys.add.f64 %rd3, [%rd1], %rd2;
+; SM90-IEEE-FAST-NEXT: st.param::func.b64 [func_retval0], %rd3;
+; SM90-IEEE-FAST-NEXT: ret;
+;
+; SM90-PRESERVESIGN-STRICT-LABEL: fadd_f64_generic(
+; SM90-PRESERVESIGN-STRICT: {
+; SM90-PRESERVESIGN-STRICT-NEXT: .reg .b64 %rd<4>;
+; SM90-PRESERVESIGN-STRICT-EMPTY:
+; SM90-PRESERVESIGN-STRICT-NEXT: // %bb.0:
+; SM90-PRESERVESIGN-STRICT-NEXT: ld.param::func.b64 %rd1, [fadd_f64_generic_param_0];
+; SM90-PRESERVESIGN-STRICT-NEXT: ld.param::func.b64 %rd2, [fadd_f64_generic_param_1];
+; SM90-PRESERVESIGN-STRICT-NEXT: atom.relaxed.sys.add.f64 %rd3, [%rd1], %rd2;
+; SM90-PRESERVESIGN-STRICT-NEXT: st.param::func.b64 [func_retval0], %rd3;
+; SM90-PRESERVESIGN-STRICT-NEXT: ret;
+;
+; SM90-PRESERVESIGN-LEGACY-LABEL: fadd_f64_generic(
+; SM90-PRESERVESIGN-LEGACY: {
+; SM90-PRESERVESIGN-LEGACY-NEXT: .reg .b64 %rd<4>;
+; SM90-PRESERVESIGN-LEGACY-EMPTY:
+; SM90-PRESERVESIGN-LEGACY-NEXT: // %bb.0:
+; SM90-PRESERVESIGN-LEGACY-NEXT: ld.param::func.b64 %rd1, [fadd_f64_generic_param_0];
+; SM90-PRESERVESIGN-LEGACY-NEXT: ld.param::func.b64 %rd2, [fadd_f64_generic_param_1];
+; SM90-PRESERVESIGN-LEGACY-NEXT: atom.relaxed.sys.add.f64 %rd3, [%rd1], %rd2;
+; SM90-PRESERVESIGN-LEGACY-NEXT: st.param::func.b64 [func_retval0], %rd3;
+; SM90-PRESERVESIGN-LEGACY-NEXT: ret;
+;
+; SM90-PRESERVESIGN-FAST-LABEL: fadd_f64_generic(
+; SM90-PRESERVESIGN-FAST: {
+; SM90-PRESERVESIGN-FAST-NEXT: .reg .b64 %rd<4>;
+; SM90-PRESERVESIGN-FAST-EMPTY:
+; SM90-PRESERVESIGN-FAST-NEXT: // %bb.0:
+; SM90-PRESERVESIGN-FAST-NEXT: ld.param::func.b64 %rd1, [fadd_f64_generic_param_0];
+; SM90-PRESERVESIGN-FAST-NEXT: ld.param::func.b64 %rd2, [fadd_f64_generic_param_1];
+; SM90-PRESERVESIGN-FAST-NEXT: atom.relaxed.sys.add.f64 %rd3, [%rd1], %rd2;
+; SM90-PRESERVESIGN-FAST-NEXT: st.param::func.b64 [func_retval0], %rd3;
+; SM90-PRESERVESIGN-FAST-NEXT: ret;
+ %r = atomicrmw fadd ptr %addr, double %val monotonic
+ ret double %r
+}
+
+define double @fadd_f64_shared(ptr addrspace(3) %addr, double %val) {
+; SM70-IEEE-STRICT-LABEL: fadd_f64_shared(
+; SM70-IEEE-STRICT: {
+; SM70-IEEE-STRICT-NEXT: .reg .b64 %rd<4>;
+; SM70-IEEE-STRICT-EMPTY:
+; SM70-IEEE-STRICT-NEXT: // %bb.0:
+; SM70-IEEE-STRICT-NEXT: ld.param::func.b64 %rd1, [fadd_f64_shared_param_0];
+; SM70-IEEE-STRICT-NEXT: ld.param::func.b64 %rd2, [fadd_f64_shared_param_1];
+; SM70-IEEE-STRICT-NEXT: atom.relaxed.sys.shared.add.f64 %rd3, [%rd1], %rd2;
+; SM70-IEEE-STRICT-NEXT: st.param::func.b64 [func_retval0], %rd3;
+; SM70-IEEE-STRICT-NEXT: ret;
+;
+; SM70-IEEE-LEGACY-LABEL: fadd_f64_shared(
+; SM70-IEEE-LEGACY: {
+; SM70-IEEE-LEGACY-NEXT: .reg .b64 %rd<4>;
+; SM70-IEEE-LEGACY-EMPTY:
+; SM70-IEEE-LEGACY-NEXT: // %bb.0:
+; SM70-IEEE-LEGACY-NEXT: ld.param::func.b64 %rd1, [fadd_f64_shared_param_0];
+; SM70-IEEE-LEGACY-NEXT: ld.param::func.b64 %rd2, [fadd_f64_shared_param_1];
+; SM70-IEEE-LEGACY-NEXT: atom.relaxed.sys.shared.add.f64 %rd3, [%rd1], %rd2;
+; SM70-IEEE-LEGACY-NEXT: st.param::func.b64 [func_retval0], %rd3;
+; SM70-IEEE-LEGACY-NEXT: ret;
+;
+; SM70-IEEE-FAST-LABEL: fadd_f64_shared(
+; SM70-IEEE-FAST: {
+; SM70-IEEE-FAST-NEXT: .reg .b64 %rd<4>;
+; SM70-IEEE-FAST-EMPTY:
+; SM70-IEEE-FAST-NEXT: // %bb.0:
+; SM70-IEEE-FAST-NEXT: ld.param::func.b64 %rd1, [fadd_f64_shared_param_0];
+; SM70-IEEE-FAST-NEXT: ld.param::func.b64 %rd2, [fadd_f64_shared_param_1];
+; SM70-IEEE-FAST-NEXT: atom.relaxed.sys.shared.add.f64 %rd3, [%rd1], %rd2;
+; SM70-IEEE-FAST-NEXT: st.param::func.b64 [func_retval0], %rd3;
+; SM70-IEEE-FAST-NEXT: ret;
+;
+; SM70-PRESERVESIGN-STRICT-LABEL: fadd_f64_shared(
+; SM70-PRESERVESIGN-STRICT: {
+; SM70-PRESERVESIGN-STRICT-NEXT: .reg .b64 %rd<4>;
+; SM70-PRESERVESIGN-STRICT-EMPTY:
+; SM70-PRESERVESIGN-STRICT-NEXT: // %bb.0:
+; SM70-PRESERVESIGN-STRICT-NEXT: ld.param::func.b64 %rd1, [fadd_f64_shared_param_0];
+; SM70-PRESERVESIGN-STRICT-NEXT: ld.param::func.b64 %rd2, [fadd_f64_shared_param_1];
+; SM70-PRESERVESIGN-STRICT-NEXT: atom.relaxed.sys.shared.add.f64 %rd3, [%rd1], %rd2;
+; SM70-PRESERVESIGN-STRICT-NEXT: st.param::func.b64 [func_retval0], %rd3;
+; SM70-PRESERVESIGN-STRICT-NEXT: ret;
+;
+; SM70-PRESERVESIGN-LEGACY-LABEL: fadd_f64_shared(
+; SM70-PRESERVESIGN-LEGACY: {
+; SM70-PRESERVESIGN-LEGACY-NEXT: .reg .b64 %rd<4>;
+; SM70-PRESERVESIGN-LEGACY-EMPTY:
+; SM70-PRESERVESIGN-LEGACY-NEXT: // %bb.0:
+; SM70-PRESERVESIGN-LEGACY-NEXT: ld.param::func.b64 %rd1, [fadd_f64_shared_param_0];
+; SM70-PRESERVESIGN-LEGACY-NEXT: ld.param::func.b64 %rd2, [fadd_f64_shared_param_1];
+; SM70-PRESERVESIGN-LEGACY-NEXT: atom.relaxed.sys.shared.add.f64 %rd3, [%rd1], %rd2;
+; SM70-PRESERVESIGN-LEGACY-NEXT: st.param::func.b64 [func_retval0], %rd3;
+; SM70-PRESERVESIGN-LEGACY-NEXT: ret;
+;
+; SM70-PRESERVESIGN-FAST-LABEL: fadd_f64_shared(
+; SM70-PRESERVESIGN-FAST: {
+; SM70-PRESERVESIGN-FAST-NEXT: .reg .b64 %rd<4>;
+; SM70-PRESERVESIGN-FAST-EMPTY:
+; SM70-PRESERVESIGN-FAST-NEXT: // %bb.0:
+; SM70-PRESERVESIGN-FAST-NEXT: ld.param::func.b64 %rd1, [fadd_f64_shared_param_0];
+; SM70-PRESERVESIGN-FAST-NEXT: ld.param::func.b64 %rd2, [fadd_f64_shared_param_1];
+; SM70-PRESERVESIGN-FAST-NEXT: atom.relaxed.sys.shared.add.f64 %rd3, [%rd1], %rd2;
+; SM70-PRESERVESIGN-FAST-NEXT: st.param::func.b64 [func_retval0], %rd3;
+; SM70-PRESERVESIGN-FAST-NEXT: ret;
+;
+; SM90-IEEE-STRICT-LABEL: fadd_f64_shared(
+; SM90-IEEE-STRICT: {
+; SM90-IEEE-STRICT-NEXT: .reg .b64 %rd<4>;
+; SM90-IEEE-STRICT-EMPTY:
+; SM90-IEEE-STRICT-NEXT: // %bb.0:
+; SM90-IEEE-STRICT-NEXT: ld.param::func.b64 %rd1, [fadd_f64_shared_param_0];
+; SM90-IEEE-STRICT-NEXT: ld.param::func.b64 %rd2, [fadd_f64_shared_param_1];
+; SM90-IEEE-STRICT-NEXT: atom.relaxed.sys.shared.add.f64 %rd3, [%rd1], %rd2;
+; SM90-IEEE-STRICT-NEXT: st.param::func.b64 [func_retval0], %rd3;
+; SM90-IEEE-STRICT-NEXT: ret;
+;
+; SM90-IEEE-LEGACY-LABEL: fadd_f64_shared(
+; SM90-IEEE-LEGACY: {
+; SM90-IEEE-LEGACY-NEXT: .reg .b64 %rd<4>;
+; SM90-IEEE-LEGACY-EMPTY:
+; SM90-IEEE-LEGACY-NEXT: // %bb.0:
+; SM90-IEEE-LEGACY-NEXT: ld.param::func.b64 %rd1, [fadd_f64_shared_param_0];
+; SM90-IEEE-LEGACY-NEXT: ld.param::func.b64 %rd2, [fadd_f64_shared_param_1];
+; SM90-IEEE-LEGACY-NEXT: atom.relaxed.sys.shared.add.f64 %rd3, [%rd1], %rd2;
+; SM90-IEEE-LEGACY-NEXT: st.param::func.b64 [func_retval0], %rd3;
+; SM90-IEEE-LEGACY-NEXT: ret;
+;
+; SM90-IEEE-FAST-LABEL: fadd_f64_shared(
+; SM90-IEEE-FAST: {
+; SM90-IEEE-FAST-NEXT: .reg .b64 %rd<4>;
+; SM90-IEEE-FAST-EMPTY:
+; SM90-IEEE-FAST-NEXT: // %bb.0:
+; SM90-IEEE-FAST-NEXT: ld.param::func.b64 %rd1, [fadd_f64_shared_param_0];
+; SM90-IEEE-FAST-NEXT: ld.param::func.b64 %rd2, [fadd_f64_shared_param_1];
+; SM90-IEEE-FAST-NEXT: atom.relaxed.sys.shared.add.f64 %rd3, [%rd1], %rd2;
+; SM90-IEEE-FAST-NEXT: st.param::func.b64 [func_retval0], %rd3;
+; SM90-IEEE-FAST-NEXT: ret;
+;
+; SM90-PRESERVESIGN-STRICT-LABEL: fadd_f64_shared(
+; SM90-PRESERVESIGN-STRICT: {
+; SM90-PRESERVESIGN-STRICT-NEXT: .reg .b64 %rd<4>;
+; SM90-PRESERVESIGN-STRICT-EMPTY:
+; SM90-PRESERVESIGN-STRICT-NEXT: // %bb.0:
+; SM90-PRESERVESIGN-STRICT-NEXT: ld.param::func.b64 %rd1, [fadd_f64_shared_param_0];
+; SM90-PRESERVESIGN-STRICT-NEXT: ld.param::func.b64 %rd2, [fadd_f64_shared_param_1];
+; SM90-PRESERVESIGN-STRICT-NEXT: atom.relaxed.sys.shared.add.f64 %rd3, [%rd1], %rd2;
+; SM90-PRESERVESIGN-STRICT-NEXT: st.param::func.b64 [func_retval0], %rd3;
+; SM90-PRESERVESIGN-STRICT-NEXT: ret;
+;
+; SM90-PRESERVESIGN-LEGACY-LABEL: fadd_f64_shared(
+; SM90-PRESERVESIGN-LEGACY: {
+; SM90-PRESERVESIGN-LEGACY-NEXT: .reg .b64 %rd<4>;
+; SM90-PRESERVESIGN-LEGACY-EMPTY:
+; SM90-PRESERVESIGN-LEGACY-NEXT: // %bb.0:
+; SM90-PRESERVESIGN-LEGACY-NEXT: ld.param::func.b64 %rd1, [fadd_f64_shared_param_0];
+; SM90-PRESERVESIGN-LEGACY-NEXT: ld.param::func.b64 %rd2, [fadd_f64_shared_param_1];
+; SM90-PRESERVESIGN-LEGACY-NEXT: atom.relaxed.sys.shared.add.f64 %rd3, [%rd1], %rd2;
+; SM90-PRESERVESIGN-LEGACY-NEXT: st.param::func.b64 [func_retval0], %rd3;
+; SM90-PRESERVESIGN-LEGACY-NEXT: ret;
+;
+; SM90-PRESERVESIGN-FAST-LABEL: fadd_f64_shared(
+; SM90-PRESERVESIGN-FAST: {
+; SM90-PRESERVESIGN-FAST-NEXT: .reg .b64 %rd<4>;
+; SM90-PRESERVESIGN-FAST-EMPTY:
+; SM90-PRESERVESIGN-FAST-NEXT: // %bb.0:
+; SM90-PRESERVESIGN-FAST-NEXT: ld.param::func.b64 %rd1, [fadd_f64_shared_param_0];
+; SM90-PRESERVESIGN-FAST-NEXT: ld.param::func.b64 %rd2, [fadd_f64_shared_param_1];
+; SM90-PRESERVESIGN-FAST-NEXT: atom.relaxed.sys.shared.add.f64 %rd3, [%rd1], %rd2;
+; SM90-PRESERVESIGN-FAST-NEXT: st.param::func.b64 [func_retval0], %rd3;
+; SM90-PRESERVESIGN-FAST-NEXT: ret;
+ %r = atomicrmw fadd ptr addrspace(3) %addr, double %val monotonic
+ ret double %r
+}
diff --git a/llvm/test/CodeGen/NVPTX/atomicrmw-sm60.ll b/llvm/test/CodeGen/NVPTX/atomicrmw-sm60.ll
index 5a43d78501ae0..63ad138d7ac5f 100644
--- a/llvm/test/CodeGen/NVPTX/atomicrmw-sm60.ll
+++ b/llvm/test/CodeGen/NVPTX/atomicrmw-sm60.ll
@@ -1,15 +1,15 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
-; RUN: llc < %s -march=nvptx64 -mcpu=sm_60 -mattr=+ptx50 -nvptx-allow-ftz-atomics=false | FileCheck %s --check-prefixes=SM60,SM60-NOFTZ-DISALLOW
-; RUN: llc < %s -march=nvptx64 -mcpu=sm_60 -mattr=+ptx50 -nvptx-allow-ftz-atomics=true | FileCheck %s --check-prefixes=SM60,SM60-NOFTZ-ALLOW
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_60 -mattr=+ptx50 -nvptx-ftz-atomics=strict | FileCheck %s --check-prefixes=SM60,SM60-NOFTZ-DISALLOW
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_60 -mattr=+ptx50 -nvptx-ftz-atomics=fast | FileCheck %s --check-prefixes=SM60,SM60-NOFTZ-ALLOW
; RUN: llc < %s -march=nvptx64 -mcpu=sm_60 -mattr=+ptx50 | FileCheck %s --check-prefixes=SM60,SM60-NOFTZ-ALLOW
-; RUN: llc < %s -march=nvptx64 -mcpu=sm_60 -mattr=+ptx50 -denormal-fp-math-f32=preserve-sign -nvptx-allow-ftz-atomics=false | FileCheck %s --check-prefixes=SM60,SM60-FTZ-DISALLOW
-; RUN: llc < %s -march=nvptx64 -mcpu=sm_60 -mattr=+ptx50 -denormal-fp-math-f32=preserve-sign -nvptx-allow-ftz-atomics=true | FileCheck %s --check-prefixes=SM60,SM60-FTZ-ALLOW
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_60 -mattr=+ptx50 -denormal-fp-math-f32=preserve-sign -nvptx-ftz-atomics=strict | FileCheck %s --check-prefixes=SM60,SM60-FTZ-DISALLOW
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_60 -mattr=+ptx50 -denormal-fp-math-f32=preserve-sign -nvptx-ftz-atomics=fast | FileCheck %s --check-prefixes=SM60,SM60-FTZ-ALLOW
; RUN: llc < %s -march=nvptx64 -mcpu=sm_60 -mattr=+ptx50 -denormal-fp-math-f32=preserve-sign | FileCheck %s --check-prefixes=SM60,SM60-FTZ-ALLOW
-; RUN: %if ptxas-sm_60 && ptxas-isa-5.0 %{ llc < %s -march=nvptx64 -mcpu=sm_60 -mattr=+ptx50 -nvptx-allow-ftz-atomics=false | %ptxas-verify -arch=sm_60 %}
-; RUN: %if ptxas-sm_60 && ptxas-isa-5.0 %{ llc < %s -march=nvptx64 -mcpu=sm_60 -mattr=+ptx50 -nvptx-allow-ftz-atomics=true | %ptxas-verify -arch=sm_60 %}
+; RUN: %if ptxas-sm_60 && ptxas-isa-5.0 %{ llc < %s -march=nvptx64 -mcpu=sm_60 -mattr=+ptx50 -nvptx-ftz-atomics=strict | %ptxas-verify -arch=sm_60 %}
+; RUN: %if ptxas-sm_60 && ptxas-isa-5.0 %{ llc < %s -march=nvptx64 -mcpu=sm_60 -mattr=+ptx50 -nvptx-ftz-atomics=fast | %ptxas-verify -arch=sm_60 %}
; RUN: %if ptxas-sm_60 && ptxas-isa-5.0 %{ llc < %s -march=nvptx64 -mcpu=sm_60 -mattr=+ptx50 | %ptxas-verify -arch=sm_60 %}
-; RUN: %if ptxas-sm_60 && ptxas-isa-5.0 %{ llc < %s -march=nvptx64 -mcpu=sm_60 -mattr=+ptx50 -denormal-fp-math-f32=preserve-sign -nvptx-allow-ftz-atomics=false | %ptxas-verify -arch=sm_60 %}
-; RUN: %if ptxas-sm_60 && ptxas-isa-5.0 %{ llc < %s -march=nvptx64 -mcpu=sm_60 -mattr=+ptx50 -denormal-fp-math-f32=preserve-sign -nvptx-allow-ftz-atomics=true | %ptxas-verify -arch=sm_60 %}
+; RUN: %if ptxas-sm_60 && ptxas-isa-5.0 %{ llc < %s -march=nvptx64 -mcpu=sm_60 -mattr=+ptx50 -denormal-fp-math-f32=preserve-sign -nvptx-ftz-atomics=strict | %ptxas-verify -arch=sm_60 %}
+; RUN: %if ptxas-sm_60 && ptxas-isa-5.0 %{ llc < %s -march=nvptx64 -mcpu=sm_60 -mattr=+ptx50 -denormal-fp-math-f32=preserve-sign -nvptx-ftz-atomics=fast | %ptxas-verify -arch=sm_60 %}
; RUN: %if ptxas-sm_60 && ptxas-isa-5.0 %{ llc < %s -march=nvptx64 -mcpu=sm_60 -mattr=+ptx50 -denormal-fp-math-f32=preserve-sign | %ptxas-verify -arch=sm_60 %}
define i8 @xchg_acq_rel_i8_global_cta(ptr addrspace(1) %addr, i8 %val) {
diff --git a/llvm/test/CodeGen/NVPTX/atomicrmw-sm70.ll b/llvm/test/CodeGen/NVPTX/atomicrmw-sm70.ll
index 5b9f1a68bceab..2d91ac35fe804 100644
--- a/llvm/test/CodeGen/NVPTX/atomicrmw-sm70.ll
+++ b/llvm/test/CodeGen/NVPTX/atomicrmw-sm70.ll
@@ -1,15 +1,15 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
-; RUN: llc < %s -march=nvptx64 -mcpu=sm_70 -mattr=+ptx63 -nvptx-allow-ftz-atomics=false | FileCheck %s --check-prefixes=SM70,SM70-NOFTZ-DISALLOW
-; RUN: llc < %s -march=nvptx64 -mcpu=sm_70 -mattr=+ptx63 -nvptx-allow-ftz-atomics=true | FileCheck %s --check-prefixes=SM70,SM70-NOFTZ-ALLOW
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_70 -mattr=+ptx63 -nvptx-ftz-atomics=strict | FileCheck %s --check-prefixes=SM70,SM70-NOFTZ-DISALLOW
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_70 -mattr=+ptx63 -nvptx-ftz-atomics=fast | FileCheck %s --check-prefixes=SM70,SM70-NOFTZ-ALLOW
; RUN: llc < %s -march=nvptx64 -mcpu=sm_70 -mattr=+ptx63 | FileCheck %s --check-prefixes=SM70,SM70-NOFTZ-ALLOW
-; RUN: llc < %s -march=nvptx64 -mcpu=sm_70 -mattr=+ptx63 -denormal-fp-math-f32=preserve-sign -nvptx-allow-ftz-atomics=false | FileCheck %s --check-prefixes=SM70,SM70-FTZ-DISALLOW
-; RUN: llc < %s -march=nvptx64 -mcpu=sm_70 -mattr=+ptx63 -denormal-fp-math-f32=preserve-sign -nvptx-allow-ftz-atomics=true | FileCheck %s --check-prefixes=SM70,SM70-FTZ-ALLOW
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_70 -mattr=+ptx63 -denormal-fp-math-f32=preserve-sign -nvptx-ftz-atomics=strict | FileCheck %s --check-prefixes=SM70,SM70-FTZ-DISALLOW
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_70 -mattr=+ptx63 -denormal-fp-math-f32=preserve-sign -nvptx-ftz-atomics=fast | FileCheck %s --check-prefixes=SM70,SM70-FTZ-ALLOW
; RUN: llc < %s -march=nvptx64 -mcpu=sm_70 -mattr=+ptx63 -denormal-fp-math-f32=preserve-sign | FileCheck %s --check-prefixes=SM70,SM70-FTZ-ALLOW
-; RUN: %if ptxas-sm_70 && ptxas-isa-6.3 %{ llc < %s -march=nvptx64 -mcpu=sm_70 -mattr=+ptx63 -nvptx-allow-ftz-atomics=false | %ptxas-verify -arch=sm_70 %}
-; RUN: %if ptxas-sm_70 && ptxas-isa-6.3 %{ llc < %s -march=nvptx64 -mcpu=sm_70 -mattr=+ptx63 -nvptx-allow-ftz-atomics=true | %ptxas-verify -arch=sm_70 %}
+; RUN: %if ptxas-sm_70 && ptxas-isa-6.3 %{ llc < %s -march=nvptx64 -mcpu=sm_70 -mattr=+ptx63 -nvptx-ftz-atomics=strict | %ptxas-verify -arch=sm_70 %}
+; RUN: %if ptxas-sm_70 && ptxas-isa-6.3 %{ llc < %s -march=nvptx64 -mcpu=sm_70 -mattr=+ptx63 -nvptx-ftz-atomics=fast | %ptxas-verify -arch=sm_70 %}
; RUN: %if ptxas-sm_70 && ptxas-isa-6.3 %{ llc < %s -march=nvptx64 -mcpu=sm_70 -mattr=+ptx63 | %ptxas-verify -arch=sm_70 %}
-; RUN: %if ptxas-sm_70 && ptxas-isa-6.3 %{ llc < %s -march=nvptx64 -mcpu=sm_70 -mattr=+ptx63 -denormal-fp-math-f32=preserve-sign -nvptx-allow-ftz-atomics=false | %ptxas-verify -arch=sm_70 %}
-; RUN: %if ptxas-sm_70 && ptxas-isa-6.3 %{ llc < %s -march=nvptx64 -mcpu=sm_70 -mattr=+ptx63 -denormal-fp-math-f32=preserve-sign -nvptx-allow-ftz-atomics=true | %ptxas-verify -arch=sm_70 %}
+; RUN: %if ptxas-sm_70 && ptxas-isa-6.3 %{ llc < %s -march=nvptx64 -mcpu=sm_70 -mattr=+ptx63 -denormal-fp-math-f32=preserve-sign -nvptx-ftz-atomics=strict | %ptxas-verify -arch=sm_70 %}
+; RUN: %if ptxas-sm_70 && ptxas-isa-6.3 %{ llc < %s -march=nvptx64 -mcpu=sm_70 -mattr=+ptx63 -denormal-fp-math-f32=preserve-sign -nvptx-ftz-atomics=fast | %ptxas-verify -arch=sm_70 %}
; RUN: %if ptxas-sm_70 && ptxas-isa-6.3 %{ llc < %s -march=nvptx64 -mcpu=sm_70 -mattr=+ptx63 -denormal-fp-math-f32=preserve-sign | %ptxas-verify -arch=sm_70 %}
define i8 @xchg_acq_rel_i8_global_cta(ptr addrspace(1) %addr, i8 %val) {
diff --git a/llvm/test/CodeGen/NVPTX/atomicrmw-sm90.ll b/llvm/test/CodeGen/NVPTX/atomicrmw-sm90.ll
index a1470453fa427..b8946861f32af 100644
--- a/llvm/test/CodeGen/NVPTX/atomicrmw-sm90.ll
+++ b/llvm/test/CodeGen/NVPTX/atomicrmw-sm90.ll
@@ -1,16 +1,16 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
-; RUN: llc < %s -march=nvptx64 -mcpu=sm_90 -mattr=+ptx87 -nvptx-allow-ftz-atomics=false | FileCheck %s --check-prefixes=SM90,SM90-NOFTZ-DISALLOW
-; RUN: llc < %s -march=nvptx64 -mcpu=sm_90 -mattr=+ptx87 -nvptx-allow-ftz-atomics=true | FileCheck %s --check-prefixes=SM90,SM90-NOFTZ-ALLOW
-; RUN: llc < %s -march=nvptx64 -mcpu=sm_90 -mattr=+ptx87 | FileCheck %s --check-prefixes=SM90,SM90-NOFTZ-ALLOW
-; RUN: llc < %s -march=nvptx64 -mcpu=sm_90 -mattr=+ptx87 -denormal-fp-math-f32=preserve-sign -nvptx-allow-ftz-atomics=false | FileCheck %s --check-prefixes=SM90,SM90-FTZ-DISALLOW
-; RUN: llc < %s -march=nvptx64 -mcpu=sm_90 -mattr=+ptx87 -denormal-fp-math-f32=preserve-sign -nvptx-allow-ftz-atomics=true | FileCheck %s --check-prefixes=SM90,SM90-FTZ-ALLOW
-; RUN: llc < %s -march=nvptx64 -mcpu=sm_90 -mattr=+ptx87 -denormal-fp-math-f32=preserve-sign | FileCheck %s --check-prefixes=SM90,SM90-FTZ-ALLOW
-; RUN: %if ptxas-sm_90 && ptxas-isa-8.7 %{ llc < %s -march=nvptx64 -mcpu=sm_90 -mattr=+ptx87 -nvptx-allow-ftz-atomics=false | %ptxas-verify -arch=sm_90 %}
-; RUN: %if ptxas-sm_90 && ptxas-isa-8.7 %{ llc < %s -march=nvptx64 -mcpu=sm_90 -mattr=+ptx87 -nvptx-allow-ftz-atomics=true | %ptxas-verify -arch=sm_90 %}
-; RUN: %if ptxas-sm_90 && ptxas-isa-8.7 %{ llc < %s -march=nvptx64 -mcpu=sm_90 -mattr=+ptx87 | %ptxas-verify -arch=sm_90 %}
-; RUN: %if ptxas-sm_90 && ptxas-isa-8.7 %{ llc < %s -march=nvptx64 -mcpu=sm_90 -mattr=+ptx87 -denormal-fp-math-f32=preserve-sign -nvptx-allow-ftz-atomics=false | %ptxas-verify -arch=sm_90 %}
-; RUN: %if ptxas-sm_90 && ptxas-isa-8.7 %{ llc < %s -march=nvptx64 -mcpu=sm_90 -mattr=+ptx87 -denormal-fp-math-f32=preserve-sign -nvptx-allow-ftz-atomics=true | %ptxas-verify -arch=sm_90 %}
-; RUN: %if ptxas-sm_90 && ptxas-isa-8.7 %{ llc < %s -march=nvptx64 -mcpu=sm_90 -mattr=+ptx87 -denormal-fp-math-f32=preserve-sign | %ptxas-verify -arch=sm_90 %}
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_90 -mattr=+ptx94 -nvptx-ftz-atomics=strict | FileCheck %s --check-prefixes=SM90,SM90-NOFTZ-DISALLOW
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_90 -mattr=+ptx94 -nvptx-ftz-atomics=fast | FileCheck %s --check-prefixes=SM90,SM90-NOFTZ-ALLOW
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_90 -mattr=+ptx94 | FileCheck %s --check-prefixes=SM90,SM90-NOFTZ-ALLOW
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_90 -mattr=+ptx94 -denormal-fp-math-f32=preserve-sign -nvptx-ftz-atomics=strict | FileCheck %s --check-prefixes=SM90,SM90-FTZ-DISALLOW
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_90 -mattr=+ptx94 -denormal-fp-math-f32=preserve-sign -nvptx-ftz-atomics=fast | FileCheck %s --check-prefixes=SM90,SM90-FTZ-ALLOW
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_90 -mattr=+ptx94 -denormal-fp-math-f32=preserve-sign | FileCheck %s --check-prefixes=SM90,SM90-FTZ-ALLOW
+; RUN: %if ptxas-sm_90 && ptxas-isa-9.4 %{ llc < %s -march=nvptx64 -mcpu=sm_90 -mattr=+ptx94 -nvptx-ftz-atomics=strict | %ptxas-verify -arch=sm_90 %}
+; RUN: %if ptxas-sm_90 && ptxas-isa-9.4 %{ llc < %s -march=nvptx64 -mcpu=sm_90 -mattr=+ptx94 -nvptx-ftz-atomics=fast | %ptxas-verify -arch=sm_90 %}
+; RUN: %if ptxas-sm_90 && ptxas-isa-9.4 %{ llc < %s -march=nvptx64 -mcpu=sm_90 -mattr=+ptx94 | %ptxas-verify -arch=sm_90 %}
+; RUN: %if ptxas-sm_90 && ptxas-isa-9.4 %{ llc < %s -march=nvptx64 -mcpu=sm_90 -mattr=+ptx94 -denormal-fp-math-f32=preserve-sign -nvptx-ftz-atomics=strict | %ptxas-verify -arch=sm_90 %}
+; RUN: %if ptxas-sm_90 && ptxas-isa-9.4 %{ llc < %s -march=nvptx64 -mcpu=sm_90 -mattr=+ptx94 -denormal-fp-math-f32=preserve-sign -nvptx-ftz-atomics=fast | %ptxas-verify -arch=sm_90 %}
+; RUN: %if ptxas-sm_90 && ptxas-isa-9.4 %{ llc < %s -march=nvptx64 -mcpu=sm_90 -mattr=+ptx94 -denormal-fp-math-f32=preserve-sign | %ptxas-verify -arch=sm_90 %}
define i8 @xchg_acq_rel_i8_global_cta(ptr addrspace(1) %addr, i8 %val) {
; SM90-LABEL: xchg_acq_rel_i8_global_cta(
@@ -1749,25 +1749,14 @@ define i64 @usub_sat_acq_rel_i64_global_cta(ptr addrspace(1) %addr, i64 %val) {
define float @fadd_acq_rel_float_global_cta(ptr addrspace(1) %addr, float %val) {
; SM90-NOFTZ-DISALLOW-LABEL: fadd_acq_rel_float_global_cta(
; SM90-NOFTZ-DISALLOW: {
-; SM90-NOFTZ-DISALLOW-NEXT: .reg .pred %p<2>;
-; SM90-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<5>;
+; SM90-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<3>;
; SM90-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
; SM90-NOFTZ-DISALLOW-EMPTY:
; SM90-NOFTZ-DISALLOW-NEXT: // %bb.0:
-; SM90-NOFTZ-DISALLOW-NEXT: ld.param::func.b32 %r2, [fadd_acq_rel_float_global_cta_param_1];
; SM90-NOFTZ-DISALLOW-NEXT: ld.param::func.b64 %rd1, [fadd_acq_rel_float_global_cta_param_0];
-; SM90-NOFTZ-DISALLOW-NEXT: fence.release.cta;
-; SM90-NOFTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
-; SM90-NOFTZ-DISALLOW-NEXT: $L__BB60_1: // %atomicrmw.start
-; SM90-NOFTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NOFTZ-DISALLOW-NEXT: add.rn.f32 %r3, %r4, %r2;
-; SM90-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
-; SM90-NOFTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
-; SM90-NOFTZ-DISALLOW-NEXT: mov.b32 %r4, %r1;
-; SM90-NOFTZ-DISALLOW-NEXT: @%p1 bra $L__BB60_1;
-; SM90-NOFTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NOFTZ-DISALLOW-NEXT: fence.acquire.cta;
-; SM90-NOFTZ-DISALLOW-NEXT: st.param::func.b32 [func_retval0], %r1;
+; SM90-NOFTZ-DISALLOW-NEXT: ld.param::func.b32 %r1, [fadd_acq_rel_float_global_cta_param_1];
+; SM90-NOFTZ-DISALLOW-NEXT: atom.acq_rel.cta.global.add.noftz.f32 %r2, [%rd1], %r1;
+; SM90-NOFTZ-DISALLOW-NEXT: st.param::func.b32 [func_retval0], %r2;
; SM90-NOFTZ-DISALLOW-NEXT: ret;
;
; SM90-NOFTZ-ALLOW-LABEL: fadd_acq_rel_float_global_cta(
@@ -1778,7 +1767,7 @@ define float @fadd_acq_rel_float_global_cta(ptr addrspace(1) %addr, float %val)
; SM90-NOFTZ-ALLOW-NEXT: // %bb.0:
; SM90-NOFTZ-ALLOW-NEXT: ld.param::func.b64 %rd1, [fadd_acq_rel_float_global_cta_param_0];
; SM90-NOFTZ-ALLOW-NEXT: ld.param::func.b32 %r1, [fadd_acq_rel_float_global_cta_param_1];
-; SM90-NOFTZ-ALLOW-NEXT: atom.acq_rel.cta.global.add.f32 %r2, [%rd1], %r1;
+; SM90-NOFTZ-ALLOW-NEXT: atom.acq_rel.cta.global.add.noftz.f32 %r2, [%rd1], %r1;
; SM90-NOFTZ-ALLOW-NEXT: st.param::func.b32 [func_retval0], %r2;
; SM90-NOFTZ-ALLOW-NEXT: ret;
;
@@ -4070,25 +4059,14 @@ define i64 @nand_acq_rel_i64_global_sys(ptr addrspace(1) %addr, i64 %val) {
define float @fadd_acq_rel_float_global_sys(ptr addrspace(1) %addr, float %val) {
; SM90-NOFTZ-DISALLOW-LABEL: fadd_acq_rel_float_global_sys(
; SM90-NOFTZ-DISALLOW: {
-; SM90-NOFTZ-DISALLOW-NEXT: .reg .pred %p<2>;
-; SM90-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<5>;
+; SM90-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<3>;
; SM90-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
; SM90-NOFTZ-DISALLOW-EMPTY:
; SM90-NOFTZ-DISALLOW-NEXT: // %bb.0:
-; SM90-NOFTZ-DISALLOW-NEXT: ld.param::func.b32 %r2, [fadd_acq_rel_float_global_sys_param_1];
; SM90-NOFTZ-DISALLOW-NEXT: ld.param::func.b64 %rd1, [fadd_acq_rel_float_global_sys_param_0];
-; SM90-NOFTZ-DISALLOW-NEXT: fence.release.sys;
-; SM90-NOFTZ-DISALLOW-NEXT: ld.relaxed.sys.global.b32 %r4, [%rd1];
-; SM90-NOFTZ-DISALLOW-NEXT: $L__BB106_1: // %atomicrmw.start
-; SM90-NOFTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NOFTZ-DISALLOW-NEXT: add.rn.f32 %r3, %r4, %r2;
-; SM90-NOFTZ-DISALLOW-NEXT: atom.relaxed.sys.global.cas.b32 %r1, [%rd1], %r4, %r3;
-; SM90-NOFTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
-; SM90-NOFTZ-DISALLOW-NEXT: mov.b32 %r4, %r1;
-; SM90-NOFTZ-DISALLOW-NEXT: @%p1 bra $L__BB106_1;
-; SM90-NOFTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NOFTZ-DISALLOW-NEXT: fence.acquire.sys;
-; SM90-NOFTZ-DISALLOW-NEXT: st.param::func.b32 [func_retval0], %r1;
+; SM90-NOFTZ-DISALLOW-NEXT: ld.param::func.b32 %r1, [fadd_acq_rel_float_global_sys_param_1];
+; SM90-NOFTZ-DISALLOW-NEXT: atom.acq_rel.sys.global.add.noftz.f32 %r2, [%rd1], %r1;
+; SM90-NOFTZ-DISALLOW-NEXT: st.param::func.b32 [func_retval0], %r2;
; SM90-NOFTZ-DISALLOW-NEXT: ret;
;
; SM90-NOFTZ-ALLOW-LABEL: fadd_acq_rel_float_global_sys(
@@ -4099,7 +4077,7 @@ define float @fadd_acq_rel_float_global_sys(ptr addrspace(1) %addr, float %val)
; SM90-NOFTZ-ALLOW-NEXT: // %bb.0:
; SM90-NOFTZ-ALLOW-NEXT: ld.param::func.b64 %rd1, [fadd_acq_rel_float_global_sys_param_0];
; SM90-NOFTZ-ALLOW-NEXT: ld.param::func.b32 %r1, [fadd_acq_rel_float_global_sys_param_1];
-; SM90-NOFTZ-ALLOW-NEXT: atom.acq_rel.sys.global.add.f32 %r2, [%rd1], %r1;
+; SM90-NOFTZ-ALLOW-NEXT: atom.acq_rel.sys.global.add.noftz.f32 %r2, [%rd1], %r1;
; SM90-NOFTZ-ALLOW-NEXT: st.param::func.b32 [func_retval0], %r2;
; SM90-NOFTZ-ALLOW-NEXT: ret;
;
@@ -4266,25 +4244,14 @@ define i64 @nand_acq_rel_i64_global_cluster(ptr addrspace(1) %addr, i64 %val) {
define float @fadd_acq_rel_float_global_cluster(ptr addrspace(1) %addr, float %val) {
; SM90-NOFTZ-DISALLOW-LABEL: fadd_acq_rel_float_global_cluster(
; SM90-NOFTZ-DISALLOW: {
-; SM90-NOFTZ-DISALLOW-NEXT: .reg .pred %p<2>;
-; SM90-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<5>;
+; SM90-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<3>;
; SM90-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
; SM90-NOFTZ-DISALLOW-EMPTY:
; SM90-NOFTZ-DISALLOW-NEXT: // %bb.0:
-; SM90-NOFTZ-DISALLOW-NEXT: ld.param::func.b32 %r2, [fadd_acq_rel_float_global_cluster_param_1];
; SM90-NOFTZ-DISALLOW-NEXT: ld.param::func.b64 %rd1, [fadd_acq_rel_float_global_cluster_param_0];
-; SM90-NOFTZ-DISALLOW-NEXT: fence.release.cluster;
-; SM90-NOFTZ-DISALLOW-NEXT: ld.relaxed.cluster.global.b32 %r4, [%rd1];
-; SM90-NOFTZ-DISALLOW-NEXT: $L__BB114_1: // %atomicrmw.start
-; SM90-NOFTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NOFTZ-DISALLOW-NEXT: add.rn.f32 %r3, %r4, %r2;
-; SM90-NOFTZ-DISALLOW-NEXT: atom.relaxed.cluster.global.cas.b32 %r1, [%rd1], %r4, %r3;
-; SM90-NOFTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
-; SM90-NOFTZ-DISALLOW-NEXT: mov.b32 %r4, %r1;
-; SM90-NOFTZ-DISALLOW-NEXT: @%p1 bra $L__BB114_1;
-; SM90-NOFTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NOFTZ-DISALLOW-NEXT: fence.acquire.cluster;
-; SM90-NOFTZ-DISALLOW-NEXT: st.param::func.b32 [func_retval0], %r1;
+; SM90-NOFTZ-DISALLOW-NEXT: ld.param::func.b32 %r1, [fadd_acq_rel_float_global_cluster_param_1];
+; SM90-NOFTZ-DISALLOW-NEXT: atom.acq_rel.cluster.global.add.noftz.f32 %r2, [%rd1], %r1;
+; SM90-NOFTZ-DISALLOW-NEXT: st.param::func.b32 [func_retval0], %r2;
; SM90-NOFTZ-DISALLOW-NEXT: ret;
;
; SM90-NOFTZ-ALLOW-LABEL: fadd_acq_rel_float_global_cluster(
@@ -4295,7 +4262,7 @@ define float @fadd_acq_rel_float_global_cluster(ptr addrspace(1) %addr, float %v
; SM90-NOFTZ-ALLOW-NEXT: // %bb.0:
; SM90-NOFTZ-ALLOW-NEXT: ld.param::func.b64 %rd1, [fadd_acq_rel_float_global_cluster_param_0];
; SM90-NOFTZ-ALLOW-NEXT: ld.param::func.b32 %r1, [fadd_acq_rel_float_global_cluster_param_1];
-; SM90-NOFTZ-ALLOW-NEXT: atom.acq_rel.cluster.global.add.f32 %r2, [%rd1], %r1;
+; SM90-NOFTZ-ALLOW-NEXT: atom.acq_rel.cluster.global.add.noftz.f32 %r2, [%rd1], %r1;
; SM90-NOFTZ-ALLOW-NEXT: st.param::func.b32 [func_retval0], %r2;
; SM90-NOFTZ-ALLOW-NEXT: ret;
;
@@ -4462,25 +4429,14 @@ define i64 @nand_acq_rel_i64_global_gpu(ptr addrspace(1) %addr, i64 %val) {
define float @fadd_acq_rel_float_global_gpu(ptr addrspace(1) %addr, float %val) {
; SM90-NOFTZ-DISALLOW-LABEL: fadd_acq_rel_float_global_gpu(
; SM90-NOFTZ-DISALLOW: {
-; SM90-NOFTZ-DISALLOW-NEXT: .reg .pred %p<2>;
-; SM90-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<5>;
+; SM90-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<3>;
; SM90-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
; SM90-NOFTZ-DISALLOW-EMPTY:
; SM90-NOFTZ-DISALLOW-NEXT: // %bb.0:
-; SM90-NOFTZ-DISALLOW-NEXT: ld.param::func.b32 %r2, [fadd_acq_rel_float_global_gpu_param_1];
; SM90-NOFTZ-DISALLOW-NEXT: ld.param::func.b64 %rd1, [fadd_acq_rel_float_global_gpu_param_0];
-; SM90-NOFTZ-DISALLOW-NEXT: fence.release.gpu;
-; SM90-NOFTZ-DISALLOW-NEXT: ld.relaxed.gpu.global.b32 %r4, [%rd1];
-; SM90-NOFTZ-DISALLOW-NEXT: $L__BB122_1: // %atomicrmw.start
-; SM90-NOFTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NOFTZ-DISALLOW-NEXT: add.rn.f32 %r3, %r4, %r2;
-; SM90-NOFTZ-DISALLOW-NEXT: atom.relaxed.gpu.global.cas.b32 %r1, [%rd1], %r4, %r3;
-; SM90-NOFTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
-; SM90-NOFTZ-DISALLOW-NEXT: mov.b32 %r4, %r1;
-; SM90-NOFTZ-DISALLOW-NEXT: @%p1 bra $L__BB122_1;
-; SM90-NOFTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NOFTZ-DISALLOW-NEXT: fence.acquire.gpu;
-; SM90-NOFTZ-DISALLOW-NEXT: st.param::func.b32 [func_retval0], %r1;
+; SM90-NOFTZ-DISALLOW-NEXT: ld.param::func.b32 %r1, [fadd_acq_rel_float_global_gpu_param_1];
+; SM90-NOFTZ-DISALLOW-NEXT: atom.acq_rel.gpu.global.add.noftz.f32 %r2, [%rd1], %r1;
+; SM90-NOFTZ-DISALLOW-NEXT: st.param::func.b32 [func_retval0], %r2;
; SM90-NOFTZ-DISALLOW-NEXT: ret;
;
; SM90-NOFTZ-ALLOW-LABEL: fadd_acq_rel_float_global_gpu(
@@ -4491,7 +4447,7 @@ define float @fadd_acq_rel_float_global_gpu(ptr addrspace(1) %addr, float %val)
; SM90-NOFTZ-ALLOW-NEXT: // %bb.0:
; SM90-NOFTZ-ALLOW-NEXT: ld.param::func.b64 %rd1, [fadd_acq_rel_float_global_gpu_param_0];
; SM90-NOFTZ-ALLOW-NEXT: ld.param::func.b32 %r1, [fadd_acq_rel_float_global_gpu_param_1];
-; SM90-NOFTZ-ALLOW-NEXT: atom.acq_rel.gpu.global.add.f32 %r2, [%rd1], %r1;
+; SM90-NOFTZ-ALLOW-NEXT: atom.acq_rel.gpu.global.add.noftz.f32 %r2, [%rd1], %r1;
; SM90-NOFTZ-ALLOW-NEXT: st.param::func.b32 [func_retval0], %r2;
; SM90-NOFTZ-ALLOW-NEXT: ret;
;
@@ -4658,25 +4614,14 @@ define i64 @nand_acq_rel_i64_generic_cta(ptr %addr, i64 %val) {
define float @fadd_acq_rel_float_generic_cta(ptr %addr, float %val) {
; SM90-NOFTZ-DISALLOW-LABEL: fadd_acq_rel_float_generic_cta(
; SM90-NOFTZ-DISALLOW: {
-; SM90-NOFTZ-DISALLOW-NEXT: .reg .pred %p<2>;
-; SM90-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<5>;
+; SM90-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<3>;
; SM90-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
; SM90-NOFTZ-DISALLOW-EMPTY:
; SM90-NOFTZ-DISALLOW-NEXT: // %bb.0:
-; SM90-NOFTZ-DISALLOW-NEXT: ld.param::func.b32 %r2, [fadd_acq_rel_float_generic_cta_param_1];
; SM90-NOFTZ-DISALLOW-NEXT: ld.param::func.b64 %rd1, [fadd_acq_rel_float_generic_cta_param_0];
-; SM90-NOFTZ-DISALLOW-NEXT: fence.release.cta;
-; SM90-NOFTZ-DISALLOW-NEXT: ld.relaxed.cta.b32 %r4, [%rd1];
-; SM90-NOFTZ-DISALLOW-NEXT: $L__BB130_1: // %atomicrmw.start
-; SM90-NOFTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NOFTZ-DISALLOW-NEXT: add.rn.f32 %r3, %r4, %r2;
-; SM90-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.cas.b32 %r1, [%rd1], %r4, %r3;
-; SM90-NOFTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
-; SM90-NOFTZ-DISALLOW-NEXT: mov.b32 %r4, %r1;
-; SM90-NOFTZ-DISALLOW-NEXT: @%p1 bra $L__BB130_1;
-; SM90-NOFTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NOFTZ-DISALLOW-NEXT: fence.acquire.cta;
-; SM90-NOFTZ-DISALLOW-NEXT: st.param::func.b32 [func_retval0], %r1;
+; SM90-NOFTZ-DISALLOW-NEXT: ld.param::func.b32 %r1, [fadd_acq_rel_float_generic_cta_param_1];
+; SM90-NOFTZ-DISALLOW-NEXT: atom.acq_rel.cta.add.noftz.f32 %r2, [%rd1], %r1;
+; SM90-NOFTZ-DISALLOW-NEXT: st.param::func.b32 [func_retval0], %r2;
; SM90-NOFTZ-DISALLOW-NEXT: ret;
;
; SM90-NOFTZ-ALLOW-LABEL: fadd_acq_rel_float_generic_cta(
@@ -4687,7 +4632,7 @@ define float @fadd_acq_rel_float_generic_cta(ptr %addr, float %val) {
; SM90-NOFTZ-ALLOW-NEXT: // %bb.0:
; SM90-NOFTZ-ALLOW-NEXT: ld.param::func.b64 %rd1, [fadd_acq_rel_float_generic_cta_param_0];
; SM90-NOFTZ-ALLOW-NEXT: ld.param::func.b32 %r1, [fadd_acq_rel_float_generic_cta_param_1];
-; SM90-NOFTZ-ALLOW-NEXT: atom.acq_rel.cta.add.f32 %r2, [%rd1], %r1;
+; SM90-NOFTZ-ALLOW-NEXT: atom.acq_rel.cta.add.noftz.f32 %r2, [%rd1], %r1;
; SM90-NOFTZ-ALLOW-NEXT: st.param::func.b32 [func_retval0], %r2;
; SM90-NOFTZ-ALLOW-NEXT: ret;
;
@@ -4871,7 +4816,7 @@ define float @fadd_acq_rel_float_shared_cta(ptr addrspace(3) %addr, float %val)
; SM90-NOFTZ-DISALLOW-NEXT: // %bb.0:
; SM90-NOFTZ-DISALLOW-NEXT: ld.param::func.b64 %rd1, [fadd_acq_rel_float_shared_cta_param_0];
; SM90-NOFTZ-DISALLOW-NEXT: ld.param::func.b32 %r1, [fadd_acq_rel_float_shared_cta_param_1];
-; SM90-NOFTZ-DISALLOW-NEXT: atom.acq_rel.cta.shared.add.f32 %r2, [%rd1], %r1;
+; SM90-NOFTZ-DISALLOW-NEXT: atom.acq_rel.cta.shared.add.noftz.f32 %r2, [%rd1], %r1;
; SM90-NOFTZ-DISALLOW-NEXT: st.param::func.b32 [func_retval0], %r2;
; SM90-NOFTZ-DISALLOW-NEXT: ret;
;
@@ -4883,7 +4828,7 @@ define float @fadd_acq_rel_float_shared_cta(ptr addrspace(3) %addr, float %val)
; SM90-NOFTZ-ALLOW-NEXT: // %bb.0:
; SM90-NOFTZ-ALLOW-NEXT: ld.param::func.b64 %rd1, [fadd_acq_rel_float_shared_cta_param_0];
; SM90-NOFTZ-ALLOW-NEXT: ld.param::func.b32 %r1, [fadd_acq_rel_float_shared_cta_param_1];
-; SM90-NOFTZ-ALLOW-NEXT: atom.acq_rel.cta.shared.add.f32 %r2, [%rd1], %r1;
+; SM90-NOFTZ-ALLOW-NEXT: atom.acq_rel.cta.shared.add.noftz.f32 %r2, [%rd1], %r1;
; SM90-NOFTZ-ALLOW-NEXT: st.param::func.b32 [func_retval0], %r2;
; SM90-NOFTZ-ALLOW-NEXT: ret;
;
diff --git a/llvm/test/CodeGen/NVPTX/atomicrmw.py b/llvm/test/CodeGen/NVPTX/atomicrmw.py
index cad1573550a6c..af3a3a49d245b 100644
--- a/llvm/test/CodeGen/NVPTX/atomicrmw.py
+++ b/llvm/test/CodeGen/NVPTX/atomicrmw.py
@@ -3,7 +3,7 @@
from string import Template
from itertools import product
-TEST_SM_ARCH_PAIRS = [(60, 50), (70, 63), (90, 87)]
+TEST_SM_ARCH_PAIRS = [(60, 50), (70, 63), (90, 94)]
SCOPE_LLVM_TO_PTX = {"": "sys", "block": "cta", "cluster": "cluster", "device": "gpu"}
@@ -57,20 +57,20 @@
# atomicrmw fadd's lowering depends on the function's FTZ (denormal) mode, so we
# check codegen both with and without it. Lines common to both runs collapse to
# the SM${sm} prefix; only the FTZ-sensitive ops diverge into SM${sm}-NOFTZ /
-# SM${sm}-FTZ. (-nvptx-allow-ftz-atomics is covered separately in
-# atomicrmw-allow-ftz-atomics.ll.)
+# SM${sm}-FTZ. (-nvptx-ftz-atomics is covered separately in
+# atomicrmw-ftz.ll.)
run_statement = Template(
- """; RUN: llc < %s -march=nvptx64 -mcpu=sm_${sm} -mattr=+ptx${ptx} -nvptx-allow-ftz-atomics=false | FileCheck %s --check-prefixes=SM${sm},SM${sm}-NOFTZ-DISALLOW
-; RUN: llc < %s -march=nvptx64 -mcpu=sm_${sm} -mattr=+ptx${ptx} -nvptx-allow-ftz-atomics=true | FileCheck %s --check-prefixes=SM${sm},SM${sm}-NOFTZ-ALLOW
+ """; RUN: llc < %s -march=nvptx64 -mcpu=sm_${sm} -mattr=+ptx${ptx} -nvptx-ftz-atomics=strict | FileCheck %s --check-prefixes=SM${sm},SM${sm}-NOFTZ-DISALLOW
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_${sm} -mattr=+ptx${ptx} -nvptx-ftz-atomics=fast | FileCheck %s --check-prefixes=SM${sm},SM${sm}-NOFTZ-ALLOW
; RUN: llc < %s -march=nvptx64 -mcpu=sm_${sm} -mattr=+ptx${ptx} | FileCheck %s --check-prefixes=SM${sm},SM${sm}-NOFTZ-ALLOW
-; RUN: llc < %s -march=nvptx64 -mcpu=sm_${sm} -mattr=+ptx${ptx} -denormal-fp-math-f32=preserve-sign -nvptx-allow-ftz-atomics=false | FileCheck %s --check-prefixes=SM${sm},SM${sm}-FTZ-DISALLOW
-; RUN: llc < %s -march=nvptx64 -mcpu=sm_${sm} -mattr=+ptx${ptx} -denormal-fp-math-f32=preserve-sign -nvptx-allow-ftz-atomics=true | FileCheck %s --check-prefixes=SM${sm},SM${sm}-FTZ-ALLOW
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_${sm} -mattr=+ptx${ptx} -denormal-fp-math-f32=preserve-sign -nvptx-ftz-atomics=strict | FileCheck %s --check-prefixes=SM${sm},SM${sm}-FTZ-DISALLOW
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_${sm} -mattr=+ptx${ptx} -denormal-fp-math-f32=preserve-sign -nvptx-ftz-atomics=fast | FileCheck %s --check-prefixes=SM${sm},SM${sm}-FTZ-ALLOW
; RUN: llc < %s -march=nvptx64 -mcpu=sm_${sm} -mattr=+ptx${ptx} -denormal-fp-math-f32=preserve-sign | FileCheck %s --check-prefixes=SM${sm},SM${sm}-FTZ-ALLOW
-; RUN: %if ptxas-sm_${sm} && ptxas-isa-${ptxfp} %{ llc < %s -march=nvptx64 -mcpu=sm_${sm} -mattr=+ptx${ptx} -nvptx-allow-ftz-atomics=false | %ptxas-verify -arch=sm_${sm} %}
-; RUN: %if ptxas-sm_${sm} && ptxas-isa-${ptxfp} %{ llc < %s -march=nvptx64 -mcpu=sm_${sm} -mattr=+ptx${ptx} -nvptx-allow-ftz-atomics=true | %ptxas-verify -arch=sm_${sm} %}
+; RUN: %if ptxas-sm_${sm} && ptxas-isa-${ptxfp} %{ llc < %s -march=nvptx64 -mcpu=sm_${sm} -mattr=+ptx${ptx} -nvptx-ftz-atomics=strict | %ptxas-verify -arch=sm_${sm} %}
+; RUN: %if ptxas-sm_${sm} && ptxas-isa-${ptxfp} %{ llc < %s -march=nvptx64 -mcpu=sm_${sm} -mattr=+ptx${ptx} -nvptx-ftz-atomics=fast | %ptxas-verify -arch=sm_${sm} %}
; RUN: %if ptxas-sm_${sm} && ptxas-isa-${ptxfp} %{ llc < %s -march=nvptx64 -mcpu=sm_${sm} -mattr=+ptx${ptx} | %ptxas-verify -arch=sm_${sm} %}
-; RUN: %if ptxas-sm_${sm} && ptxas-isa-${ptxfp} %{ llc < %s -march=nvptx64 -mcpu=sm_${sm} -mattr=+ptx${ptx} -denormal-fp-math-f32=preserve-sign -nvptx-allow-ftz-atomics=false | %ptxas-verify -arch=sm_${sm} %}
-; RUN: %if ptxas-sm_${sm} && ptxas-isa-${ptxfp} %{ llc < %s -march=nvptx64 -mcpu=sm_${sm} -mattr=+ptx${ptx} -denormal-fp-math-f32=preserve-sign -nvptx-allow-ftz-atomics=true | %ptxas-verify -arch=sm_${sm} %}
+; RUN: %if ptxas-sm_${sm} && ptxas-isa-${ptxfp} %{ llc < %s -march=nvptx64 -mcpu=sm_${sm} -mattr=+ptx${ptx} -denormal-fp-math-f32=preserve-sign -nvptx-ftz-atomics=strict | %ptxas-verify -arch=sm_${sm} %}
+; RUN: %if ptxas-sm_${sm} && ptxas-isa-${ptxfp} %{ llc < %s -march=nvptx64 -mcpu=sm_${sm} -mattr=+ptx${ptx} -denormal-fp-math-f32=preserve-sign -nvptx-ftz-atomics=fast | %ptxas-verify -arch=sm_${sm} %}
; RUN: %if ptxas-sm_${sm} && ptxas-isa-${ptxfp} %{ llc < %s -march=nvptx64 -mcpu=sm_${sm} -mattr=+ptx${ptx} -denormal-fp-math-f32=preserve-sign | %ptxas-verify -arch=sm_${sm} %}
"""
)
diff --git a/llvm/test/CodeGen/NVPTX/atomics.ll b/llvm/test/CodeGen/NVPTX/atomics.ll
index e1853f1b00cf3..14e202b981f26 100644
--- a/llvm/test/CodeGen/NVPTX/atomics.ll
+++ b/llvm/test/CodeGen/NVPTX/atomics.ll
@@ -1,9 +1,9 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
-; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_32 -nvptx-allow-ftz-atomics=false | FileCheck %s --check-prefixes=CHECK,DISALLOW
-; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_32 -nvptx-allow-ftz-atomics=true | FileCheck %s --check-prefixes=CHECK,ALLOW
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_32 -nvptx-ftz-atomics=strict | FileCheck %s --check-prefixes=CHECK,DISALLOW
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_32 -nvptx-ftz-atomics=fast | FileCheck %s --check-prefixes=CHECK,ALLOW
; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_32 | FileCheck %s --check-prefixes=CHECK,ALLOW
-; RUN: %if ptxas %{ llc < %s -mtriple=nvptx64 -mcpu=sm_32 -nvptx-allow-ftz-atomics=false | %ptxas-verify %}
-; RUN: %if ptxas %{ llc < %s -mtriple=nvptx64 -mcpu=sm_32 -nvptx-allow-ftz-atomics=true | %ptxas-verify %}
+; RUN: %if ptxas %{ llc < %s -mtriple=nvptx64 -mcpu=sm_32 -nvptx-ftz-atomics=strict | %ptxas-verify %}
+; RUN: %if ptxas %{ llc < %s -mtriple=nvptx64 -mcpu=sm_32 -nvptx-ftz-atomics=fast | %ptxas-verify %}
; RUN: %if ptxas %{ llc < %s -mtriple=nvptx64 -mcpu=sm_32 | %ptxas-verify %}
declare float @llvm.nvvm.atomic.load.add.f32.p0(ptr %addr, float %val)
>From d40efc4475bc5a9688b34d0f4c1a87d9694c157d Mon Sep 17 00:00:00 2001
From: Akshay Deodhar <adeodhar at nvidia.com>
Date: Tue, 15 Sep 2026 21:10:28 +0000
Subject: [PATCH 2/3] clang-format
---
llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
index 64b040f84e5f9..7c5dc04cc4655 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
@@ -7732,8 +7732,10 @@ NVPTXTargetLowering::shouldExpandAtomicRMWInIR(const AtomicRMWInst *AI) const {
// - atom.add.f32 on shared memory does not flush denormals
// - atom.add.f32.noftz is supported for SM90+, PTX 9.4+
// - atom.add.f16 and atomic.add.bf16 never flush denormals
- // - atom.add.f64 never flushes denormals, and add.f64 never flushes denormals
- // We lower to atom.add only if the function's FTZ behavior is natively supported. otherwise, we lower to a CAS loop. But we always allow
+ // - atom.add.f64 never flushes denormals, and add.f64 never flushes
+ // denormals
+ // We lower to atom.add only if the function's FTZ behavior is natively
+ // supported. otherwise, we lower to a CAS loop. But we always allow
// atomic.add.bf16; even though it never flushes denormals, we never flush
// bf16 denormals when doing regular arithmetic, even when FTZ is enabled.
if (AI->isFloatingPointOperation() &&
>From 827a51602e6870cec11ac2db85f71cb1287ed7cf Mon Sep 17 00:00:00 2001
From: Akshay Deodhar <adeodhar at nvidia.com>
Date: Mon, 28 Sep 2026 21:46:19 +0000
Subject: [PATCH 3/3] address review comments
---
llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp | 40 ++++++++++++---------
llvm/lib/Target/NVPTX/NVPTXInstrInfo.td | 2 +-
llvm/lib/Target/NVPTX/NVPTXIntrinsics.td | 6 ++--
llvm/lib/Target/NVPTX/NVPTXSubtarget.h | 2 +-
4 files changed, 30 insertions(+), 20 deletions(-)
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
index 7c5dc04cc4655..10576f5c9342b 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
@@ -7734,10 +7734,13 @@ NVPTXTargetLowering::shouldExpandAtomicRMWInIR(const AtomicRMWInst *AI) const {
// - atom.add.f16 and atomic.add.bf16 never flush denormals
// - atom.add.f64 never flushes denormals, and add.f64 never flushes
// denormals
- // We lower to atom.add only if the function's FTZ behavior is natively
- // supported. otherwise, we lower to a CAS loop. But we always allow
- // atomic.add.bf16; even though it never flushes denormals, we never flush
- // bf16 denormals when doing regular arithmetic, even when FTZ is enabled.
+ // When the specified atomic add behavior is "fast" (the default), we always
+ // lower to an atom.add instruction, and try to match the function's FTZ
+ // behavior. When the mode is "strict", we lower to a native atom.add only if
+ // the function's FTZ behavior is supported. otherwise, we lower to a CAS
+ // loop. But we always allow atomic.add.bf16; even though it never flushes
+ // denormals, we never flush bf16 denormals when doing regular arithmetic,
+ // even when FTZ is enabled.
if (AI->isFloatingPointOperation() &&
AI->getOperation() == AtomicRMWInst::BinOp::FAdd) {
const Function *F = AI->getFunction();
@@ -7745,18 +7748,17 @@ NVPTXTargetLowering::shouldExpandAtomicRMWInIR(const AtomicRMWInst *AI) const {
// AllowFTZAtomics forces atom.add regardless of a denormal mode mismatch
if (Ty->isFloatTy()) {
+ if (AllowFTZAtomics)
+ return AtomicExpansionKind::None;
const auto Mode = F->getDenormalMode(APFloat::IEEEsingle());
const bool IEEE =
Mode.Input == DenormalMode::IEEE ||
Mode.Output == DenormalMode::IEEE; // must not flush denormals
- const bool FTZ =
- Mode.inputsAreZero(); // must have input denormals flushed
- const bool NativeNoFTZ = STI.hasAtomAddNoFTZ();
- bool UseNative = AllowFTZAtomics;
- // Generic has no case below, so it keeps UseNative = AllowFTZAtomics:
- // when AtomicAddBehavior is strict we always generate a CmpXchg loop,
- // whose non-atomic fadd is correctly lowered to ftz/noftz based on the
- // function's DenormalMode.
+ // TODO: Fixup useF32FTZ to resolve to Mode.inputsAreZero(). Until then
+ // mimic useF32FTZ behavior here.
+ const bool FTZ = Mode.Output == DenormalMode::PreserveSign;
+ const bool NativeNoFTZ = STI.hasAtomAddF32NoFTZ();
+ bool UseNative = false;
// TODO: Handle cases where input mode and output mode are mismatched,
// once the bug is fixed in nonatomic add.
switch (AI->getPointerAddressSpace()) {
@@ -7768,6 +7770,11 @@ NVPTXTargetLowering::shouldExpandAtomicRMWInIR(const AtomicRMWInst *AI) const {
UseNative |= !FTZ;
break;
case llvm::ADDRESS_SPACE_GENERIC:
+ // We can only guarantee conformant behavior when the DenormalMode
+ // is IEEE, and the atom.add.noftz instruction is supported. In all
+ // other cases, since the address space is unknown at compile time,
+ // we generate a CAS loop with a nonatomic add instruction, which
+ // gets lowered with the right DenormalMode.
UseNative |= (NativeNoFTZ && IEEE);
break;
}
@@ -7779,18 +7786,19 @@ NVPTXTargetLowering::shouldExpandAtomicRMWInIR(const AtomicRMWInst *AI) const {
// atom.add.f16 never flushes denormals, so it only agrees with a
// function that is not in FTZ mode for f16. The f32 denormal mode is
// irrelevant here.
- const bool FTZ = F->getDenormalMode(APFloat::IEEEhalf()).inputsAreZero();
+ const bool FTZ = F->getDenormalMode(APFloat::IEEEhalf()).Output ==
+ DenormalMode::PreserveSign;
if ((!FTZ || AllowFTZAtomics) && STI.hasFeature(NVPTX::SM70) &&
STI.hasFeature(NVPTX::PTX63))
return AtomicExpansionKind::None;
}
- if (Ty->isBFloatTy() && STI.hasFeature(NVPTX::SM90) &&
- STI.hasFeature(NVPTX::PTX78))
+ if (Ty->isBFloatTy() && STI.hasFeature(NVPTX::SM90))
return AtomicExpansionKind::None;
// We cannot support FTZ mode for fp64, as neither atom.add.f64 nor add.f64
- // has an ftz mode that flushes denormals
+ // has an ftz mode that flushes denormals. This matches nonatomic floating
+ // point add behavior. where we ignore FTZ mode for fp64.
if (Ty->isDoubleTy() && STI.hasAtomAddF64())
return AtomicExpansionKind::None;
}
diff --git a/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td b/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td
index 0511d392bd4fb..c4d5adc79f5e6 100644
--- a/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td
+++ b/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td
@@ -148,7 +148,7 @@ class PredNot<Predicate p> : Predicate<"!(" # p.CondString # ")">;
class SubtargetPredicate : Predicate<"Subtarget->" # NAME # "()">;
-def hasAtomAddNoFTZ : SubtargetPredicate;
+def hasAtomAddF32NoFTZ : SubtargetPredicate;
def hasAtomAddF64 : SubtargetPredicate;
def hasAtomScope : SubtargetPredicate;
def hasAtomBitwise64 : SubtargetPredicate;
diff --git a/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td b/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
index f8db000933920..a764658fcd040 100644
--- a/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
+++ b/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
@@ -3541,10 +3541,12 @@ defm INT_PTX_ATOM_ADD_32 : F_ATOMIC_2<I32RT, atomic_load_add_i32, "add", "u32",
defm INT_PTX_ATOM_ADD_64 : F_ATOMIC_2<I64RT, atomic_load_add_i64, "add", "u64", atomic_load_add>;
defm INT_PTX_ATOM_ADD_F16 : F_ATOMIC_2<F16RT, atomic_load_fadd, "add.noftz", "f16", atomic_load_fadd, [SM70, PTX63]>;
-defm INT_PTX_ATOM_ADD_BF16 : F_ATOMIC_2<BF16RT, atomic_load_fadd, "add.noftz", "bf16", atomic_load_fadd, [SM90, PTX78]>;
+defm INT_PTX_ATOM_ADD_BF16 : F_ATOMIC_2<BF16RT, atomic_load_fadd, "add.noftz", "bf16", atomic_load_fadd, [SM90]>;
// If the function denormal mode is IEEE, and noftz is supported, lower to NoFTZ
-defm INT_PTX_ATOM_ADD_NOFTZ_F32 : F_ATOMIC_2<F32RT, atomic_load_fadd, "add.noftz", "f32", atomic_load_fadd, [hasAtomAddNoFTZ, doNoF32FTZ, noLegacyF32AtomAdd]>;
+let AddedComplexity = 1 in { // prefer this pattern
+ defm INT_PTX_ATOM_ADD_NOFTZ_F32 : F_ATOMIC_2<F32RT, atomic_load_fadd, "add.noftz", "f32", atomic_load_fadd, [hasAtomAddF32NoFTZ, doNoF32FTZ, noLegacyF32AtomAdd]>;
+}
// Otherwise, lower to the default native instruction. Passes prior to ISel
// must handle the case where there is a mismatch between default semantics and
// the denormal mode
diff --git a/llvm/lib/Target/NVPTX/NVPTXSubtarget.h b/llvm/lib/Target/NVPTX/NVPTXSubtarget.h
index 4f56a8e0df7db..82501a3f3e447 100644
--- a/llvm/lib/Target/NVPTX/NVPTXSubtarget.h
+++ b/llvm/lib/Target/NVPTX/NVPTXSubtarget.h
@@ -85,7 +85,7 @@ class NVPTXSubtarget : public NVPTXGenSubtargetInfo {
bool hasUsedBytesMaskPragma() const {
return hasFeature(NVPTX::SM50) && hasFeature(NVPTX::PTX83);
}
- bool hasAtomAddNoFTZ() const {
+ bool hasAtomAddF32NoFTZ() const {
return hasFeature(NVPTX::SM90) && hasFeature(NVPTX::PTX94);
}
bool hasAtomAddF64() const { return hasFeature(NVPTX::SM60); }
More information about the llvm-commits
mailing list