[llvm-branch-commits] [llvm] release/23.x: [NVPTX] Set default value of nvptx-allow-ftz-atomics to true (#206154) (PR #229003)
via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Mon Oct 5 01:18:56 PDT 2026
https://github.com/llvmbot created https://github.com/llvm/llvm-project/pull/229003
Backport 2b6e9d261433f5ec8c49502f91d63dd7d07346c3
Requested by: @ilya-biryukov
>From 8fecbd580ec55790f583ca62d7dc6b6501ccf39e Mon Sep 17 00:00:00 2001
From: Artem Belevich <tra at google.com>
Date: Fri, 17 Jul 2026 11:12:07 -0700
Subject: [PATCH] [NVPTX] Set default value of nvptx-allow-ftz-atomics to true
(#206154)
This is a follow-up to https://github.com/llvm/llvm-project/pull/200732.
Setting the default value of nvptx-allow-ftz-atomics to true preserves
the status quo. It allows the NVPTX backend to emit native
floating-point atomic add instructions by default, rather than falling
back to the CAS loop required for strict FTZ compliance.
Correct FTZ handling forces a lot of existing code to emit CAS loops.
This introduces thread divergence, breaking code that relies on atomic
operations being non-divergent (for example, warp-wide atomic additions
followed by a load of the reduced value without explicit
synchronization).
While FTZ-correct atomic behavior is desirable as a long-term default,
introducing thread divergence into a common operation that previously
"happened to work" without synchronization is highly disruptive.
Defaulting to true makes the FTZ-correct behavior opt-in for now,
allowing users time to audit and adapt their code.
Test cases are updated to verify both enabled and disabled states of the
option, alongside the new default behavior.
(cherry picked from commit 2b6e9d261433f5ec8c49502f91d63dd7d07346c3)
---
llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp | 9 +-
.../NVPTX/atomicrmw-allow-ftz-atomics.ll | 102 +-
llvm/test/CodeGen/NVPTX/atomicrmw-sm60.ll | 4436 +++++++++++------
llvm/test/CodeGen/NVPTX/atomicrmw-sm70.ll | 4332 ++++++++++------
llvm/test/CodeGen/NVPTX/atomicrmw-sm90.ll | 2484 +++++----
llvm/test/CodeGen/NVPTX/atomicrmw.py | 12 +-
llvm/test/CodeGen/NVPTX/atomics.ll | 112 +-
7 files changed, 7594 insertions(+), 3893 deletions(-)
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
index 59d04dde1eeb0..6cca7f4acd858 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
@@ -110,14 +110,15 @@ static cl::opt<bool> UsePrecSqrtF32(
cl::init(true));
// PTX atom.add.f32 has fixed FTZ behavior that may not match the function's
-// (see shouldExpandAtomicRMWInIR), so by default we fall back to a CAS loop
-// when they disagree. This flag is an escape hatch to use atom.add anyway,
-// trading correct denormal handling for the speed of the native instruction.
+// (see shouldExpandAtomicRMWInIR), so we'd normally fall back to a CAS loop
+// when they disagree. This option (enabled by default) allows using atom.add
+// anyway, trading correct denormal handling for the speed of the native
+// instruction.
static cl::opt<bool> AllowFTZAtomics(
"nvptx-allow-ftz-atomics", cl::Hidden,
cl::desc("NVPTX Specific: Lower atomicrmw fadd to atom.add even when its "
"FTZ behavior does not match the function's denormal mode."),
- cl::init(false));
+ cl::init(true));
/// Whereas CUDA's implementation (see libdevice) uses ex2.approx for exp2(), it
/// does NOT use lg2.approx for log2, so this is disabled by default.
diff --git a/llvm/test/CodeGen/NVPTX/atomicrmw-allow-ftz-atomics.ll b/llvm/test/CodeGen/NVPTX/atomicrmw-allow-ftz-atomics.ll
index 727c13bb4e47e..c1d223082eb30 100644
--- a/llvm/test/CodeGen/NVPTX/atomicrmw-allow-ftz-atomics.ll
+++ b/llvm/test/CodeGen/NVPTX/atomicrmw-allow-ftz-atomics.ll
@@ -1,36 +1,39 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; -nvptx-allow-ftz-atomics forces atomicrmw fadd to lower to atom.add even when
; the hardware's FTZ behavior doesn't match the function's denormal mode, where
-; we'd otherwise emit a CAS loop. Compare the default lowering (DEFAULT) against
-; the flag (ALLOW).
-; RUN: llc < %s -march=nvptx64 -mcpu=sm_90 -mattr=+ptx87 | FileCheck %s --check-prefixes=CHECK,DEFAULT
-; RUN: llc < %s -march=nvptx64 -mcpu=sm_90 -mattr=+ptx87 -nvptx-allow-ftz-atomics | FileCheck %s --check-prefixes=CHECK,ALLOW
+; we'd otherwise emit a CAS loop. Compare the lowering with the option disabled
+; (DISALLOW) against the option enabled or default (ALLOW).
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_90 -mattr=+ptx87 -nvptx-allow-ftz-atomics=false | FileCheck %s --check-prefixes=CHECK,DISALLOW
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_90 -mattr=+ptx87 -nvptx-allow-ftz-atomics=true | FileCheck %s --check-prefixes=CHECK,ALLOW
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_90 -mattr=+ptx87 | FileCheck %s --check-prefixes=CHECK,ALLOW
+; RUN: %if ptxas-sm_90 && ptxas-isa-8.7 %{ llc < %s -march=nvptx64 -mcpu=sm_90 -mattr=+ptx87 -nvptx-allow-ftz-atomics=false | %ptxas-verify -arch=sm_90 %}
+; RUN: %if ptxas-sm_90 && ptxas-isa-8.7 %{ llc < %s -march=nvptx64 -mcpu=sm_90 -mattr=+ptx87 -nvptx-allow-ftz-atomics=true | %ptxas-verify -arch=sm_90 %}
; RUN: %if ptxas-sm_90 && ptxas-isa-8.7 %{ llc < %s -march=nvptx64 -mcpu=sm_90 -mattr=+ptx87 | %ptxas-verify -arch=sm_90 %}
-; RUN: %if ptxas-sm_90 && ptxas-isa-8.7 %{ llc < %s -march=nvptx64 -mcpu=sm_90 -mattr=+ptx87 -nvptx-allow-ftz-atomics | %ptxas-verify -arch=sm_90 %}
-; global f32: atom.add flushes, so default uses a CAS loop (function is not FTZ);
-; the flag forces atom.add.
+; global f32: atom.add flushes, so we'd normally use a CAS loop (function is not FTZ)
+; when the option is disabled; when enabled (default), it lowers to atom.add.
define float @fadd_f32_global(ptr addrspace(1) %addr, float %val) {
-; DEFAULT-LABEL: fadd_f32_global(
-; DEFAULT: {
-; DEFAULT-NEXT: .reg .pred %p<2>;
-; DEFAULT-NEXT: .reg .b32 %r<5>;
-; DEFAULT-NEXT: .reg .b64 %rd<2>;
-; DEFAULT-EMPTY:
-; DEFAULT-NEXT: // %bb.0:
-; DEFAULT-NEXT: ld.param.b32 %r2, [fadd_f32_global_param_1];
-; DEFAULT-NEXT: ld.param.b64 %rd1, [fadd_f32_global_param_0];
-; DEFAULT-NEXT: ld.relaxed.sys.global.b32 %r4, [%rd1];
-; DEFAULT-NEXT: $L__BB0_1: // %atomicrmw.start
-; DEFAULT-NEXT: // =>This Inner Loop Header: Depth=1
-; DEFAULT-NEXT: add.rn.f32 %r3, %r4, %r2;
-; DEFAULT-NEXT: atom.relaxed.sys.global.cas.b32 %r1, [%rd1], %r4, %r3;
-; DEFAULT-NEXT: setp.ne.b32 %p1, %r1, %r4;
-; DEFAULT-NEXT: mov.b32 %r4, %r1;
-; DEFAULT-NEXT: @%p1 bra $L__BB0_1;
-; DEFAULT-NEXT: // %bb.2: // %atomicrmw.end
-; DEFAULT-NEXT: st.param.b32 [func_retval0], %r1;
-; DEFAULT-NEXT: ret;
+;
+; DISALLOW-LABEL: fadd_f32_global(
+; DISALLOW: {
+; DISALLOW-NEXT: .reg .pred %p<2>;
+; DISALLOW-NEXT: .reg .b32 %r<5>;
+; DISALLOW-NEXT: .reg .b64 %rd<2>;
+; DISALLOW-EMPTY:
+; DISALLOW-NEXT: // %bb.0:
+; DISALLOW-NEXT: ld.param.b32 %r2, [fadd_f32_global_param_1];
+; DISALLOW-NEXT: ld.param.b64 %rd1, [fadd_f32_global_param_0];
+; DISALLOW-NEXT: ld.relaxed.sys.global.b32 %r4, [%rd1];
+; DISALLOW-NEXT: $L__BB0_1: // %atomicrmw.start
+; DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; DISALLOW-NEXT: add.rn.f32 %r3, %r4, %r2;
+; DISALLOW-NEXT: atom.relaxed.sys.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; DISALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; DISALLOW-NEXT: mov.b32 %r4, %r1;
+; DISALLOW-NEXT: @%p1 bra $L__BB0_1;
+; DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; DISALLOW-NEXT: st.param.b32 [func_retval0], %r1;
+; DISALLOW-NEXT: ret;
;
; ALLOW-LABEL: fadd_f32_global(
; ALLOW: {
@@ -47,29 +50,30 @@ define float @fadd_f32_global(ptr addrspace(1) %addr, float %val) {
ret float %r
}
-; generic f32: we can't know the memory space, so default always uses a CAS loop;
-; the flag forces atom.add.
+; generic f32: we can't know the memory space, so we'd normally use a CAS loop
+; when the option is disabled; when enabled (default), it lowers to atom.add.
define float @fadd_f32_generic(ptr %addr, float %val) {
-; DEFAULT-LABEL: fadd_f32_generic(
-; DEFAULT: {
-; DEFAULT-NEXT: .reg .pred %p<2>;
-; DEFAULT-NEXT: .reg .b32 %r<5>;
-; DEFAULT-NEXT: .reg .b64 %rd<2>;
-; DEFAULT-EMPTY:
-; DEFAULT-NEXT: // %bb.0:
-; DEFAULT-NEXT: ld.param.b32 %r2, [fadd_f32_generic_param_1];
-; DEFAULT-NEXT: ld.param.b64 %rd1, [fadd_f32_generic_param_0];
-; DEFAULT-NEXT: ld.relaxed.sys.b32 %r4, [%rd1];
-; DEFAULT-NEXT: $L__BB1_1: // %atomicrmw.start
-; DEFAULT-NEXT: // =>This Inner Loop Header: Depth=1
-; DEFAULT-NEXT: add.rn.f32 %r3, %r4, %r2;
-; DEFAULT-NEXT: atom.relaxed.sys.cas.b32 %r1, [%rd1], %r4, %r3;
-; DEFAULT-NEXT: setp.ne.b32 %p1, %r1, %r4;
-; DEFAULT-NEXT: mov.b32 %r4, %r1;
-; DEFAULT-NEXT: @%p1 bra $L__BB1_1;
-; DEFAULT-NEXT: // %bb.2: // %atomicrmw.end
-; DEFAULT-NEXT: st.param.b32 [func_retval0], %r1;
-; DEFAULT-NEXT: ret;
+;
+; DISALLOW-LABEL: fadd_f32_generic(
+; DISALLOW: {
+; DISALLOW-NEXT: .reg .pred %p<2>;
+; DISALLOW-NEXT: .reg .b32 %r<5>;
+; DISALLOW-NEXT: .reg .b64 %rd<2>;
+; DISALLOW-EMPTY:
+; DISALLOW-NEXT: // %bb.0:
+; DISALLOW-NEXT: ld.param.b32 %r2, [fadd_f32_generic_param_1];
+; DISALLOW-NEXT: ld.param.b64 %rd1, [fadd_f32_generic_param_0];
+; DISALLOW-NEXT: ld.relaxed.sys.b32 %r4, [%rd1];
+; DISALLOW-NEXT: $L__BB1_1: // %atomicrmw.start
+; DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; DISALLOW-NEXT: add.rn.f32 %r3, %r4, %r2;
+; DISALLOW-NEXT: atom.relaxed.sys.cas.b32 %r1, [%rd1], %r4, %r3;
+; DISALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; DISALLOW-NEXT: mov.b32 %r4, %r1;
+; DISALLOW-NEXT: @%p1 bra $L__BB1_1;
+; DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; DISALLOW-NEXT: st.param.b32 [func_retval0], %r1;
+; DISALLOW-NEXT: ret;
;
; ALLOW-LABEL: fadd_f32_generic(
; ALLOW: {
diff --git a/llvm/test/CodeGen/NVPTX/atomicrmw-sm60.ll b/llvm/test/CodeGen/NVPTX/atomicrmw-sm60.ll
index 40927ea1f68b4..5a43d78501ae0 100644
--- a/llvm/test/CodeGen/NVPTX/atomicrmw-sm60.ll
+++ b/llvm/test/CodeGen/NVPTX/atomicrmw-sm60.ll
@@ -1,7 +1,15 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
-; RUN: llc < %s -march=nvptx64 -mcpu=sm_60 -mattr=+ptx50 | FileCheck %s --check-prefixes=SM60,SM60-NOFTZ
-; RUN: llc < %s -march=nvptx64 -mcpu=sm_60 -mattr=+ptx50 -denormal-fp-math-f32=preserve-sign | FileCheck %s --check-prefixes=SM60,SM60-FTZ
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_60 -mattr=+ptx50 -nvptx-allow-ftz-atomics=false | FileCheck %s --check-prefixes=SM60,SM60-NOFTZ-DISALLOW
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_60 -mattr=+ptx50 -nvptx-allow-ftz-atomics=true | FileCheck %s --check-prefixes=SM60,SM60-NOFTZ-ALLOW
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_60 -mattr=+ptx50 | FileCheck %s --check-prefixes=SM60,SM60-NOFTZ-ALLOW
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_60 -mattr=+ptx50 -denormal-fp-math-f32=preserve-sign -nvptx-allow-ftz-atomics=false | FileCheck %s --check-prefixes=SM60,SM60-FTZ-DISALLOW
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_60 -mattr=+ptx50 -denormal-fp-math-f32=preserve-sign -nvptx-allow-ftz-atomics=true | FileCheck %s --check-prefixes=SM60,SM60-FTZ-ALLOW
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_60 -mattr=+ptx50 -denormal-fp-math-f32=preserve-sign | FileCheck %s --check-prefixes=SM60,SM60-FTZ-ALLOW
+; RUN: %if ptxas-sm_60 && ptxas-isa-5.0 %{ llc < %s -march=nvptx64 -mcpu=sm_60 -mattr=+ptx50 -nvptx-allow-ftz-atomics=false | %ptxas-verify -arch=sm_60 %}
+; RUN: %if ptxas-sm_60 && ptxas-isa-5.0 %{ llc < %s -march=nvptx64 -mcpu=sm_60 -mattr=+ptx50 -nvptx-allow-ftz-atomics=true | %ptxas-verify -arch=sm_60 %}
; RUN: %if ptxas-sm_60 && ptxas-isa-5.0 %{ llc < %s -march=nvptx64 -mcpu=sm_60 -mattr=+ptx50 | %ptxas-verify -arch=sm_60 %}
+; RUN: %if ptxas-sm_60 && ptxas-isa-5.0 %{ llc < %s -march=nvptx64 -mcpu=sm_60 -mattr=+ptx50 -denormal-fp-math-f32=preserve-sign -nvptx-allow-ftz-atomics=false | %ptxas-verify -arch=sm_60 %}
+; RUN: %if ptxas-sm_60 && ptxas-isa-5.0 %{ llc < %s -march=nvptx64 -mcpu=sm_60 -mattr=+ptx50 -denormal-fp-math-f32=preserve-sign -nvptx-allow-ftz-atomics=true | %ptxas-verify -arch=sm_60 %}
; RUN: %if ptxas-sm_60 && ptxas-isa-5.0 %{ llc < %s -march=nvptx64 -mcpu=sm_60 -mattr=+ptx50 -denormal-fp-math-f32=preserve-sign | %ptxas-verify -arch=sm_60 %}
define i8 @xchg_acq_rel_i8_global_cta(ptr addrspace(1) %addr, i8 %val) {
@@ -1739,314 +1747,592 @@ define i64 @usub_sat_acq_rel_i64_global_cta(ptr addrspace(1) %addr, i64 %val) {
}
define float @fadd_acq_rel_float_global_cta(ptr addrspace(1) %addr, float %val) {
-; SM60-NOFTZ-LABEL: fadd_acq_rel_float_global_cta(
-; SM60-NOFTZ: {
-; SM60-NOFTZ-NEXT: .reg .pred %p<2>;
-; SM60-NOFTZ-NEXT: .reg .b32 %r<5>;
-; SM60-NOFTZ-NEXT: .reg .b64 %rd<2>;
-; SM60-NOFTZ-EMPTY:
-; SM60-NOFTZ-NEXT: // %bb.0:
-; SM60-NOFTZ-NEXT: ld.param.b32 %r2, [fadd_acq_rel_float_global_cta_param_1];
-; SM60-NOFTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_cta_param_0];
-; SM60-NOFTZ-NEXT: membar.cta;
-; SM60-NOFTZ-NEXT: ld.volatile.global.b32 %r4, [%rd1];
-; SM60-NOFTZ-NEXT: $L__BB60_1: // %atomicrmw.start
-; SM60-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NOFTZ-NEXT: add.rn.f32 %r3, %r4, %r2;
-; SM60-NOFTZ-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
-; SM60-NOFTZ-NEXT: setp.ne.b32 %p1, %r1, %r4;
-; SM60-NOFTZ-NEXT: mov.b32 %r4, %r1;
-; SM60-NOFTZ-NEXT: @%p1 bra $L__BB60_1;
-; SM60-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end
-; SM60-NOFTZ-NEXT: membar.cta;
-; SM60-NOFTZ-NEXT: st.param.b32 [func_retval0], %r1;
-; SM60-NOFTZ-NEXT: ret;
+; SM60-NOFTZ-DISALLOW-LABEL: fadd_acq_rel_float_global_cta(
+; SM60-NOFTZ-DISALLOW: {
+; SM60-NOFTZ-DISALLOW-NEXT: .reg .pred %p<2>;
+; SM60-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<5>;
+; SM60-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
+; SM60-NOFTZ-DISALLOW-EMPTY:
+; SM60-NOFTZ-DISALLOW-NEXT: // %bb.0:
+; SM60-NOFTZ-DISALLOW-NEXT: ld.param.b32 %r2, [fadd_acq_rel_float_global_cta_param_1];
+; SM60-NOFTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_cta_param_0];
+; SM60-NOFTZ-DISALLOW-NEXT: membar.cta;
+; SM60-NOFTZ-DISALLOW-NEXT: ld.volatile.global.b32 %r4, [%rd1];
+; SM60-NOFTZ-DISALLOW-NEXT: $L__BB60_1: // %atomicrmw.start
+; SM60-NOFTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NOFTZ-DISALLOW-NEXT: add.rn.f32 %r3, %r4, %r2;
+; SM60-NOFTZ-DISALLOW-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM60-NOFTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM60-NOFTZ-DISALLOW-NEXT: mov.b32 %r4, %r1;
+; SM60-NOFTZ-DISALLOW-NEXT: @%p1 bra $L__BB60_1;
+; SM60-NOFTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NOFTZ-DISALLOW-NEXT: membar.cta;
+; SM60-NOFTZ-DISALLOW-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NOFTZ-DISALLOW-NEXT: ret;
;
-; SM60-FTZ-LABEL: fadd_acq_rel_float_global_cta(
-; SM60-FTZ: {
-; SM60-FTZ-NEXT: .reg .b32 %r<3>;
-; SM60-FTZ-NEXT: .reg .b64 %rd<2>;
-; SM60-FTZ-EMPTY:
-; SM60-FTZ-NEXT: // %bb.0:
-; SM60-FTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_cta_param_0];
-; SM60-FTZ-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_global_cta_param_1];
-; SM60-FTZ-NEXT: atom.cta.global.add.f32 %r2, [%rd1], %r1;
-; SM60-FTZ-NEXT: st.param.b32 [func_retval0], %r2;
-; SM60-FTZ-NEXT: ret;
+; SM60-NOFTZ-ALLOW-LABEL: fadd_acq_rel_float_global_cta(
+; SM60-NOFTZ-ALLOW: {
+; SM60-NOFTZ-ALLOW-NEXT: .reg .b32 %r<3>;
+; SM60-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
+; SM60-NOFTZ-ALLOW-EMPTY:
+; SM60-NOFTZ-ALLOW-NEXT: // %bb.0:
+; SM60-NOFTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_cta_param_0];
+; SM60-NOFTZ-ALLOW-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_global_cta_param_1];
+; SM60-NOFTZ-ALLOW-NEXT: atom.cta.global.add.f32 %r2, [%rd1], %r1;
+; SM60-NOFTZ-ALLOW-NEXT: st.param.b32 [func_retval0], %r2;
+; SM60-NOFTZ-ALLOW-NEXT: ret;
+;
+; SM60-FTZ-DISALLOW-LABEL: fadd_acq_rel_float_global_cta(
+; SM60-FTZ-DISALLOW: {
+; SM60-FTZ-DISALLOW-NEXT: .reg .b32 %r<3>;
+; SM60-FTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
+; SM60-FTZ-DISALLOW-EMPTY:
+; SM60-FTZ-DISALLOW-NEXT: // %bb.0:
+; SM60-FTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_cta_param_0];
+; SM60-FTZ-DISALLOW-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_global_cta_param_1];
+; SM60-FTZ-DISALLOW-NEXT: atom.cta.global.add.f32 %r2, [%rd1], %r1;
+; SM60-FTZ-DISALLOW-NEXT: st.param.b32 [func_retval0], %r2;
+; SM60-FTZ-DISALLOW-NEXT: ret;
+;
+; SM60-FTZ-ALLOW-LABEL: fadd_acq_rel_float_global_cta(
+; SM60-FTZ-ALLOW: {
+; SM60-FTZ-ALLOW-NEXT: .reg .b32 %r<3>;
+; SM60-FTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
+; SM60-FTZ-ALLOW-EMPTY:
+; SM60-FTZ-ALLOW-NEXT: // %bb.0:
+; SM60-FTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_cta_param_0];
+; SM60-FTZ-ALLOW-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_global_cta_param_1];
+; SM60-FTZ-ALLOW-NEXT: atom.cta.global.add.f32 %r2, [%rd1], %r1;
+; SM60-FTZ-ALLOW-NEXT: st.param.b32 [func_retval0], %r2;
+; SM60-FTZ-ALLOW-NEXT: ret;
%retval = atomicrmw fadd ptr addrspace(1) %addr, float %val syncscope("block") acq_rel
ret float %retval
}
define float @fsub_acq_rel_float_global_cta(ptr addrspace(1) %addr, float %val) {
-; SM60-NOFTZ-LABEL: fsub_acq_rel_float_global_cta(
-; SM60-NOFTZ: {
-; SM60-NOFTZ-NEXT: .reg .pred %p<2>;
-; SM60-NOFTZ-NEXT: .reg .b32 %r<5>;
-; SM60-NOFTZ-NEXT: .reg .b64 %rd<2>;
-; SM60-NOFTZ-EMPTY:
-; SM60-NOFTZ-NEXT: // %bb.0:
-; SM60-NOFTZ-NEXT: ld.param.b32 %r2, [fsub_acq_rel_float_global_cta_param_1];
-; SM60-NOFTZ-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_float_global_cta_param_0];
-; SM60-NOFTZ-NEXT: membar.cta;
-; SM60-NOFTZ-NEXT: ld.volatile.global.b32 %r4, [%rd1];
-; SM60-NOFTZ-NEXT: $L__BB61_1: // %atomicrmw.start
-; SM60-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NOFTZ-NEXT: sub.rn.f32 %r3, %r4, %r2;
-; SM60-NOFTZ-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
-; SM60-NOFTZ-NEXT: setp.ne.b32 %p1, %r1, %r4;
-; SM60-NOFTZ-NEXT: mov.b32 %r4, %r1;
-; SM60-NOFTZ-NEXT: @%p1 bra $L__BB61_1;
-; SM60-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end
-; SM60-NOFTZ-NEXT: membar.cta;
-; SM60-NOFTZ-NEXT: st.param.b32 [func_retval0], %r1;
-; SM60-NOFTZ-NEXT: ret;
+; SM60-NOFTZ-DISALLOW-LABEL: fsub_acq_rel_float_global_cta(
+; SM60-NOFTZ-DISALLOW: {
+; SM60-NOFTZ-DISALLOW-NEXT: .reg .pred %p<2>;
+; SM60-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<5>;
+; SM60-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
+; SM60-NOFTZ-DISALLOW-EMPTY:
+; SM60-NOFTZ-DISALLOW-NEXT: // %bb.0:
+; SM60-NOFTZ-DISALLOW-NEXT: ld.param.b32 %r2, [fsub_acq_rel_float_global_cta_param_1];
+; SM60-NOFTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_float_global_cta_param_0];
+; SM60-NOFTZ-DISALLOW-NEXT: membar.cta;
+; SM60-NOFTZ-DISALLOW-NEXT: ld.volatile.global.b32 %r4, [%rd1];
+; SM60-NOFTZ-DISALLOW-NEXT: $L__BB61_1: // %atomicrmw.start
+; SM60-NOFTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NOFTZ-DISALLOW-NEXT: sub.rn.f32 %r3, %r4, %r2;
+; SM60-NOFTZ-DISALLOW-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM60-NOFTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM60-NOFTZ-DISALLOW-NEXT: mov.b32 %r4, %r1;
+; SM60-NOFTZ-DISALLOW-NEXT: @%p1 bra $L__BB61_1;
+; SM60-NOFTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NOFTZ-DISALLOW-NEXT: membar.cta;
+; SM60-NOFTZ-DISALLOW-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NOFTZ-DISALLOW-NEXT: ret;
+;
+; SM60-NOFTZ-ALLOW-LABEL: fsub_acq_rel_float_global_cta(
+; SM60-NOFTZ-ALLOW: {
+; SM60-NOFTZ-ALLOW-NEXT: .reg .pred %p<2>;
+; SM60-NOFTZ-ALLOW-NEXT: .reg .b32 %r<5>;
+; SM60-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
+; SM60-NOFTZ-ALLOW-EMPTY:
+; SM60-NOFTZ-ALLOW-NEXT: // %bb.0:
+; SM60-NOFTZ-ALLOW-NEXT: ld.param.b32 %r2, [fsub_acq_rel_float_global_cta_param_1];
+; SM60-NOFTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_float_global_cta_param_0];
+; SM60-NOFTZ-ALLOW-NEXT: membar.cta;
+; SM60-NOFTZ-ALLOW-NEXT: ld.volatile.global.b32 %r4, [%rd1];
+; SM60-NOFTZ-ALLOW-NEXT: $L__BB61_1: // %atomicrmw.start
+; SM60-NOFTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NOFTZ-ALLOW-NEXT: sub.rn.f32 %r3, %r4, %r2;
+; SM60-NOFTZ-ALLOW-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM60-NOFTZ-ALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM60-NOFTZ-ALLOW-NEXT: mov.b32 %r4, %r1;
+; SM60-NOFTZ-ALLOW-NEXT: @%p1 bra $L__BB61_1;
+; SM60-NOFTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NOFTZ-ALLOW-NEXT: membar.cta;
+; SM60-NOFTZ-ALLOW-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NOFTZ-ALLOW-NEXT: ret;
;
-; SM60-FTZ-LABEL: fsub_acq_rel_float_global_cta(
-; SM60-FTZ: {
-; SM60-FTZ-NEXT: .reg .pred %p<2>;
-; SM60-FTZ-NEXT: .reg .b32 %r<5>;
-; SM60-FTZ-NEXT: .reg .b64 %rd<2>;
-; SM60-FTZ-EMPTY:
-; SM60-FTZ-NEXT: // %bb.0:
-; SM60-FTZ-NEXT: ld.param.b32 %r2, [fsub_acq_rel_float_global_cta_param_1];
-; SM60-FTZ-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_float_global_cta_param_0];
-; SM60-FTZ-NEXT: membar.cta;
-; SM60-FTZ-NEXT: ld.volatile.global.b32 %r4, [%rd1];
-; SM60-FTZ-NEXT: $L__BB61_1: // %atomicrmw.start
-; SM60-FTZ-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-FTZ-NEXT: sub.rn.ftz.f32 %r3, %r4, %r2;
-; SM60-FTZ-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
-; SM60-FTZ-NEXT: setp.ne.b32 %p1, %r1, %r4;
-; SM60-FTZ-NEXT: mov.b32 %r4, %r1;
-; SM60-FTZ-NEXT: @%p1 bra $L__BB61_1;
-; SM60-FTZ-NEXT: // %bb.2: // %atomicrmw.end
-; SM60-FTZ-NEXT: membar.cta;
-; SM60-FTZ-NEXT: st.param.b32 [func_retval0], %r1;
-; SM60-FTZ-NEXT: ret;
+; SM60-FTZ-DISALLOW-LABEL: fsub_acq_rel_float_global_cta(
+; SM60-FTZ-DISALLOW: {
+; SM60-FTZ-DISALLOW-NEXT: .reg .pred %p<2>;
+; SM60-FTZ-DISALLOW-NEXT: .reg .b32 %r<5>;
+; SM60-FTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
+; SM60-FTZ-DISALLOW-EMPTY:
+; SM60-FTZ-DISALLOW-NEXT: // %bb.0:
+; SM60-FTZ-DISALLOW-NEXT: ld.param.b32 %r2, [fsub_acq_rel_float_global_cta_param_1];
+; SM60-FTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_float_global_cta_param_0];
+; SM60-FTZ-DISALLOW-NEXT: membar.cta;
+; SM60-FTZ-DISALLOW-NEXT: ld.volatile.global.b32 %r4, [%rd1];
+; SM60-FTZ-DISALLOW-NEXT: $L__BB61_1: // %atomicrmw.start
+; SM60-FTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-FTZ-DISALLOW-NEXT: sub.rn.ftz.f32 %r3, %r4, %r2;
+; SM60-FTZ-DISALLOW-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM60-FTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM60-FTZ-DISALLOW-NEXT: mov.b32 %r4, %r1;
+; SM60-FTZ-DISALLOW-NEXT: @%p1 bra $L__BB61_1;
+; SM60-FTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-FTZ-DISALLOW-NEXT: membar.cta;
+; SM60-FTZ-DISALLOW-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-FTZ-DISALLOW-NEXT: ret;
+;
+; SM60-FTZ-ALLOW-LABEL: fsub_acq_rel_float_global_cta(
+; SM60-FTZ-ALLOW: {
+; SM60-FTZ-ALLOW-NEXT: .reg .pred %p<2>;
+; SM60-FTZ-ALLOW-NEXT: .reg .b32 %r<5>;
+; SM60-FTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
+; SM60-FTZ-ALLOW-EMPTY:
+; SM60-FTZ-ALLOW-NEXT: // %bb.0:
+; SM60-FTZ-ALLOW-NEXT: ld.param.b32 %r2, [fsub_acq_rel_float_global_cta_param_1];
+; SM60-FTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_float_global_cta_param_0];
+; SM60-FTZ-ALLOW-NEXT: membar.cta;
+; SM60-FTZ-ALLOW-NEXT: ld.volatile.global.b32 %r4, [%rd1];
+; SM60-FTZ-ALLOW-NEXT: $L__BB61_1: // %atomicrmw.start
+; SM60-FTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-FTZ-ALLOW-NEXT: sub.rn.ftz.f32 %r3, %r4, %r2;
+; SM60-FTZ-ALLOW-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM60-FTZ-ALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM60-FTZ-ALLOW-NEXT: mov.b32 %r4, %r1;
+; SM60-FTZ-ALLOW-NEXT: @%p1 bra $L__BB61_1;
+; SM60-FTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-FTZ-ALLOW-NEXT: membar.cta;
+; SM60-FTZ-ALLOW-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-FTZ-ALLOW-NEXT: ret;
%retval = atomicrmw fsub ptr addrspace(1) %addr, float %val syncscope("block") acq_rel
ret float %retval
}
define float @fmin_acq_rel_float_global_cta(ptr addrspace(1) %addr, float %val) {
-; SM60-NOFTZ-LABEL: fmin_acq_rel_float_global_cta(
-; SM60-NOFTZ: {
-; SM60-NOFTZ-NEXT: .reg .pred %p<2>;
-; SM60-NOFTZ-NEXT: .reg .b32 %r<5>;
-; SM60-NOFTZ-NEXT: .reg .b64 %rd<2>;
-; SM60-NOFTZ-EMPTY:
-; SM60-NOFTZ-NEXT: // %bb.0:
-; SM60-NOFTZ-NEXT: ld.param.b32 %r2, [fmin_acq_rel_float_global_cta_param_1];
-; SM60-NOFTZ-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_float_global_cta_param_0];
-; SM60-NOFTZ-NEXT: membar.cta;
-; SM60-NOFTZ-NEXT: ld.volatile.global.b32 %r4, [%rd1];
-; SM60-NOFTZ-NEXT: $L__BB62_1: // %atomicrmw.start
-; SM60-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NOFTZ-NEXT: min.f32 %r3, %r4, %r2;
-; SM60-NOFTZ-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
-; SM60-NOFTZ-NEXT: setp.ne.b32 %p1, %r1, %r4;
-; SM60-NOFTZ-NEXT: mov.b32 %r4, %r1;
-; SM60-NOFTZ-NEXT: @%p1 bra $L__BB62_1;
-; SM60-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end
-; SM60-NOFTZ-NEXT: membar.cta;
-; SM60-NOFTZ-NEXT: st.param.b32 [func_retval0], %r1;
-; SM60-NOFTZ-NEXT: ret;
+; SM60-NOFTZ-DISALLOW-LABEL: fmin_acq_rel_float_global_cta(
+; SM60-NOFTZ-DISALLOW: {
+; SM60-NOFTZ-DISALLOW-NEXT: .reg .pred %p<2>;
+; SM60-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<5>;
+; SM60-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
+; SM60-NOFTZ-DISALLOW-EMPTY:
+; SM60-NOFTZ-DISALLOW-NEXT: // %bb.0:
+; SM60-NOFTZ-DISALLOW-NEXT: ld.param.b32 %r2, [fmin_acq_rel_float_global_cta_param_1];
+; SM60-NOFTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_float_global_cta_param_0];
+; SM60-NOFTZ-DISALLOW-NEXT: membar.cta;
+; SM60-NOFTZ-DISALLOW-NEXT: ld.volatile.global.b32 %r4, [%rd1];
+; SM60-NOFTZ-DISALLOW-NEXT: $L__BB62_1: // %atomicrmw.start
+; SM60-NOFTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NOFTZ-DISALLOW-NEXT: min.f32 %r3, %r4, %r2;
+; SM60-NOFTZ-DISALLOW-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM60-NOFTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM60-NOFTZ-DISALLOW-NEXT: mov.b32 %r4, %r1;
+; SM60-NOFTZ-DISALLOW-NEXT: @%p1 bra $L__BB62_1;
+; SM60-NOFTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NOFTZ-DISALLOW-NEXT: membar.cta;
+; SM60-NOFTZ-DISALLOW-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NOFTZ-DISALLOW-NEXT: ret;
+;
+; SM60-NOFTZ-ALLOW-LABEL: fmin_acq_rel_float_global_cta(
+; SM60-NOFTZ-ALLOW: {
+; SM60-NOFTZ-ALLOW-NEXT: .reg .pred %p<2>;
+; SM60-NOFTZ-ALLOW-NEXT: .reg .b32 %r<5>;
+; SM60-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
+; SM60-NOFTZ-ALLOW-EMPTY:
+; SM60-NOFTZ-ALLOW-NEXT: // %bb.0:
+; SM60-NOFTZ-ALLOW-NEXT: ld.param.b32 %r2, [fmin_acq_rel_float_global_cta_param_1];
+; SM60-NOFTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_float_global_cta_param_0];
+; SM60-NOFTZ-ALLOW-NEXT: membar.cta;
+; SM60-NOFTZ-ALLOW-NEXT: ld.volatile.global.b32 %r4, [%rd1];
+; SM60-NOFTZ-ALLOW-NEXT: $L__BB62_1: // %atomicrmw.start
+; SM60-NOFTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NOFTZ-ALLOW-NEXT: min.f32 %r3, %r4, %r2;
+; SM60-NOFTZ-ALLOW-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM60-NOFTZ-ALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM60-NOFTZ-ALLOW-NEXT: mov.b32 %r4, %r1;
+; SM60-NOFTZ-ALLOW-NEXT: @%p1 bra $L__BB62_1;
+; SM60-NOFTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NOFTZ-ALLOW-NEXT: membar.cta;
+; SM60-NOFTZ-ALLOW-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NOFTZ-ALLOW-NEXT: ret;
+;
+; SM60-FTZ-DISALLOW-LABEL: fmin_acq_rel_float_global_cta(
+; SM60-FTZ-DISALLOW: {
+; SM60-FTZ-DISALLOW-NEXT: .reg .pred %p<2>;
+; SM60-FTZ-DISALLOW-NEXT: .reg .b32 %r<5>;
+; SM60-FTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
+; SM60-FTZ-DISALLOW-EMPTY:
+; SM60-FTZ-DISALLOW-NEXT: // %bb.0:
+; SM60-FTZ-DISALLOW-NEXT: ld.param.b32 %r2, [fmin_acq_rel_float_global_cta_param_1];
+; SM60-FTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_float_global_cta_param_0];
+; SM60-FTZ-DISALLOW-NEXT: membar.cta;
+; SM60-FTZ-DISALLOW-NEXT: ld.volatile.global.b32 %r4, [%rd1];
+; SM60-FTZ-DISALLOW-NEXT: $L__BB62_1: // %atomicrmw.start
+; SM60-FTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-FTZ-DISALLOW-NEXT: min.ftz.f32 %r3, %r4, %r2;
+; SM60-FTZ-DISALLOW-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM60-FTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM60-FTZ-DISALLOW-NEXT: mov.b32 %r4, %r1;
+; SM60-FTZ-DISALLOW-NEXT: @%p1 bra $L__BB62_1;
+; SM60-FTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-FTZ-DISALLOW-NEXT: membar.cta;
+; SM60-FTZ-DISALLOW-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-FTZ-DISALLOW-NEXT: ret;
;
-; SM60-FTZ-LABEL: fmin_acq_rel_float_global_cta(
-; SM60-FTZ: {
-; SM60-FTZ-NEXT: .reg .pred %p<2>;
-; SM60-FTZ-NEXT: .reg .b32 %r<5>;
-; SM60-FTZ-NEXT: .reg .b64 %rd<2>;
-; SM60-FTZ-EMPTY:
-; SM60-FTZ-NEXT: // %bb.0:
-; SM60-FTZ-NEXT: ld.param.b32 %r2, [fmin_acq_rel_float_global_cta_param_1];
-; SM60-FTZ-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_float_global_cta_param_0];
-; SM60-FTZ-NEXT: membar.cta;
-; SM60-FTZ-NEXT: ld.volatile.global.b32 %r4, [%rd1];
-; SM60-FTZ-NEXT: $L__BB62_1: // %atomicrmw.start
-; SM60-FTZ-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-FTZ-NEXT: min.ftz.f32 %r3, %r4, %r2;
-; SM60-FTZ-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
-; SM60-FTZ-NEXT: setp.ne.b32 %p1, %r1, %r4;
-; SM60-FTZ-NEXT: mov.b32 %r4, %r1;
-; SM60-FTZ-NEXT: @%p1 bra $L__BB62_1;
-; SM60-FTZ-NEXT: // %bb.2: // %atomicrmw.end
-; SM60-FTZ-NEXT: membar.cta;
-; SM60-FTZ-NEXT: st.param.b32 [func_retval0], %r1;
-; SM60-FTZ-NEXT: ret;
+; SM60-FTZ-ALLOW-LABEL: fmin_acq_rel_float_global_cta(
+; SM60-FTZ-ALLOW: {
+; SM60-FTZ-ALLOW-NEXT: .reg .pred %p<2>;
+; SM60-FTZ-ALLOW-NEXT: .reg .b32 %r<5>;
+; SM60-FTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
+; SM60-FTZ-ALLOW-EMPTY:
+; SM60-FTZ-ALLOW-NEXT: // %bb.0:
+; SM60-FTZ-ALLOW-NEXT: ld.param.b32 %r2, [fmin_acq_rel_float_global_cta_param_1];
+; SM60-FTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_float_global_cta_param_0];
+; SM60-FTZ-ALLOW-NEXT: membar.cta;
+; SM60-FTZ-ALLOW-NEXT: ld.volatile.global.b32 %r4, [%rd1];
+; SM60-FTZ-ALLOW-NEXT: $L__BB62_1: // %atomicrmw.start
+; SM60-FTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-FTZ-ALLOW-NEXT: min.ftz.f32 %r3, %r4, %r2;
+; SM60-FTZ-ALLOW-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM60-FTZ-ALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM60-FTZ-ALLOW-NEXT: mov.b32 %r4, %r1;
+; SM60-FTZ-ALLOW-NEXT: @%p1 bra $L__BB62_1;
+; SM60-FTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-FTZ-ALLOW-NEXT: membar.cta;
+; SM60-FTZ-ALLOW-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-FTZ-ALLOW-NEXT: ret;
%retval = atomicrmw fmin ptr addrspace(1) %addr, float %val syncscope("block") acq_rel
ret float %retval
}
define float @fmax_acq_rel_float_global_cta(ptr addrspace(1) %addr, float %val) {
-; SM60-NOFTZ-LABEL: fmax_acq_rel_float_global_cta(
-; SM60-NOFTZ: {
-; SM60-NOFTZ-NEXT: .reg .pred %p<2>;
-; SM60-NOFTZ-NEXT: .reg .b32 %r<5>;
-; SM60-NOFTZ-NEXT: .reg .b64 %rd<2>;
-; SM60-NOFTZ-EMPTY:
-; SM60-NOFTZ-NEXT: // %bb.0:
-; SM60-NOFTZ-NEXT: ld.param.b32 %r2, [fmax_acq_rel_float_global_cta_param_1];
-; SM60-NOFTZ-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_float_global_cta_param_0];
-; SM60-NOFTZ-NEXT: membar.cta;
-; SM60-NOFTZ-NEXT: ld.volatile.global.b32 %r4, [%rd1];
-; SM60-NOFTZ-NEXT: $L__BB63_1: // %atomicrmw.start
-; SM60-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NOFTZ-NEXT: max.f32 %r3, %r4, %r2;
-; SM60-NOFTZ-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
-; SM60-NOFTZ-NEXT: setp.ne.b32 %p1, %r1, %r4;
-; SM60-NOFTZ-NEXT: mov.b32 %r4, %r1;
-; SM60-NOFTZ-NEXT: @%p1 bra $L__BB63_1;
-; SM60-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end
-; SM60-NOFTZ-NEXT: membar.cta;
-; SM60-NOFTZ-NEXT: st.param.b32 [func_retval0], %r1;
-; SM60-NOFTZ-NEXT: ret;
+; SM60-NOFTZ-DISALLOW-LABEL: fmax_acq_rel_float_global_cta(
+; SM60-NOFTZ-DISALLOW: {
+; SM60-NOFTZ-DISALLOW-NEXT: .reg .pred %p<2>;
+; SM60-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<5>;
+; SM60-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
+; SM60-NOFTZ-DISALLOW-EMPTY:
+; SM60-NOFTZ-DISALLOW-NEXT: // %bb.0:
+; SM60-NOFTZ-DISALLOW-NEXT: ld.param.b32 %r2, [fmax_acq_rel_float_global_cta_param_1];
+; SM60-NOFTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_float_global_cta_param_0];
+; SM60-NOFTZ-DISALLOW-NEXT: membar.cta;
+; SM60-NOFTZ-DISALLOW-NEXT: ld.volatile.global.b32 %r4, [%rd1];
+; SM60-NOFTZ-DISALLOW-NEXT: $L__BB63_1: // %atomicrmw.start
+; SM60-NOFTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NOFTZ-DISALLOW-NEXT: max.f32 %r3, %r4, %r2;
+; SM60-NOFTZ-DISALLOW-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM60-NOFTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM60-NOFTZ-DISALLOW-NEXT: mov.b32 %r4, %r1;
+; SM60-NOFTZ-DISALLOW-NEXT: @%p1 bra $L__BB63_1;
+; SM60-NOFTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NOFTZ-DISALLOW-NEXT: membar.cta;
+; SM60-NOFTZ-DISALLOW-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NOFTZ-DISALLOW-NEXT: ret;
;
-; SM60-FTZ-LABEL: fmax_acq_rel_float_global_cta(
-; SM60-FTZ: {
-; SM60-FTZ-NEXT: .reg .pred %p<2>;
-; SM60-FTZ-NEXT: .reg .b32 %r<5>;
-; SM60-FTZ-NEXT: .reg .b64 %rd<2>;
-; SM60-FTZ-EMPTY:
-; SM60-FTZ-NEXT: // %bb.0:
-; SM60-FTZ-NEXT: ld.param.b32 %r2, [fmax_acq_rel_float_global_cta_param_1];
-; SM60-FTZ-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_float_global_cta_param_0];
-; SM60-FTZ-NEXT: membar.cta;
-; SM60-FTZ-NEXT: ld.volatile.global.b32 %r4, [%rd1];
-; SM60-FTZ-NEXT: $L__BB63_1: // %atomicrmw.start
-; SM60-FTZ-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-FTZ-NEXT: max.ftz.f32 %r3, %r4, %r2;
-; SM60-FTZ-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
-; SM60-FTZ-NEXT: setp.ne.b32 %p1, %r1, %r4;
-; SM60-FTZ-NEXT: mov.b32 %r4, %r1;
-; SM60-FTZ-NEXT: @%p1 bra $L__BB63_1;
-; SM60-FTZ-NEXT: // %bb.2: // %atomicrmw.end
-; SM60-FTZ-NEXT: membar.cta;
-; SM60-FTZ-NEXT: st.param.b32 [func_retval0], %r1;
-; SM60-FTZ-NEXT: ret;
+; SM60-NOFTZ-ALLOW-LABEL: fmax_acq_rel_float_global_cta(
+; SM60-NOFTZ-ALLOW: {
+; SM60-NOFTZ-ALLOW-NEXT: .reg .pred %p<2>;
+; SM60-NOFTZ-ALLOW-NEXT: .reg .b32 %r<5>;
+; SM60-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
+; SM60-NOFTZ-ALLOW-EMPTY:
+; SM60-NOFTZ-ALLOW-NEXT: // %bb.0:
+; SM60-NOFTZ-ALLOW-NEXT: ld.param.b32 %r2, [fmax_acq_rel_float_global_cta_param_1];
+; SM60-NOFTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_float_global_cta_param_0];
+; SM60-NOFTZ-ALLOW-NEXT: membar.cta;
+; SM60-NOFTZ-ALLOW-NEXT: ld.volatile.global.b32 %r4, [%rd1];
+; SM60-NOFTZ-ALLOW-NEXT: $L__BB63_1: // %atomicrmw.start
+; SM60-NOFTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NOFTZ-ALLOW-NEXT: max.f32 %r3, %r4, %r2;
+; SM60-NOFTZ-ALLOW-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM60-NOFTZ-ALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM60-NOFTZ-ALLOW-NEXT: mov.b32 %r4, %r1;
+; SM60-NOFTZ-ALLOW-NEXT: @%p1 bra $L__BB63_1;
+; SM60-NOFTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NOFTZ-ALLOW-NEXT: membar.cta;
+; SM60-NOFTZ-ALLOW-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NOFTZ-ALLOW-NEXT: ret;
+;
+; SM60-FTZ-DISALLOW-LABEL: fmax_acq_rel_float_global_cta(
+; SM60-FTZ-DISALLOW: {
+; SM60-FTZ-DISALLOW-NEXT: .reg .pred %p<2>;
+; SM60-FTZ-DISALLOW-NEXT: .reg .b32 %r<5>;
+; SM60-FTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
+; SM60-FTZ-DISALLOW-EMPTY:
+; SM60-FTZ-DISALLOW-NEXT: // %bb.0:
+; SM60-FTZ-DISALLOW-NEXT: ld.param.b32 %r2, [fmax_acq_rel_float_global_cta_param_1];
+; SM60-FTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_float_global_cta_param_0];
+; SM60-FTZ-DISALLOW-NEXT: membar.cta;
+; SM60-FTZ-DISALLOW-NEXT: ld.volatile.global.b32 %r4, [%rd1];
+; SM60-FTZ-DISALLOW-NEXT: $L__BB63_1: // %atomicrmw.start
+; SM60-FTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-FTZ-DISALLOW-NEXT: max.ftz.f32 %r3, %r4, %r2;
+; SM60-FTZ-DISALLOW-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM60-FTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM60-FTZ-DISALLOW-NEXT: mov.b32 %r4, %r1;
+; SM60-FTZ-DISALLOW-NEXT: @%p1 bra $L__BB63_1;
+; SM60-FTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-FTZ-DISALLOW-NEXT: membar.cta;
+; SM60-FTZ-DISALLOW-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-FTZ-DISALLOW-NEXT: ret;
+;
+; SM60-FTZ-ALLOW-LABEL: fmax_acq_rel_float_global_cta(
+; SM60-FTZ-ALLOW: {
+; SM60-FTZ-ALLOW-NEXT: .reg .pred %p<2>;
+; SM60-FTZ-ALLOW-NEXT: .reg .b32 %r<5>;
+; SM60-FTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
+; SM60-FTZ-ALLOW-EMPTY:
+; SM60-FTZ-ALLOW-NEXT: // %bb.0:
+; SM60-FTZ-ALLOW-NEXT: ld.param.b32 %r2, [fmax_acq_rel_float_global_cta_param_1];
+; SM60-FTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_float_global_cta_param_0];
+; SM60-FTZ-ALLOW-NEXT: membar.cta;
+; SM60-FTZ-ALLOW-NEXT: ld.volatile.global.b32 %r4, [%rd1];
+; SM60-FTZ-ALLOW-NEXT: $L__BB63_1: // %atomicrmw.start
+; SM60-FTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-FTZ-ALLOW-NEXT: max.ftz.f32 %r3, %r4, %r2;
+; SM60-FTZ-ALLOW-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM60-FTZ-ALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM60-FTZ-ALLOW-NEXT: mov.b32 %r4, %r1;
+; SM60-FTZ-ALLOW-NEXT: @%p1 bra $L__BB63_1;
+; SM60-FTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-FTZ-ALLOW-NEXT: membar.cta;
+; SM60-FTZ-ALLOW-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-FTZ-ALLOW-NEXT: ret;
%retval = atomicrmw fmax ptr addrspace(1) %addr, float %val syncscope("block") acq_rel
ret float %retval
}
define float @fminimum_acq_rel_float_global_cta(ptr addrspace(1) %addr, float %val) {
-; SM60-NOFTZ-LABEL: fminimum_acq_rel_float_global_cta(
-; SM60-NOFTZ: {
-; SM60-NOFTZ-NEXT: .reg .pred %p<5>;
-; SM60-NOFTZ-NEXT: .reg .b32 %r<8>;
-; SM60-NOFTZ-NEXT: .reg .b64 %rd<2>;
-; SM60-NOFTZ-EMPTY:
-; SM60-NOFTZ-NEXT: // %bb.0:
-; SM60-NOFTZ-NEXT: ld.param.b32 %r2, [fminimum_acq_rel_float_global_cta_param_1];
-; SM60-NOFTZ-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_float_global_cta_param_0];
-; SM60-NOFTZ-NEXT: membar.cta;
-; SM60-NOFTZ-NEXT: ld.volatile.global.b32 %r7, [%rd1];
-; SM60-NOFTZ-NEXT: $L__BB64_1: // %atomicrmw.start
-; SM60-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NOFTZ-NEXT: setp.eq.b32 %p1, %r7, -2147483648;
-; SM60-NOFTZ-NEXT: selp.f32 %r3, %r7, %r2, %p1;
-; SM60-NOFTZ-NEXT: setp.nan.f32 %p2, %r7, %r2;
-; SM60-NOFTZ-NEXT: min.f32 %r4, %r7, %r2;
-; SM60-NOFTZ-NEXT: selp.f32 %r5, 0f7FC00000, %r4, %p2;
-; SM60-NOFTZ-NEXT: setp.eq.f32 %p3, %r7, %r2;
-; SM60-NOFTZ-NEXT: selp.f32 %r6, %r3, %r5, %p3;
-; SM60-NOFTZ-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r7, %r6;
-; SM60-NOFTZ-NEXT: setp.ne.b32 %p4, %r1, %r7;
-; SM60-NOFTZ-NEXT: mov.b32 %r7, %r1;
-; SM60-NOFTZ-NEXT: @%p4 bra $L__BB64_1;
-; SM60-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end
-; SM60-NOFTZ-NEXT: membar.cta;
-; SM60-NOFTZ-NEXT: st.param.b32 [func_retval0], %r1;
-; SM60-NOFTZ-NEXT: ret;
+; SM60-NOFTZ-DISALLOW-LABEL: fminimum_acq_rel_float_global_cta(
+; SM60-NOFTZ-DISALLOW: {
+; SM60-NOFTZ-DISALLOW-NEXT: .reg .pred %p<5>;
+; SM60-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<8>;
+; SM60-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
+; SM60-NOFTZ-DISALLOW-EMPTY:
+; SM60-NOFTZ-DISALLOW-NEXT: // %bb.0:
+; SM60-NOFTZ-DISALLOW-NEXT: ld.param.b32 %r2, [fminimum_acq_rel_float_global_cta_param_1];
+; SM60-NOFTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_float_global_cta_param_0];
+; SM60-NOFTZ-DISALLOW-NEXT: membar.cta;
+; SM60-NOFTZ-DISALLOW-NEXT: ld.volatile.global.b32 %r7, [%rd1];
+; SM60-NOFTZ-DISALLOW-NEXT: $L__BB64_1: // %atomicrmw.start
+; SM60-NOFTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NOFTZ-DISALLOW-NEXT: setp.eq.b32 %p1, %r7, -2147483648;
+; SM60-NOFTZ-DISALLOW-NEXT: selp.f32 %r3, %r7, %r2, %p1;
+; SM60-NOFTZ-DISALLOW-NEXT: setp.nan.f32 %p2, %r7, %r2;
+; SM60-NOFTZ-DISALLOW-NEXT: min.f32 %r4, %r7, %r2;
+; SM60-NOFTZ-DISALLOW-NEXT: selp.f32 %r5, 0f7FC00000, %r4, %p2;
+; SM60-NOFTZ-DISALLOW-NEXT: setp.eq.f32 %p3, %r7, %r2;
+; SM60-NOFTZ-DISALLOW-NEXT: selp.f32 %r6, %r3, %r5, %p3;
+; SM60-NOFTZ-DISALLOW-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r7, %r6;
+; SM60-NOFTZ-DISALLOW-NEXT: setp.ne.b32 %p4, %r1, %r7;
+; SM60-NOFTZ-DISALLOW-NEXT: mov.b32 %r7, %r1;
+; SM60-NOFTZ-DISALLOW-NEXT: @%p4 bra $L__BB64_1;
+; SM60-NOFTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NOFTZ-DISALLOW-NEXT: membar.cta;
+; SM60-NOFTZ-DISALLOW-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NOFTZ-DISALLOW-NEXT: ret;
+;
+; SM60-NOFTZ-ALLOW-LABEL: fminimum_acq_rel_float_global_cta(
+; SM60-NOFTZ-ALLOW: {
+; SM60-NOFTZ-ALLOW-NEXT: .reg .pred %p<5>;
+; SM60-NOFTZ-ALLOW-NEXT: .reg .b32 %r<8>;
+; SM60-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
+; SM60-NOFTZ-ALLOW-EMPTY:
+; SM60-NOFTZ-ALLOW-NEXT: // %bb.0:
+; SM60-NOFTZ-ALLOW-NEXT: ld.param.b32 %r2, [fminimum_acq_rel_float_global_cta_param_1];
+; SM60-NOFTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_float_global_cta_param_0];
+; SM60-NOFTZ-ALLOW-NEXT: membar.cta;
+; SM60-NOFTZ-ALLOW-NEXT: ld.volatile.global.b32 %r7, [%rd1];
+; SM60-NOFTZ-ALLOW-NEXT: $L__BB64_1: // %atomicrmw.start
+; SM60-NOFTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NOFTZ-ALLOW-NEXT: setp.eq.b32 %p1, %r7, -2147483648;
+; SM60-NOFTZ-ALLOW-NEXT: selp.f32 %r3, %r7, %r2, %p1;
+; SM60-NOFTZ-ALLOW-NEXT: setp.nan.f32 %p2, %r7, %r2;
+; SM60-NOFTZ-ALLOW-NEXT: min.f32 %r4, %r7, %r2;
+; SM60-NOFTZ-ALLOW-NEXT: selp.f32 %r5, 0f7FC00000, %r4, %p2;
+; SM60-NOFTZ-ALLOW-NEXT: setp.eq.f32 %p3, %r7, %r2;
+; SM60-NOFTZ-ALLOW-NEXT: selp.f32 %r6, %r3, %r5, %p3;
+; SM60-NOFTZ-ALLOW-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r7, %r6;
+; SM60-NOFTZ-ALLOW-NEXT: setp.ne.b32 %p4, %r1, %r7;
+; SM60-NOFTZ-ALLOW-NEXT: mov.b32 %r7, %r1;
+; SM60-NOFTZ-ALLOW-NEXT: @%p4 bra $L__BB64_1;
+; SM60-NOFTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NOFTZ-ALLOW-NEXT: membar.cta;
+; SM60-NOFTZ-ALLOW-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NOFTZ-ALLOW-NEXT: ret;
;
-; SM60-FTZ-LABEL: fminimum_acq_rel_float_global_cta(
-; SM60-FTZ: {
-; SM60-FTZ-NEXT: .reg .pred %p<5>;
-; SM60-FTZ-NEXT: .reg .b32 %r<8>;
-; SM60-FTZ-NEXT: .reg .b64 %rd<2>;
-; SM60-FTZ-EMPTY:
-; SM60-FTZ-NEXT: // %bb.0:
-; SM60-FTZ-NEXT: ld.param.b32 %r2, [fminimum_acq_rel_float_global_cta_param_1];
-; SM60-FTZ-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_float_global_cta_param_0];
-; SM60-FTZ-NEXT: membar.cta;
-; SM60-FTZ-NEXT: ld.volatile.global.b32 %r7, [%rd1];
-; SM60-FTZ-NEXT: $L__BB64_1: // %atomicrmw.start
-; SM60-FTZ-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-FTZ-NEXT: setp.eq.b32 %p1, %r7, -2147483648;
-; SM60-FTZ-NEXT: selp.f32 %r3, %r7, %r2, %p1;
-; SM60-FTZ-NEXT: setp.nan.ftz.f32 %p2, %r7, %r2;
-; SM60-FTZ-NEXT: min.ftz.f32 %r4, %r7, %r2;
-; SM60-FTZ-NEXT: selp.f32 %r5, 0f7FC00000, %r4, %p2;
-; SM60-FTZ-NEXT: setp.eq.ftz.f32 %p3, %r7, %r2;
-; SM60-FTZ-NEXT: selp.f32 %r6, %r3, %r5, %p3;
-; SM60-FTZ-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r7, %r6;
-; SM60-FTZ-NEXT: setp.ne.b32 %p4, %r1, %r7;
-; SM60-FTZ-NEXT: mov.b32 %r7, %r1;
-; SM60-FTZ-NEXT: @%p4 bra $L__BB64_1;
-; SM60-FTZ-NEXT: // %bb.2: // %atomicrmw.end
-; SM60-FTZ-NEXT: membar.cta;
-; SM60-FTZ-NEXT: st.param.b32 [func_retval0], %r1;
-; SM60-FTZ-NEXT: ret;
+; SM60-FTZ-DISALLOW-LABEL: fminimum_acq_rel_float_global_cta(
+; SM60-FTZ-DISALLOW: {
+; SM60-FTZ-DISALLOW-NEXT: .reg .pred %p<5>;
+; SM60-FTZ-DISALLOW-NEXT: .reg .b32 %r<8>;
+; SM60-FTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
+; SM60-FTZ-DISALLOW-EMPTY:
+; SM60-FTZ-DISALLOW-NEXT: // %bb.0:
+; SM60-FTZ-DISALLOW-NEXT: ld.param.b32 %r2, [fminimum_acq_rel_float_global_cta_param_1];
+; SM60-FTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_float_global_cta_param_0];
+; SM60-FTZ-DISALLOW-NEXT: membar.cta;
+; SM60-FTZ-DISALLOW-NEXT: ld.volatile.global.b32 %r7, [%rd1];
+; SM60-FTZ-DISALLOW-NEXT: $L__BB64_1: // %atomicrmw.start
+; SM60-FTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-FTZ-DISALLOW-NEXT: setp.eq.b32 %p1, %r7, -2147483648;
+; SM60-FTZ-DISALLOW-NEXT: selp.f32 %r3, %r7, %r2, %p1;
+; SM60-FTZ-DISALLOW-NEXT: setp.nan.ftz.f32 %p2, %r7, %r2;
+; SM60-FTZ-DISALLOW-NEXT: min.ftz.f32 %r4, %r7, %r2;
+; SM60-FTZ-DISALLOW-NEXT: selp.f32 %r5, 0f7FC00000, %r4, %p2;
+; SM60-FTZ-DISALLOW-NEXT: setp.eq.ftz.f32 %p3, %r7, %r2;
+; SM60-FTZ-DISALLOW-NEXT: selp.f32 %r6, %r3, %r5, %p3;
+; SM60-FTZ-DISALLOW-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r7, %r6;
+; SM60-FTZ-DISALLOW-NEXT: setp.ne.b32 %p4, %r1, %r7;
+; SM60-FTZ-DISALLOW-NEXT: mov.b32 %r7, %r1;
+; SM60-FTZ-DISALLOW-NEXT: @%p4 bra $L__BB64_1;
+; SM60-FTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-FTZ-DISALLOW-NEXT: membar.cta;
+; SM60-FTZ-DISALLOW-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-FTZ-DISALLOW-NEXT: ret;
+;
+; SM60-FTZ-ALLOW-LABEL: fminimum_acq_rel_float_global_cta(
+; SM60-FTZ-ALLOW: {
+; SM60-FTZ-ALLOW-NEXT: .reg .pred %p<5>;
+; SM60-FTZ-ALLOW-NEXT: .reg .b32 %r<8>;
+; SM60-FTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
+; SM60-FTZ-ALLOW-EMPTY:
+; SM60-FTZ-ALLOW-NEXT: // %bb.0:
+; SM60-FTZ-ALLOW-NEXT: ld.param.b32 %r2, [fminimum_acq_rel_float_global_cta_param_1];
+; SM60-FTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_float_global_cta_param_0];
+; SM60-FTZ-ALLOW-NEXT: membar.cta;
+; SM60-FTZ-ALLOW-NEXT: ld.volatile.global.b32 %r7, [%rd1];
+; SM60-FTZ-ALLOW-NEXT: $L__BB64_1: // %atomicrmw.start
+; SM60-FTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-FTZ-ALLOW-NEXT: setp.eq.b32 %p1, %r7, -2147483648;
+; SM60-FTZ-ALLOW-NEXT: selp.f32 %r3, %r7, %r2, %p1;
+; SM60-FTZ-ALLOW-NEXT: setp.nan.ftz.f32 %p2, %r7, %r2;
+; SM60-FTZ-ALLOW-NEXT: min.ftz.f32 %r4, %r7, %r2;
+; SM60-FTZ-ALLOW-NEXT: selp.f32 %r5, 0f7FC00000, %r4, %p2;
+; SM60-FTZ-ALLOW-NEXT: setp.eq.ftz.f32 %p3, %r7, %r2;
+; SM60-FTZ-ALLOW-NEXT: selp.f32 %r6, %r3, %r5, %p3;
+; SM60-FTZ-ALLOW-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r7, %r6;
+; SM60-FTZ-ALLOW-NEXT: setp.ne.b32 %p4, %r1, %r7;
+; SM60-FTZ-ALLOW-NEXT: mov.b32 %r7, %r1;
+; SM60-FTZ-ALLOW-NEXT: @%p4 bra $L__BB64_1;
+; SM60-FTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-FTZ-ALLOW-NEXT: membar.cta;
+; SM60-FTZ-ALLOW-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-FTZ-ALLOW-NEXT: ret;
%retval = atomicrmw fminimum ptr addrspace(1) %addr, float %val syncscope("block") acq_rel
ret float %retval
}
define float @fmaximum_acq_rel_float_global_cta(ptr addrspace(1) %addr, float %val) {
-; SM60-NOFTZ-LABEL: fmaximum_acq_rel_float_global_cta(
-; SM60-NOFTZ: {
-; SM60-NOFTZ-NEXT: .reg .pred %p<5>;
-; SM60-NOFTZ-NEXT: .reg .b32 %r<8>;
-; SM60-NOFTZ-NEXT: .reg .b64 %rd<2>;
-; SM60-NOFTZ-EMPTY:
-; SM60-NOFTZ-NEXT: // %bb.0:
-; SM60-NOFTZ-NEXT: ld.param.b32 %r2, [fmaximum_acq_rel_float_global_cta_param_1];
-; SM60-NOFTZ-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_float_global_cta_param_0];
-; SM60-NOFTZ-NEXT: membar.cta;
-; SM60-NOFTZ-NEXT: ld.volatile.global.b32 %r7, [%rd1];
-; SM60-NOFTZ-NEXT: $L__BB65_1: // %atomicrmw.start
-; SM60-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NOFTZ-NEXT: setp.eq.b32 %p1, %r7, 0;
-; SM60-NOFTZ-NEXT: selp.f32 %r3, %r7, %r2, %p1;
-; SM60-NOFTZ-NEXT: setp.nan.f32 %p2, %r7, %r2;
-; SM60-NOFTZ-NEXT: max.f32 %r4, %r7, %r2;
-; SM60-NOFTZ-NEXT: selp.f32 %r5, 0f7FC00000, %r4, %p2;
-; SM60-NOFTZ-NEXT: setp.eq.f32 %p3, %r7, %r2;
-; SM60-NOFTZ-NEXT: selp.f32 %r6, %r3, %r5, %p3;
-; SM60-NOFTZ-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r7, %r6;
-; SM60-NOFTZ-NEXT: setp.ne.b32 %p4, %r1, %r7;
-; SM60-NOFTZ-NEXT: mov.b32 %r7, %r1;
-; SM60-NOFTZ-NEXT: @%p4 bra $L__BB65_1;
-; SM60-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end
-; SM60-NOFTZ-NEXT: membar.cta;
-; SM60-NOFTZ-NEXT: st.param.b32 [func_retval0], %r1;
-; SM60-NOFTZ-NEXT: ret;
+; SM60-NOFTZ-DISALLOW-LABEL: fmaximum_acq_rel_float_global_cta(
+; SM60-NOFTZ-DISALLOW: {
+; SM60-NOFTZ-DISALLOW-NEXT: .reg .pred %p<5>;
+; SM60-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<8>;
+; SM60-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
+; SM60-NOFTZ-DISALLOW-EMPTY:
+; SM60-NOFTZ-DISALLOW-NEXT: // %bb.0:
+; SM60-NOFTZ-DISALLOW-NEXT: ld.param.b32 %r2, [fmaximum_acq_rel_float_global_cta_param_1];
+; SM60-NOFTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_float_global_cta_param_0];
+; SM60-NOFTZ-DISALLOW-NEXT: membar.cta;
+; SM60-NOFTZ-DISALLOW-NEXT: ld.volatile.global.b32 %r7, [%rd1];
+; SM60-NOFTZ-DISALLOW-NEXT: $L__BB65_1: // %atomicrmw.start
+; SM60-NOFTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NOFTZ-DISALLOW-NEXT: setp.eq.b32 %p1, %r7, 0;
+; SM60-NOFTZ-DISALLOW-NEXT: selp.f32 %r3, %r7, %r2, %p1;
+; SM60-NOFTZ-DISALLOW-NEXT: setp.nan.f32 %p2, %r7, %r2;
+; SM60-NOFTZ-DISALLOW-NEXT: max.f32 %r4, %r7, %r2;
+; SM60-NOFTZ-DISALLOW-NEXT: selp.f32 %r5, 0f7FC00000, %r4, %p2;
+; SM60-NOFTZ-DISALLOW-NEXT: setp.eq.f32 %p3, %r7, %r2;
+; SM60-NOFTZ-DISALLOW-NEXT: selp.f32 %r6, %r3, %r5, %p3;
+; SM60-NOFTZ-DISALLOW-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r7, %r6;
+; SM60-NOFTZ-DISALLOW-NEXT: setp.ne.b32 %p4, %r1, %r7;
+; SM60-NOFTZ-DISALLOW-NEXT: mov.b32 %r7, %r1;
+; SM60-NOFTZ-DISALLOW-NEXT: @%p4 bra $L__BB65_1;
+; SM60-NOFTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NOFTZ-DISALLOW-NEXT: membar.cta;
+; SM60-NOFTZ-DISALLOW-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NOFTZ-DISALLOW-NEXT: ret;
+;
+; SM60-NOFTZ-ALLOW-LABEL: fmaximum_acq_rel_float_global_cta(
+; SM60-NOFTZ-ALLOW: {
+; SM60-NOFTZ-ALLOW-NEXT: .reg .pred %p<5>;
+; SM60-NOFTZ-ALLOW-NEXT: .reg .b32 %r<8>;
+; SM60-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
+; SM60-NOFTZ-ALLOW-EMPTY:
+; SM60-NOFTZ-ALLOW-NEXT: // %bb.0:
+; SM60-NOFTZ-ALLOW-NEXT: ld.param.b32 %r2, [fmaximum_acq_rel_float_global_cta_param_1];
+; SM60-NOFTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_float_global_cta_param_0];
+; SM60-NOFTZ-ALLOW-NEXT: membar.cta;
+; SM60-NOFTZ-ALLOW-NEXT: ld.volatile.global.b32 %r7, [%rd1];
+; SM60-NOFTZ-ALLOW-NEXT: $L__BB65_1: // %atomicrmw.start
+; SM60-NOFTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NOFTZ-ALLOW-NEXT: setp.eq.b32 %p1, %r7, 0;
+; SM60-NOFTZ-ALLOW-NEXT: selp.f32 %r3, %r7, %r2, %p1;
+; SM60-NOFTZ-ALLOW-NEXT: setp.nan.f32 %p2, %r7, %r2;
+; SM60-NOFTZ-ALLOW-NEXT: max.f32 %r4, %r7, %r2;
+; SM60-NOFTZ-ALLOW-NEXT: selp.f32 %r5, 0f7FC00000, %r4, %p2;
+; SM60-NOFTZ-ALLOW-NEXT: setp.eq.f32 %p3, %r7, %r2;
+; SM60-NOFTZ-ALLOW-NEXT: selp.f32 %r6, %r3, %r5, %p3;
+; SM60-NOFTZ-ALLOW-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r7, %r6;
+; SM60-NOFTZ-ALLOW-NEXT: setp.ne.b32 %p4, %r1, %r7;
+; SM60-NOFTZ-ALLOW-NEXT: mov.b32 %r7, %r1;
+; SM60-NOFTZ-ALLOW-NEXT: @%p4 bra $L__BB65_1;
+; SM60-NOFTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NOFTZ-ALLOW-NEXT: membar.cta;
+; SM60-NOFTZ-ALLOW-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NOFTZ-ALLOW-NEXT: ret;
+;
+; SM60-FTZ-DISALLOW-LABEL: fmaximum_acq_rel_float_global_cta(
+; SM60-FTZ-DISALLOW: {
+; SM60-FTZ-DISALLOW-NEXT: .reg .pred %p<5>;
+; SM60-FTZ-DISALLOW-NEXT: .reg .b32 %r<8>;
+; SM60-FTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
+; SM60-FTZ-DISALLOW-EMPTY:
+; SM60-FTZ-DISALLOW-NEXT: // %bb.0:
+; SM60-FTZ-DISALLOW-NEXT: ld.param.b32 %r2, [fmaximum_acq_rel_float_global_cta_param_1];
+; SM60-FTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_float_global_cta_param_0];
+; SM60-FTZ-DISALLOW-NEXT: membar.cta;
+; SM60-FTZ-DISALLOW-NEXT: ld.volatile.global.b32 %r7, [%rd1];
+; SM60-FTZ-DISALLOW-NEXT: $L__BB65_1: // %atomicrmw.start
+; SM60-FTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-FTZ-DISALLOW-NEXT: setp.eq.b32 %p1, %r7, 0;
+; SM60-FTZ-DISALLOW-NEXT: selp.f32 %r3, %r7, %r2, %p1;
+; SM60-FTZ-DISALLOW-NEXT: setp.nan.ftz.f32 %p2, %r7, %r2;
+; SM60-FTZ-DISALLOW-NEXT: max.ftz.f32 %r4, %r7, %r2;
+; SM60-FTZ-DISALLOW-NEXT: selp.f32 %r5, 0f7FC00000, %r4, %p2;
+; SM60-FTZ-DISALLOW-NEXT: setp.eq.ftz.f32 %p3, %r7, %r2;
+; SM60-FTZ-DISALLOW-NEXT: selp.f32 %r6, %r3, %r5, %p3;
+; SM60-FTZ-DISALLOW-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r7, %r6;
+; SM60-FTZ-DISALLOW-NEXT: setp.ne.b32 %p4, %r1, %r7;
+; SM60-FTZ-DISALLOW-NEXT: mov.b32 %r7, %r1;
+; SM60-FTZ-DISALLOW-NEXT: @%p4 bra $L__BB65_1;
+; SM60-FTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-FTZ-DISALLOW-NEXT: membar.cta;
+; SM60-FTZ-DISALLOW-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-FTZ-DISALLOW-NEXT: ret;
;
-; SM60-FTZ-LABEL: fmaximum_acq_rel_float_global_cta(
-; SM60-FTZ: {
-; SM60-FTZ-NEXT: .reg .pred %p<5>;
-; SM60-FTZ-NEXT: .reg .b32 %r<8>;
-; SM60-FTZ-NEXT: .reg .b64 %rd<2>;
-; SM60-FTZ-EMPTY:
-; SM60-FTZ-NEXT: // %bb.0:
-; SM60-FTZ-NEXT: ld.param.b32 %r2, [fmaximum_acq_rel_float_global_cta_param_1];
-; SM60-FTZ-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_float_global_cta_param_0];
-; SM60-FTZ-NEXT: membar.cta;
-; SM60-FTZ-NEXT: ld.volatile.global.b32 %r7, [%rd1];
-; SM60-FTZ-NEXT: $L__BB65_1: // %atomicrmw.start
-; SM60-FTZ-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-FTZ-NEXT: setp.eq.b32 %p1, %r7, 0;
-; SM60-FTZ-NEXT: selp.f32 %r3, %r7, %r2, %p1;
-; SM60-FTZ-NEXT: setp.nan.ftz.f32 %p2, %r7, %r2;
-; SM60-FTZ-NEXT: max.ftz.f32 %r4, %r7, %r2;
-; SM60-FTZ-NEXT: selp.f32 %r5, 0f7FC00000, %r4, %p2;
-; SM60-FTZ-NEXT: setp.eq.ftz.f32 %p3, %r7, %r2;
-; SM60-FTZ-NEXT: selp.f32 %r6, %r3, %r5, %p3;
-; SM60-FTZ-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r7, %r6;
-; SM60-FTZ-NEXT: setp.ne.b32 %p4, %r1, %r7;
-; SM60-FTZ-NEXT: mov.b32 %r7, %r1;
-; SM60-FTZ-NEXT: @%p4 bra $L__BB65_1;
-; SM60-FTZ-NEXT: // %bb.2: // %atomicrmw.end
-; SM60-FTZ-NEXT: membar.cta;
-; SM60-FTZ-NEXT: st.param.b32 [func_retval0], %r1;
-; SM60-FTZ-NEXT: ret;
+; SM60-FTZ-ALLOW-LABEL: fmaximum_acq_rel_float_global_cta(
+; SM60-FTZ-ALLOW: {
+; SM60-FTZ-ALLOW-NEXT: .reg .pred %p<5>;
+; SM60-FTZ-ALLOW-NEXT: .reg .b32 %r<8>;
+; SM60-FTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
+; SM60-FTZ-ALLOW-EMPTY:
+; SM60-FTZ-ALLOW-NEXT: // %bb.0:
+; SM60-FTZ-ALLOW-NEXT: ld.param.b32 %r2, [fmaximum_acq_rel_float_global_cta_param_1];
+; SM60-FTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_float_global_cta_param_0];
+; SM60-FTZ-ALLOW-NEXT: membar.cta;
+; SM60-FTZ-ALLOW-NEXT: ld.volatile.global.b32 %r7, [%rd1];
+; SM60-FTZ-ALLOW-NEXT: $L__BB65_1: // %atomicrmw.start
+; SM60-FTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-FTZ-ALLOW-NEXT: setp.eq.b32 %p1, %r7, 0;
+; SM60-FTZ-ALLOW-NEXT: selp.f32 %r3, %r7, %r2, %p1;
+; SM60-FTZ-ALLOW-NEXT: setp.nan.ftz.f32 %p2, %r7, %r2;
+; SM60-FTZ-ALLOW-NEXT: max.ftz.f32 %r4, %r7, %r2;
+; SM60-FTZ-ALLOW-NEXT: selp.f32 %r5, 0f7FC00000, %r4, %p2;
+; SM60-FTZ-ALLOW-NEXT: setp.eq.ftz.f32 %p3, %r7, %r2;
+; SM60-FTZ-ALLOW-NEXT: selp.f32 %r6, %r3, %r5, %p3;
+; SM60-FTZ-ALLOW-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r7, %r6;
+; SM60-FTZ-ALLOW-NEXT: setp.ne.b32 %p4, %r1, %r7;
+; SM60-FTZ-ALLOW-NEXT: mov.b32 %r7, %r1;
+; SM60-FTZ-ALLOW-NEXT: @%p4 bra $L__BB65_1;
+; SM60-FTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-FTZ-ALLOW-NEXT: membar.cta;
+; SM60-FTZ-ALLOW-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-FTZ-ALLOW-NEXT: ret;
%retval = atomicrmw fmaximum ptr addrspace(1) %addr, float %val syncscope("block") acq_rel
ret float %retval
}
@@ -2209,1105 +2495,2161 @@ define double @fmaximum_acq_rel_double_global_cta(ptr addrspace(1) %addr, double
}
define half @fadd_acq_rel_half_global_cta(ptr addrspace(1) %addr, half %val) {
-; SM60-NOFTZ-LABEL: fadd_acq_rel_half_global_cta(
-; SM60-NOFTZ: {
-; SM60-NOFTZ-NEXT: .reg .pred %p<2>;
-; SM60-NOFTZ-NEXT: .reg .b16 %rs<4>;
-; SM60-NOFTZ-NEXT: .reg .b32 %r<15>;
-; SM60-NOFTZ-NEXT: .reg .b64 %rd<3>;
-; SM60-NOFTZ-EMPTY:
-; SM60-NOFTZ-NEXT: // %bb.0:
-; SM60-NOFTZ-NEXT: ld.param.b16 %rs1, [fadd_acq_rel_half_global_cta_param_1];
-; SM60-NOFTZ-NEXT: ld.param.b64 %rd2, [fadd_acq_rel_half_global_cta_param_0];
-; SM60-NOFTZ-NEXT: membar.cta;
-; SM60-NOFTZ-NEXT: and.b64 %rd1, %rd2, -4;
-; SM60-NOFTZ-NEXT: cvt.u32.u64 %r4, %rd2;
-; SM60-NOFTZ-NEXT: and.b32 %r5, %r4, 3;
-; SM60-NOFTZ-NEXT: shl.b32 %r1, %r5, 3;
-; SM60-NOFTZ-NEXT: mov.b32 %r6, 65535;
-; SM60-NOFTZ-NEXT: shl.b32 %r7, %r6, %r1;
-; SM60-NOFTZ-NEXT: not.b32 %r2, %r7;
-; SM60-NOFTZ-NEXT: ld.volatile.global.b32 %r14, [%rd1];
-; SM60-NOFTZ-NEXT: $L__BB72_1: // %atomicrmw.start
-; SM60-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NOFTZ-NEXT: shr.u32 %r8, %r14, %r1;
-; SM60-NOFTZ-NEXT: cvt.u16.u32 %rs2, %r8;
-; SM60-NOFTZ-NEXT: add.rn.f16 %rs3, %rs2, %rs1;
-; SM60-NOFTZ-NEXT: cvt.u32.u16 %r9, %rs3;
-; SM60-NOFTZ-NEXT: shl.b32 %r10, %r9, %r1;
-; SM60-NOFTZ-NEXT: and.b32 %r11, %r14, %r2;
-; SM60-NOFTZ-NEXT: or.b32 %r12, %r11, %r10;
-; SM60-NOFTZ-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
-; SM60-NOFTZ-NEXT: setp.ne.b32 %p1, %r3, %r14;
-; SM60-NOFTZ-NEXT: mov.b32 %r14, %r3;
-; SM60-NOFTZ-NEXT: @%p1 bra $L__BB72_1;
-; SM60-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end
-; SM60-NOFTZ-NEXT: shr.u32 %r13, %r3, %r1;
-; SM60-NOFTZ-NEXT: membar.cta;
-; SM60-NOFTZ-NEXT: st.param.b16 [func_retval0], %r13;
-; SM60-NOFTZ-NEXT: ret;
+; SM60-NOFTZ-DISALLOW-LABEL: fadd_acq_rel_half_global_cta(
+; SM60-NOFTZ-DISALLOW: {
+; SM60-NOFTZ-DISALLOW-NEXT: .reg .pred %p<2>;
+; SM60-NOFTZ-DISALLOW-NEXT: .reg .b16 %rs<4>;
+; SM60-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<15>;
+; SM60-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
+; SM60-NOFTZ-DISALLOW-EMPTY:
+; SM60-NOFTZ-DISALLOW-NEXT: // %bb.0:
+; SM60-NOFTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fadd_acq_rel_half_global_cta_param_1];
+; SM60-NOFTZ-DISALLOW-NEXT: ld.param.b64 %rd2, [fadd_acq_rel_half_global_cta_param_0];
+; SM60-NOFTZ-DISALLOW-NEXT: membar.cta;
+; SM60-NOFTZ-DISALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-NOFTZ-DISALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM60-NOFTZ-DISALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-NOFTZ-DISALLOW-NEXT: mov.b32 %r6, 65535;
+; SM60-NOFTZ-DISALLOW-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-NOFTZ-DISALLOW-NEXT: not.b32 %r2, %r7;
+; SM60-NOFTZ-DISALLOW-NEXT: ld.volatile.global.b32 %r14, [%rd1];
+; SM60-NOFTZ-DISALLOW-NEXT: $L__BB72_1: // %atomicrmw.start
+; SM60-NOFTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NOFTZ-DISALLOW-NEXT: shr.u32 %r8, %r14, %r1;
+; SM60-NOFTZ-DISALLOW-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM60-NOFTZ-DISALLOW-NEXT: add.rn.f16 %rs3, %rs2, %rs1;
+; SM60-NOFTZ-DISALLOW-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM60-NOFTZ-DISALLOW-NEXT: shl.b32 %r10, %r9, %r1;
+; SM60-NOFTZ-DISALLOW-NEXT: and.b32 %r11, %r14, %r2;
+; SM60-NOFTZ-DISALLOW-NEXT: or.b32 %r12, %r11, %r10;
+; SM60-NOFTZ-DISALLOW-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM60-NOFTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM60-NOFTZ-DISALLOW-NEXT: mov.b32 %r14, %r3;
+; SM60-NOFTZ-DISALLOW-NEXT: @%p1 bra $L__BB72_1;
+; SM60-NOFTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NOFTZ-DISALLOW-NEXT: shr.u32 %r13, %r3, %r1;
+; SM60-NOFTZ-DISALLOW-NEXT: membar.cta;
+; SM60-NOFTZ-DISALLOW-NEXT: st.param.b16 [func_retval0], %r13;
+; SM60-NOFTZ-DISALLOW-NEXT: ret;
;
-; SM60-FTZ-LABEL: fadd_acq_rel_half_global_cta(
-; SM60-FTZ: {
-; SM60-FTZ-NEXT: .reg .pred %p<2>;
-; SM60-FTZ-NEXT: .reg .b16 %rs<4>;
-; SM60-FTZ-NEXT: .reg .b32 %r<15>;
-; SM60-FTZ-NEXT: .reg .b64 %rd<3>;
-; SM60-FTZ-EMPTY:
-; SM60-FTZ-NEXT: // %bb.0:
-; SM60-FTZ-NEXT: ld.param.b16 %rs1, [fadd_acq_rel_half_global_cta_param_1];
-; SM60-FTZ-NEXT: ld.param.b64 %rd2, [fadd_acq_rel_half_global_cta_param_0];
-; SM60-FTZ-NEXT: membar.cta;
-; SM60-FTZ-NEXT: and.b64 %rd1, %rd2, -4;
-; SM60-FTZ-NEXT: cvt.u32.u64 %r4, %rd2;
-; SM60-FTZ-NEXT: and.b32 %r5, %r4, 3;
-; SM60-FTZ-NEXT: shl.b32 %r1, %r5, 3;
-; SM60-FTZ-NEXT: mov.b32 %r6, 65535;
-; SM60-FTZ-NEXT: shl.b32 %r7, %r6, %r1;
-; SM60-FTZ-NEXT: not.b32 %r2, %r7;
-; SM60-FTZ-NEXT: ld.volatile.global.b32 %r14, [%rd1];
-; SM60-FTZ-NEXT: $L__BB72_1: // %atomicrmw.start
-; SM60-FTZ-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-FTZ-NEXT: shr.u32 %r8, %r14, %r1;
-; SM60-FTZ-NEXT: cvt.u16.u32 %rs2, %r8;
-; SM60-FTZ-NEXT: add.rn.ftz.f16 %rs3, %rs2, %rs1;
-; SM60-FTZ-NEXT: cvt.u32.u16 %r9, %rs3;
-; SM60-FTZ-NEXT: shl.b32 %r10, %r9, %r1;
-; SM60-FTZ-NEXT: and.b32 %r11, %r14, %r2;
-; SM60-FTZ-NEXT: or.b32 %r12, %r11, %r10;
-; SM60-FTZ-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
-; SM60-FTZ-NEXT: setp.ne.b32 %p1, %r3, %r14;
-; SM60-FTZ-NEXT: mov.b32 %r14, %r3;
-; SM60-FTZ-NEXT: @%p1 bra $L__BB72_1;
-; SM60-FTZ-NEXT: // %bb.2: // %atomicrmw.end
-; SM60-FTZ-NEXT: shr.u32 %r13, %r3, %r1;
-; SM60-FTZ-NEXT: membar.cta;
-; SM60-FTZ-NEXT: st.param.b16 [func_retval0], %r13;
-; SM60-FTZ-NEXT: ret;
+; SM60-NOFTZ-ALLOW-LABEL: fadd_acq_rel_half_global_cta(
+; SM60-NOFTZ-ALLOW: {
+; SM60-NOFTZ-ALLOW-NEXT: .reg .pred %p<2>;
+; SM60-NOFTZ-ALLOW-NEXT: .reg .b16 %rs<4>;
+; SM60-NOFTZ-ALLOW-NEXT: .reg .b32 %r<15>;
+; SM60-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
+; SM60-NOFTZ-ALLOW-EMPTY:
+; SM60-NOFTZ-ALLOW-NEXT: // %bb.0:
+; SM60-NOFTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fadd_acq_rel_half_global_cta_param_1];
+; SM60-NOFTZ-ALLOW-NEXT: ld.param.b64 %rd2, [fadd_acq_rel_half_global_cta_param_0];
+; SM60-NOFTZ-ALLOW-NEXT: membar.cta;
+; SM60-NOFTZ-ALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NOFTZ-ALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-NOFTZ-ALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM60-NOFTZ-ALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-NOFTZ-ALLOW-NEXT: mov.b32 %r6, 65535;
+; SM60-NOFTZ-ALLOW-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-NOFTZ-ALLOW-NEXT: not.b32 %r2, %r7;
+; SM60-NOFTZ-ALLOW-NEXT: ld.volatile.global.b32 %r14, [%rd1];
+; SM60-NOFTZ-ALLOW-NEXT: $L__BB72_1: // %atomicrmw.start
+; SM60-NOFTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NOFTZ-ALLOW-NEXT: shr.u32 %r8, %r14, %r1;
+; SM60-NOFTZ-ALLOW-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM60-NOFTZ-ALLOW-NEXT: add.rn.f16 %rs3, %rs2, %rs1;
+; SM60-NOFTZ-ALLOW-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM60-NOFTZ-ALLOW-NEXT: shl.b32 %r10, %r9, %r1;
+; SM60-NOFTZ-ALLOW-NEXT: and.b32 %r11, %r14, %r2;
+; SM60-NOFTZ-ALLOW-NEXT: or.b32 %r12, %r11, %r10;
+; SM60-NOFTZ-ALLOW-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM60-NOFTZ-ALLOW-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM60-NOFTZ-ALLOW-NEXT: mov.b32 %r14, %r3;
+; SM60-NOFTZ-ALLOW-NEXT: @%p1 bra $L__BB72_1;
+; SM60-NOFTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NOFTZ-ALLOW-NEXT: shr.u32 %r13, %r3, %r1;
+; SM60-NOFTZ-ALLOW-NEXT: membar.cta;
+; SM60-NOFTZ-ALLOW-NEXT: st.param.b16 [func_retval0], %r13;
+; SM60-NOFTZ-ALLOW-NEXT: ret;
+;
+; SM60-FTZ-DISALLOW-LABEL: fadd_acq_rel_half_global_cta(
+; SM60-FTZ-DISALLOW: {
+; SM60-FTZ-DISALLOW-NEXT: .reg .pred %p<2>;
+; SM60-FTZ-DISALLOW-NEXT: .reg .b16 %rs<4>;
+; SM60-FTZ-DISALLOW-NEXT: .reg .b32 %r<15>;
+; SM60-FTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
+; SM60-FTZ-DISALLOW-EMPTY:
+; SM60-FTZ-DISALLOW-NEXT: // %bb.0:
+; SM60-FTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fadd_acq_rel_half_global_cta_param_1];
+; SM60-FTZ-DISALLOW-NEXT: ld.param.b64 %rd2, [fadd_acq_rel_half_global_cta_param_0];
+; SM60-FTZ-DISALLOW-NEXT: membar.cta;
+; SM60-FTZ-DISALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-FTZ-DISALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-FTZ-DISALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM60-FTZ-DISALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-FTZ-DISALLOW-NEXT: mov.b32 %r6, 65535;
+; SM60-FTZ-DISALLOW-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-FTZ-DISALLOW-NEXT: not.b32 %r2, %r7;
+; SM60-FTZ-DISALLOW-NEXT: ld.volatile.global.b32 %r14, [%rd1];
+; SM60-FTZ-DISALLOW-NEXT: $L__BB72_1: // %atomicrmw.start
+; SM60-FTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-FTZ-DISALLOW-NEXT: shr.u32 %r8, %r14, %r1;
+; SM60-FTZ-DISALLOW-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM60-FTZ-DISALLOW-NEXT: add.rn.ftz.f16 %rs3, %rs2, %rs1;
+; SM60-FTZ-DISALLOW-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM60-FTZ-DISALLOW-NEXT: shl.b32 %r10, %r9, %r1;
+; SM60-FTZ-DISALLOW-NEXT: and.b32 %r11, %r14, %r2;
+; SM60-FTZ-DISALLOW-NEXT: or.b32 %r12, %r11, %r10;
+; SM60-FTZ-DISALLOW-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM60-FTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM60-FTZ-DISALLOW-NEXT: mov.b32 %r14, %r3;
+; SM60-FTZ-DISALLOW-NEXT: @%p1 bra $L__BB72_1;
+; SM60-FTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-FTZ-DISALLOW-NEXT: shr.u32 %r13, %r3, %r1;
+; SM60-FTZ-DISALLOW-NEXT: membar.cta;
+; SM60-FTZ-DISALLOW-NEXT: st.param.b16 [func_retval0], %r13;
+; SM60-FTZ-DISALLOW-NEXT: ret;
+;
+; SM60-FTZ-ALLOW-LABEL: fadd_acq_rel_half_global_cta(
+; SM60-FTZ-ALLOW: {
+; SM60-FTZ-ALLOW-NEXT: .reg .pred %p<2>;
+; SM60-FTZ-ALLOW-NEXT: .reg .b16 %rs<4>;
+; SM60-FTZ-ALLOW-NEXT: .reg .b32 %r<15>;
+; SM60-FTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
+; SM60-FTZ-ALLOW-EMPTY:
+; SM60-FTZ-ALLOW-NEXT: // %bb.0:
+; SM60-FTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fadd_acq_rel_half_global_cta_param_1];
+; SM60-FTZ-ALLOW-NEXT: ld.param.b64 %rd2, [fadd_acq_rel_half_global_cta_param_0];
+; SM60-FTZ-ALLOW-NEXT: membar.cta;
+; SM60-FTZ-ALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-FTZ-ALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-FTZ-ALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM60-FTZ-ALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-FTZ-ALLOW-NEXT: mov.b32 %r6, 65535;
+; SM60-FTZ-ALLOW-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-FTZ-ALLOW-NEXT: not.b32 %r2, %r7;
+; SM60-FTZ-ALLOW-NEXT: ld.volatile.global.b32 %r14, [%rd1];
+; SM60-FTZ-ALLOW-NEXT: $L__BB72_1: // %atomicrmw.start
+; SM60-FTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-FTZ-ALLOW-NEXT: shr.u32 %r8, %r14, %r1;
+; SM60-FTZ-ALLOW-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM60-FTZ-ALLOW-NEXT: add.rn.ftz.f16 %rs3, %rs2, %rs1;
+; SM60-FTZ-ALLOW-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM60-FTZ-ALLOW-NEXT: shl.b32 %r10, %r9, %r1;
+; SM60-FTZ-ALLOW-NEXT: and.b32 %r11, %r14, %r2;
+; SM60-FTZ-ALLOW-NEXT: or.b32 %r12, %r11, %r10;
+; SM60-FTZ-ALLOW-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM60-FTZ-ALLOW-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM60-FTZ-ALLOW-NEXT: mov.b32 %r14, %r3;
+; SM60-FTZ-ALLOW-NEXT: @%p1 bra $L__BB72_1;
+; SM60-FTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-FTZ-ALLOW-NEXT: shr.u32 %r13, %r3, %r1;
+; SM60-FTZ-ALLOW-NEXT: membar.cta;
+; SM60-FTZ-ALLOW-NEXT: st.param.b16 [func_retval0], %r13;
+; SM60-FTZ-ALLOW-NEXT: ret;
%retval = atomicrmw fadd ptr addrspace(1) %addr, half %val syncscope("block") acq_rel
ret half %retval
}
define half @fsub_acq_rel_half_global_cta(ptr addrspace(1) %addr, half %val) {
-; SM60-NOFTZ-LABEL: fsub_acq_rel_half_global_cta(
-; SM60-NOFTZ: {
-; SM60-NOFTZ-NEXT: .reg .pred %p<2>;
-; SM60-NOFTZ-NEXT: .reg .b16 %rs<4>;
-; SM60-NOFTZ-NEXT: .reg .b32 %r<15>;
-; SM60-NOFTZ-NEXT: .reg .b64 %rd<3>;
-; SM60-NOFTZ-EMPTY:
-; SM60-NOFTZ-NEXT: // %bb.0:
-; SM60-NOFTZ-NEXT: ld.param.b16 %rs1, [fsub_acq_rel_half_global_cta_param_1];
-; SM60-NOFTZ-NEXT: ld.param.b64 %rd2, [fsub_acq_rel_half_global_cta_param_0];
-; SM60-NOFTZ-NEXT: membar.cta;
-; SM60-NOFTZ-NEXT: and.b64 %rd1, %rd2, -4;
-; SM60-NOFTZ-NEXT: cvt.u32.u64 %r4, %rd2;
-; SM60-NOFTZ-NEXT: and.b32 %r5, %r4, 3;
-; SM60-NOFTZ-NEXT: shl.b32 %r1, %r5, 3;
-; SM60-NOFTZ-NEXT: mov.b32 %r6, 65535;
-; SM60-NOFTZ-NEXT: shl.b32 %r7, %r6, %r1;
-; SM60-NOFTZ-NEXT: not.b32 %r2, %r7;
-; SM60-NOFTZ-NEXT: ld.volatile.global.b32 %r14, [%rd1];
-; SM60-NOFTZ-NEXT: $L__BB73_1: // %atomicrmw.start
-; SM60-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NOFTZ-NEXT: shr.u32 %r8, %r14, %r1;
-; SM60-NOFTZ-NEXT: cvt.u16.u32 %rs2, %r8;
-; SM60-NOFTZ-NEXT: sub.rn.f16 %rs3, %rs2, %rs1;
-; SM60-NOFTZ-NEXT: cvt.u32.u16 %r9, %rs3;
-; SM60-NOFTZ-NEXT: shl.b32 %r10, %r9, %r1;
-; SM60-NOFTZ-NEXT: and.b32 %r11, %r14, %r2;
-; SM60-NOFTZ-NEXT: or.b32 %r12, %r11, %r10;
-; SM60-NOFTZ-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
-; SM60-NOFTZ-NEXT: setp.ne.b32 %p1, %r3, %r14;
-; SM60-NOFTZ-NEXT: mov.b32 %r14, %r3;
-; SM60-NOFTZ-NEXT: @%p1 bra $L__BB73_1;
-; SM60-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end
-; SM60-NOFTZ-NEXT: shr.u32 %r13, %r3, %r1;
-; SM60-NOFTZ-NEXT: membar.cta;
-; SM60-NOFTZ-NEXT: st.param.b16 [func_retval0], %r13;
-; SM60-NOFTZ-NEXT: ret;
+; SM60-NOFTZ-DISALLOW-LABEL: fsub_acq_rel_half_global_cta(
+; SM60-NOFTZ-DISALLOW: {
+; SM60-NOFTZ-DISALLOW-NEXT: .reg .pred %p<2>;
+; SM60-NOFTZ-DISALLOW-NEXT: .reg .b16 %rs<4>;
+; SM60-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<15>;
+; SM60-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
+; SM60-NOFTZ-DISALLOW-EMPTY:
+; SM60-NOFTZ-DISALLOW-NEXT: // %bb.0:
+; SM60-NOFTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fsub_acq_rel_half_global_cta_param_1];
+; SM60-NOFTZ-DISALLOW-NEXT: ld.param.b64 %rd2, [fsub_acq_rel_half_global_cta_param_0];
+; SM60-NOFTZ-DISALLOW-NEXT: membar.cta;
+; SM60-NOFTZ-DISALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-NOFTZ-DISALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM60-NOFTZ-DISALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-NOFTZ-DISALLOW-NEXT: mov.b32 %r6, 65535;
+; SM60-NOFTZ-DISALLOW-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-NOFTZ-DISALLOW-NEXT: not.b32 %r2, %r7;
+; SM60-NOFTZ-DISALLOW-NEXT: ld.volatile.global.b32 %r14, [%rd1];
+; SM60-NOFTZ-DISALLOW-NEXT: $L__BB73_1: // %atomicrmw.start
+; SM60-NOFTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NOFTZ-DISALLOW-NEXT: shr.u32 %r8, %r14, %r1;
+; SM60-NOFTZ-DISALLOW-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM60-NOFTZ-DISALLOW-NEXT: sub.rn.f16 %rs3, %rs2, %rs1;
+; SM60-NOFTZ-DISALLOW-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM60-NOFTZ-DISALLOW-NEXT: shl.b32 %r10, %r9, %r1;
+; SM60-NOFTZ-DISALLOW-NEXT: and.b32 %r11, %r14, %r2;
+; SM60-NOFTZ-DISALLOW-NEXT: or.b32 %r12, %r11, %r10;
+; SM60-NOFTZ-DISALLOW-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM60-NOFTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM60-NOFTZ-DISALLOW-NEXT: mov.b32 %r14, %r3;
+; SM60-NOFTZ-DISALLOW-NEXT: @%p1 bra $L__BB73_1;
+; SM60-NOFTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NOFTZ-DISALLOW-NEXT: shr.u32 %r13, %r3, %r1;
+; SM60-NOFTZ-DISALLOW-NEXT: membar.cta;
+; SM60-NOFTZ-DISALLOW-NEXT: st.param.b16 [func_retval0], %r13;
+; SM60-NOFTZ-DISALLOW-NEXT: ret;
+;
+; SM60-NOFTZ-ALLOW-LABEL: fsub_acq_rel_half_global_cta(
+; SM60-NOFTZ-ALLOW: {
+; SM60-NOFTZ-ALLOW-NEXT: .reg .pred %p<2>;
+; SM60-NOFTZ-ALLOW-NEXT: .reg .b16 %rs<4>;
+; SM60-NOFTZ-ALLOW-NEXT: .reg .b32 %r<15>;
+; SM60-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
+; SM60-NOFTZ-ALLOW-EMPTY:
+; SM60-NOFTZ-ALLOW-NEXT: // %bb.0:
+; SM60-NOFTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fsub_acq_rel_half_global_cta_param_1];
+; SM60-NOFTZ-ALLOW-NEXT: ld.param.b64 %rd2, [fsub_acq_rel_half_global_cta_param_0];
+; SM60-NOFTZ-ALLOW-NEXT: membar.cta;
+; SM60-NOFTZ-ALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NOFTZ-ALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-NOFTZ-ALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM60-NOFTZ-ALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-NOFTZ-ALLOW-NEXT: mov.b32 %r6, 65535;
+; SM60-NOFTZ-ALLOW-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-NOFTZ-ALLOW-NEXT: not.b32 %r2, %r7;
+; SM60-NOFTZ-ALLOW-NEXT: ld.volatile.global.b32 %r14, [%rd1];
+; SM60-NOFTZ-ALLOW-NEXT: $L__BB73_1: // %atomicrmw.start
+; SM60-NOFTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NOFTZ-ALLOW-NEXT: shr.u32 %r8, %r14, %r1;
+; SM60-NOFTZ-ALLOW-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM60-NOFTZ-ALLOW-NEXT: sub.rn.f16 %rs3, %rs2, %rs1;
+; SM60-NOFTZ-ALLOW-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM60-NOFTZ-ALLOW-NEXT: shl.b32 %r10, %r9, %r1;
+; SM60-NOFTZ-ALLOW-NEXT: and.b32 %r11, %r14, %r2;
+; SM60-NOFTZ-ALLOW-NEXT: or.b32 %r12, %r11, %r10;
+; SM60-NOFTZ-ALLOW-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM60-NOFTZ-ALLOW-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM60-NOFTZ-ALLOW-NEXT: mov.b32 %r14, %r3;
+; SM60-NOFTZ-ALLOW-NEXT: @%p1 bra $L__BB73_1;
+; SM60-NOFTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NOFTZ-ALLOW-NEXT: shr.u32 %r13, %r3, %r1;
+; SM60-NOFTZ-ALLOW-NEXT: membar.cta;
+; SM60-NOFTZ-ALLOW-NEXT: st.param.b16 [func_retval0], %r13;
+; SM60-NOFTZ-ALLOW-NEXT: ret;
;
-; SM60-FTZ-LABEL: fsub_acq_rel_half_global_cta(
-; SM60-FTZ: {
-; SM60-FTZ-NEXT: .reg .pred %p<2>;
-; SM60-FTZ-NEXT: .reg .b16 %rs<4>;
-; SM60-FTZ-NEXT: .reg .b32 %r<15>;
-; SM60-FTZ-NEXT: .reg .b64 %rd<3>;
-; SM60-FTZ-EMPTY:
-; SM60-FTZ-NEXT: // %bb.0:
-; SM60-FTZ-NEXT: ld.param.b16 %rs1, [fsub_acq_rel_half_global_cta_param_1];
-; SM60-FTZ-NEXT: ld.param.b64 %rd2, [fsub_acq_rel_half_global_cta_param_0];
-; SM60-FTZ-NEXT: membar.cta;
-; SM60-FTZ-NEXT: and.b64 %rd1, %rd2, -4;
-; SM60-FTZ-NEXT: cvt.u32.u64 %r4, %rd2;
-; SM60-FTZ-NEXT: and.b32 %r5, %r4, 3;
-; SM60-FTZ-NEXT: shl.b32 %r1, %r5, 3;
-; SM60-FTZ-NEXT: mov.b32 %r6, 65535;
-; SM60-FTZ-NEXT: shl.b32 %r7, %r6, %r1;
-; SM60-FTZ-NEXT: not.b32 %r2, %r7;
-; SM60-FTZ-NEXT: ld.volatile.global.b32 %r14, [%rd1];
-; SM60-FTZ-NEXT: $L__BB73_1: // %atomicrmw.start
-; SM60-FTZ-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-FTZ-NEXT: shr.u32 %r8, %r14, %r1;
-; SM60-FTZ-NEXT: cvt.u16.u32 %rs2, %r8;
-; SM60-FTZ-NEXT: sub.rn.ftz.f16 %rs3, %rs2, %rs1;
-; SM60-FTZ-NEXT: cvt.u32.u16 %r9, %rs3;
-; SM60-FTZ-NEXT: shl.b32 %r10, %r9, %r1;
-; SM60-FTZ-NEXT: and.b32 %r11, %r14, %r2;
-; SM60-FTZ-NEXT: or.b32 %r12, %r11, %r10;
-; SM60-FTZ-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
-; SM60-FTZ-NEXT: setp.ne.b32 %p1, %r3, %r14;
-; SM60-FTZ-NEXT: mov.b32 %r14, %r3;
-; SM60-FTZ-NEXT: @%p1 bra $L__BB73_1;
-; SM60-FTZ-NEXT: // %bb.2: // %atomicrmw.end
-; SM60-FTZ-NEXT: shr.u32 %r13, %r3, %r1;
-; SM60-FTZ-NEXT: membar.cta;
-; SM60-FTZ-NEXT: st.param.b16 [func_retval0], %r13;
-; SM60-FTZ-NEXT: ret;
+; SM60-FTZ-DISALLOW-LABEL: fsub_acq_rel_half_global_cta(
+; SM60-FTZ-DISALLOW: {
+; SM60-FTZ-DISALLOW-NEXT: .reg .pred %p<2>;
+; SM60-FTZ-DISALLOW-NEXT: .reg .b16 %rs<4>;
+; SM60-FTZ-DISALLOW-NEXT: .reg .b32 %r<15>;
+; SM60-FTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
+; SM60-FTZ-DISALLOW-EMPTY:
+; SM60-FTZ-DISALLOW-NEXT: // %bb.0:
+; SM60-FTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fsub_acq_rel_half_global_cta_param_1];
+; SM60-FTZ-DISALLOW-NEXT: ld.param.b64 %rd2, [fsub_acq_rel_half_global_cta_param_0];
+; SM60-FTZ-DISALLOW-NEXT: membar.cta;
+; SM60-FTZ-DISALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-FTZ-DISALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-FTZ-DISALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM60-FTZ-DISALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-FTZ-DISALLOW-NEXT: mov.b32 %r6, 65535;
+; SM60-FTZ-DISALLOW-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-FTZ-DISALLOW-NEXT: not.b32 %r2, %r7;
+; SM60-FTZ-DISALLOW-NEXT: ld.volatile.global.b32 %r14, [%rd1];
+; SM60-FTZ-DISALLOW-NEXT: $L__BB73_1: // %atomicrmw.start
+; SM60-FTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-FTZ-DISALLOW-NEXT: shr.u32 %r8, %r14, %r1;
+; SM60-FTZ-DISALLOW-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM60-FTZ-DISALLOW-NEXT: sub.rn.ftz.f16 %rs3, %rs2, %rs1;
+; SM60-FTZ-DISALLOW-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM60-FTZ-DISALLOW-NEXT: shl.b32 %r10, %r9, %r1;
+; SM60-FTZ-DISALLOW-NEXT: and.b32 %r11, %r14, %r2;
+; SM60-FTZ-DISALLOW-NEXT: or.b32 %r12, %r11, %r10;
+; SM60-FTZ-DISALLOW-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM60-FTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM60-FTZ-DISALLOW-NEXT: mov.b32 %r14, %r3;
+; SM60-FTZ-DISALLOW-NEXT: @%p1 bra $L__BB73_1;
+; SM60-FTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-FTZ-DISALLOW-NEXT: shr.u32 %r13, %r3, %r1;
+; SM60-FTZ-DISALLOW-NEXT: membar.cta;
+; SM60-FTZ-DISALLOW-NEXT: st.param.b16 [func_retval0], %r13;
+; SM60-FTZ-DISALLOW-NEXT: ret;
+;
+; SM60-FTZ-ALLOW-LABEL: fsub_acq_rel_half_global_cta(
+; SM60-FTZ-ALLOW: {
+; SM60-FTZ-ALLOW-NEXT: .reg .pred %p<2>;
+; SM60-FTZ-ALLOW-NEXT: .reg .b16 %rs<4>;
+; SM60-FTZ-ALLOW-NEXT: .reg .b32 %r<15>;
+; SM60-FTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
+; SM60-FTZ-ALLOW-EMPTY:
+; SM60-FTZ-ALLOW-NEXT: // %bb.0:
+; SM60-FTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fsub_acq_rel_half_global_cta_param_1];
+; SM60-FTZ-ALLOW-NEXT: ld.param.b64 %rd2, [fsub_acq_rel_half_global_cta_param_0];
+; SM60-FTZ-ALLOW-NEXT: membar.cta;
+; SM60-FTZ-ALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-FTZ-ALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-FTZ-ALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM60-FTZ-ALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-FTZ-ALLOW-NEXT: mov.b32 %r6, 65535;
+; SM60-FTZ-ALLOW-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-FTZ-ALLOW-NEXT: not.b32 %r2, %r7;
+; SM60-FTZ-ALLOW-NEXT: ld.volatile.global.b32 %r14, [%rd1];
+; SM60-FTZ-ALLOW-NEXT: $L__BB73_1: // %atomicrmw.start
+; SM60-FTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-FTZ-ALLOW-NEXT: shr.u32 %r8, %r14, %r1;
+; SM60-FTZ-ALLOW-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM60-FTZ-ALLOW-NEXT: sub.rn.ftz.f16 %rs3, %rs2, %rs1;
+; SM60-FTZ-ALLOW-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM60-FTZ-ALLOW-NEXT: shl.b32 %r10, %r9, %r1;
+; SM60-FTZ-ALLOW-NEXT: and.b32 %r11, %r14, %r2;
+; SM60-FTZ-ALLOW-NEXT: or.b32 %r12, %r11, %r10;
+; SM60-FTZ-ALLOW-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM60-FTZ-ALLOW-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM60-FTZ-ALLOW-NEXT: mov.b32 %r14, %r3;
+; SM60-FTZ-ALLOW-NEXT: @%p1 bra $L__BB73_1;
+; SM60-FTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-FTZ-ALLOW-NEXT: shr.u32 %r13, %r3, %r1;
+; SM60-FTZ-ALLOW-NEXT: membar.cta;
+; SM60-FTZ-ALLOW-NEXT: st.param.b16 [func_retval0], %r13;
+; SM60-FTZ-ALLOW-NEXT: ret;
%retval = atomicrmw fsub ptr addrspace(1) %addr, half %val syncscope("block") acq_rel
ret half %retval
}
define half @fmin_acq_rel_half_global_cta(ptr addrspace(1) %addr, half %val) {
-; SM60-NOFTZ-LABEL: fmin_acq_rel_half_global_cta(
-; SM60-NOFTZ: {
-; SM60-NOFTZ-NEXT: .reg .pred %p<2>;
-; SM60-NOFTZ-NEXT: .reg .b16 %rs<4>;
-; SM60-NOFTZ-NEXT: .reg .b32 %r<18>;
-; SM60-NOFTZ-NEXT: .reg .b64 %rd<3>;
-; SM60-NOFTZ-EMPTY:
-; SM60-NOFTZ-NEXT: // %bb.0:
-; SM60-NOFTZ-NEXT: ld.param.b16 %rs1, [fmin_acq_rel_half_global_cta_param_1];
-; SM60-NOFTZ-NEXT: ld.param.b64 %rd2, [fmin_acq_rel_half_global_cta_param_0];
-; SM60-NOFTZ-NEXT: membar.cta;
-; SM60-NOFTZ-NEXT: and.b64 %rd1, %rd2, -4;
-; SM60-NOFTZ-NEXT: cvt.u32.u64 %r4, %rd2;
-; SM60-NOFTZ-NEXT: and.b32 %r5, %r4, 3;
-; SM60-NOFTZ-NEXT: shl.b32 %r1, %r5, 3;
-; SM60-NOFTZ-NEXT: mov.b32 %r6, 65535;
-; SM60-NOFTZ-NEXT: shl.b32 %r7, %r6, %r1;
-; SM60-NOFTZ-NEXT: not.b32 %r2, %r7;
-; SM60-NOFTZ-NEXT: ld.volatile.global.b32 %r17, [%rd1];
-; SM60-NOFTZ-NEXT: cvt.f32.f16 %r10, %rs1;
-; SM60-NOFTZ-NEXT: $L__BB74_1: // %atomicrmw.start
-; SM60-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NOFTZ-NEXT: shr.u32 %r8, %r17, %r1;
-; SM60-NOFTZ-NEXT: cvt.u16.u32 %rs2, %r8;
-; SM60-NOFTZ-NEXT: cvt.f32.f16 %r9, %rs2;
-; SM60-NOFTZ-NEXT: min.f32 %r11, %r9, %r10;
-; SM60-NOFTZ-NEXT: cvt.rn.f16.f32 %rs3, %r11;
-; SM60-NOFTZ-NEXT: cvt.u32.u16 %r12, %rs3;
-; SM60-NOFTZ-NEXT: shl.b32 %r13, %r12, %r1;
-; SM60-NOFTZ-NEXT: and.b32 %r14, %r17, %r2;
-; SM60-NOFTZ-NEXT: or.b32 %r15, %r14, %r13;
-; SM60-NOFTZ-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r17, %r15;
-; SM60-NOFTZ-NEXT: setp.ne.b32 %p1, %r3, %r17;
-; SM60-NOFTZ-NEXT: mov.b32 %r17, %r3;
-; SM60-NOFTZ-NEXT: @%p1 bra $L__BB74_1;
-; SM60-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end
-; SM60-NOFTZ-NEXT: shr.u32 %r16, %r3, %r1;
-; SM60-NOFTZ-NEXT: membar.cta;
-; SM60-NOFTZ-NEXT: st.param.b16 [func_retval0], %r16;
-; SM60-NOFTZ-NEXT: ret;
+; SM60-NOFTZ-DISALLOW-LABEL: fmin_acq_rel_half_global_cta(
+; SM60-NOFTZ-DISALLOW: {
+; SM60-NOFTZ-DISALLOW-NEXT: .reg .pred %p<2>;
+; SM60-NOFTZ-DISALLOW-NEXT: .reg .b16 %rs<4>;
+; SM60-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<18>;
+; SM60-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
+; SM60-NOFTZ-DISALLOW-EMPTY:
+; SM60-NOFTZ-DISALLOW-NEXT: // %bb.0:
+; SM60-NOFTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fmin_acq_rel_half_global_cta_param_1];
+; SM60-NOFTZ-DISALLOW-NEXT: ld.param.b64 %rd2, [fmin_acq_rel_half_global_cta_param_0];
+; SM60-NOFTZ-DISALLOW-NEXT: membar.cta;
+; SM60-NOFTZ-DISALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-NOFTZ-DISALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM60-NOFTZ-DISALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-NOFTZ-DISALLOW-NEXT: mov.b32 %r6, 65535;
+; SM60-NOFTZ-DISALLOW-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-NOFTZ-DISALLOW-NEXT: not.b32 %r2, %r7;
+; SM60-NOFTZ-DISALLOW-NEXT: ld.volatile.global.b32 %r17, [%rd1];
+; SM60-NOFTZ-DISALLOW-NEXT: cvt.f32.f16 %r10, %rs1;
+; SM60-NOFTZ-DISALLOW-NEXT: $L__BB74_1: // %atomicrmw.start
+; SM60-NOFTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NOFTZ-DISALLOW-NEXT: shr.u32 %r8, %r17, %r1;
+; SM60-NOFTZ-DISALLOW-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM60-NOFTZ-DISALLOW-NEXT: cvt.f32.f16 %r9, %rs2;
+; SM60-NOFTZ-DISALLOW-NEXT: min.f32 %r11, %r9, %r10;
+; SM60-NOFTZ-DISALLOW-NEXT: cvt.rn.f16.f32 %rs3, %r11;
+; SM60-NOFTZ-DISALLOW-NEXT: cvt.u32.u16 %r12, %rs3;
+; SM60-NOFTZ-DISALLOW-NEXT: shl.b32 %r13, %r12, %r1;
+; SM60-NOFTZ-DISALLOW-NEXT: and.b32 %r14, %r17, %r2;
+; SM60-NOFTZ-DISALLOW-NEXT: or.b32 %r15, %r14, %r13;
+; SM60-NOFTZ-DISALLOW-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r17, %r15;
+; SM60-NOFTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r3, %r17;
+; SM60-NOFTZ-DISALLOW-NEXT: mov.b32 %r17, %r3;
+; SM60-NOFTZ-DISALLOW-NEXT: @%p1 bra $L__BB74_1;
+; SM60-NOFTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NOFTZ-DISALLOW-NEXT: shr.u32 %r16, %r3, %r1;
+; SM60-NOFTZ-DISALLOW-NEXT: membar.cta;
+; SM60-NOFTZ-DISALLOW-NEXT: st.param.b16 [func_retval0], %r16;
+; SM60-NOFTZ-DISALLOW-NEXT: ret;
+;
+; SM60-NOFTZ-ALLOW-LABEL: fmin_acq_rel_half_global_cta(
+; SM60-NOFTZ-ALLOW: {
+; SM60-NOFTZ-ALLOW-NEXT: .reg .pred %p<2>;
+; SM60-NOFTZ-ALLOW-NEXT: .reg .b16 %rs<4>;
+; SM60-NOFTZ-ALLOW-NEXT: .reg .b32 %r<18>;
+; SM60-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
+; SM60-NOFTZ-ALLOW-EMPTY:
+; SM60-NOFTZ-ALLOW-NEXT: // %bb.0:
+; SM60-NOFTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fmin_acq_rel_half_global_cta_param_1];
+; SM60-NOFTZ-ALLOW-NEXT: ld.param.b64 %rd2, [fmin_acq_rel_half_global_cta_param_0];
+; SM60-NOFTZ-ALLOW-NEXT: membar.cta;
+; SM60-NOFTZ-ALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NOFTZ-ALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-NOFTZ-ALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM60-NOFTZ-ALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-NOFTZ-ALLOW-NEXT: mov.b32 %r6, 65535;
+; SM60-NOFTZ-ALLOW-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-NOFTZ-ALLOW-NEXT: not.b32 %r2, %r7;
+; SM60-NOFTZ-ALLOW-NEXT: ld.volatile.global.b32 %r17, [%rd1];
+; SM60-NOFTZ-ALLOW-NEXT: cvt.f32.f16 %r10, %rs1;
+; SM60-NOFTZ-ALLOW-NEXT: $L__BB74_1: // %atomicrmw.start
+; SM60-NOFTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NOFTZ-ALLOW-NEXT: shr.u32 %r8, %r17, %r1;
+; SM60-NOFTZ-ALLOW-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM60-NOFTZ-ALLOW-NEXT: cvt.f32.f16 %r9, %rs2;
+; SM60-NOFTZ-ALLOW-NEXT: min.f32 %r11, %r9, %r10;
+; SM60-NOFTZ-ALLOW-NEXT: cvt.rn.f16.f32 %rs3, %r11;
+; SM60-NOFTZ-ALLOW-NEXT: cvt.u32.u16 %r12, %rs3;
+; SM60-NOFTZ-ALLOW-NEXT: shl.b32 %r13, %r12, %r1;
+; SM60-NOFTZ-ALLOW-NEXT: and.b32 %r14, %r17, %r2;
+; SM60-NOFTZ-ALLOW-NEXT: or.b32 %r15, %r14, %r13;
+; SM60-NOFTZ-ALLOW-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r17, %r15;
+; SM60-NOFTZ-ALLOW-NEXT: setp.ne.b32 %p1, %r3, %r17;
+; SM60-NOFTZ-ALLOW-NEXT: mov.b32 %r17, %r3;
+; SM60-NOFTZ-ALLOW-NEXT: @%p1 bra $L__BB74_1;
+; SM60-NOFTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NOFTZ-ALLOW-NEXT: shr.u32 %r16, %r3, %r1;
+; SM60-NOFTZ-ALLOW-NEXT: membar.cta;
+; SM60-NOFTZ-ALLOW-NEXT: st.param.b16 [func_retval0], %r16;
+; SM60-NOFTZ-ALLOW-NEXT: ret;
+;
+; SM60-FTZ-DISALLOW-LABEL: fmin_acq_rel_half_global_cta(
+; SM60-FTZ-DISALLOW: {
+; SM60-FTZ-DISALLOW-NEXT: .reg .pred %p<2>;
+; SM60-FTZ-DISALLOW-NEXT: .reg .b16 %rs<4>;
+; SM60-FTZ-DISALLOW-NEXT: .reg .b32 %r<18>;
+; SM60-FTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
+; SM60-FTZ-DISALLOW-EMPTY:
+; SM60-FTZ-DISALLOW-NEXT: // %bb.0:
+; SM60-FTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fmin_acq_rel_half_global_cta_param_1];
+; SM60-FTZ-DISALLOW-NEXT: ld.param.b64 %rd2, [fmin_acq_rel_half_global_cta_param_0];
+; SM60-FTZ-DISALLOW-NEXT: membar.cta;
+; SM60-FTZ-DISALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-FTZ-DISALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-FTZ-DISALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM60-FTZ-DISALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-FTZ-DISALLOW-NEXT: mov.b32 %r6, 65535;
+; SM60-FTZ-DISALLOW-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-FTZ-DISALLOW-NEXT: not.b32 %r2, %r7;
+; SM60-FTZ-DISALLOW-NEXT: ld.volatile.global.b32 %r17, [%rd1];
+; SM60-FTZ-DISALLOW-NEXT: cvt.ftz.f32.f16 %r10, %rs1;
+; SM60-FTZ-DISALLOW-NEXT: $L__BB74_1: // %atomicrmw.start
+; SM60-FTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-FTZ-DISALLOW-NEXT: shr.u32 %r8, %r17, %r1;
+; SM60-FTZ-DISALLOW-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM60-FTZ-DISALLOW-NEXT: cvt.ftz.f32.f16 %r9, %rs2;
+; SM60-FTZ-DISALLOW-NEXT: min.ftz.f32 %r11, %r9, %r10;
+; SM60-FTZ-DISALLOW-NEXT: cvt.rn.f16.f32 %rs3, %r11;
+; SM60-FTZ-DISALLOW-NEXT: cvt.u32.u16 %r12, %rs3;
+; SM60-FTZ-DISALLOW-NEXT: shl.b32 %r13, %r12, %r1;
+; SM60-FTZ-DISALLOW-NEXT: and.b32 %r14, %r17, %r2;
+; SM60-FTZ-DISALLOW-NEXT: or.b32 %r15, %r14, %r13;
+; SM60-FTZ-DISALLOW-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r17, %r15;
+; SM60-FTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r3, %r17;
+; SM60-FTZ-DISALLOW-NEXT: mov.b32 %r17, %r3;
+; SM60-FTZ-DISALLOW-NEXT: @%p1 bra $L__BB74_1;
+; SM60-FTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-FTZ-DISALLOW-NEXT: shr.u32 %r16, %r3, %r1;
+; SM60-FTZ-DISALLOW-NEXT: membar.cta;
+; SM60-FTZ-DISALLOW-NEXT: st.param.b16 [func_retval0], %r16;
+; SM60-FTZ-DISALLOW-NEXT: ret;
;
-; SM60-FTZ-LABEL: fmin_acq_rel_half_global_cta(
-; SM60-FTZ: {
-; SM60-FTZ-NEXT: .reg .pred %p<2>;
-; SM60-FTZ-NEXT: .reg .b16 %rs<4>;
-; SM60-FTZ-NEXT: .reg .b32 %r<18>;
-; SM60-FTZ-NEXT: .reg .b64 %rd<3>;
-; SM60-FTZ-EMPTY:
-; SM60-FTZ-NEXT: // %bb.0:
-; SM60-FTZ-NEXT: ld.param.b16 %rs1, [fmin_acq_rel_half_global_cta_param_1];
-; SM60-FTZ-NEXT: ld.param.b64 %rd2, [fmin_acq_rel_half_global_cta_param_0];
-; SM60-FTZ-NEXT: membar.cta;
-; SM60-FTZ-NEXT: and.b64 %rd1, %rd2, -4;
-; SM60-FTZ-NEXT: cvt.u32.u64 %r4, %rd2;
-; SM60-FTZ-NEXT: and.b32 %r5, %r4, 3;
-; SM60-FTZ-NEXT: shl.b32 %r1, %r5, 3;
-; SM60-FTZ-NEXT: mov.b32 %r6, 65535;
-; SM60-FTZ-NEXT: shl.b32 %r7, %r6, %r1;
-; SM60-FTZ-NEXT: not.b32 %r2, %r7;
-; SM60-FTZ-NEXT: ld.volatile.global.b32 %r17, [%rd1];
-; SM60-FTZ-NEXT: cvt.ftz.f32.f16 %r10, %rs1;
-; SM60-FTZ-NEXT: $L__BB74_1: // %atomicrmw.start
-; SM60-FTZ-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-FTZ-NEXT: shr.u32 %r8, %r17, %r1;
-; SM60-FTZ-NEXT: cvt.u16.u32 %rs2, %r8;
-; SM60-FTZ-NEXT: cvt.ftz.f32.f16 %r9, %rs2;
-; SM60-FTZ-NEXT: min.ftz.f32 %r11, %r9, %r10;
-; SM60-FTZ-NEXT: cvt.rn.f16.f32 %rs3, %r11;
-; SM60-FTZ-NEXT: cvt.u32.u16 %r12, %rs3;
-; SM60-FTZ-NEXT: shl.b32 %r13, %r12, %r1;
-; SM60-FTZ-NEXT: and.b32 %r14, %r17, %r2;
-; SM60-FTZ-NEXT: or.b32 %r15, %r14, %r13;
-; SM60-FTZ-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r17, %r15;
-; SM60-FTZ-NEXT: setp.ne.b32 %p1, %r3, %r17;
-; SM60-FTZ-NEXT: mov.b32 %r17, %r3;
-; SM60-FTZ-NEXT: @%p1 bra $L__BB74_1;
-; SM60-FTZ-NEXT: // %bb.2: // %atomicrmw.end
-; SM60-FTZ-NEXT: shr.u32 %r16, %r3, %r1;
-; SM60-FTZ-NEXT: membar.cta;
-; SM60-FTZ-NEXT: st.param.b16 [func_retval0], %r16;
-; SM60-FTZ-NEXT: ret;
+; SM60-FTZ-ALLOW-LABEL: fmin_acq_rel_half_global_cta(
+; SM60-FTZ-ALLOW: {
+; SM60-FTZ-ALLOW-NEXT: .reg .pred %p<2>;
+; SM60-FTZ-ALLOW-NEXT: .reg .b16 %rs<4>;
+; SM60-FTZ-ALLOW-NEXT: .reg .b32 %r<18>;
+; SM60-FTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
+; SM60-FTZ-ALLOW-EMPTY:
+; SM60-FTZ-ALLOW-NEXT: // %bb.0:
+; SM60-FTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fmin_acq_rel_half_global_cta_param_1];
+; SM60-FTZ-ALLOW-NEXT: ld.param.b64 %rd2, [fmin_acq_rel_half_global_cta_param_0];
+; SM60-FTZ-ALLOW-NEXT: membar.cta;
+; SM60-FTZ-ALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-FTZ-ALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-FTZ-ALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM60-FTZ-ALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-FTZ-ALLOW-NEXT: mov.b32 %r6, 65535;
+; SM60-FTZ-ALLOW-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-FTZ-ALLOW-NEXT: not.b32 %r2, %r7;
+; SM60-FTZ-ALLOW-NEXT: ld.volatile.global.b32 %r17, [%rd1];
+; SM60-FTZ-ALLOW-NEXT: cvt.ftz.f32.f16 %r10, %rs1;
+; SM60-FTZ-ALLOW-NEXT: $L__BB74_1: // %atomicrmw.start
+; SM60-FTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-FTZ-ALLOW-NEXT: shr.u32 %r8, %r17, %r1;
+; SM60-FTZ-ALLOW-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM60-FTZ-ALLOW-NEXT: cvt.ftz.f32.f16 %r9, %rs2;
+; SM60-FTZ-ALLOW-NEXT: min.ftz.f32 %r11, %r9, %r10;
+; SM60-FTZ-ALLOW-NEXT: cvt.rn.f16.f32 %rs3, %r11;
+; SM60-FTZ-ALLOW-NEXT: cvt.u32.u16 %r12, %rs3;
+; SM60-FTZ-ALLOW-NEXT: shl.b32 %r13, %r12, %r1;
+; SM60-FTZ-ALLOW-NEXT: and.b32 %r14, %r17, %r2;
+; SM60-FTZ-ALLOW-NEXT: or.b32 %r15, %r14, %r13;
+; SM60-FTZ-ALLOW-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r17, %r15;
+; SM60-FTZ-ALLOW-NEXT: setp.ne.b32 %p1, %r3, %r17;
+; SM60-FTZ-ALLOW-NEXT: mov.b32 %r17, %r3;
+; SM60-FTZ-ALLOW-NEXT: @%p1 bra $L__BB74_1;
+; SM60-FTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-FTZ-ALLOW-NEXT: shr.u32 %r16, %r3, %r1;
+; SM60-FTZ-ALLOW-NEXT: membar.cta;
+; SM60-FTZ-ALLOW-NEXT: st.param.b16 [func_retval0], %r16;
+; SM60-FTZ-ALLOW-NEXT: ret;
%retval = atomicrmw fmin ptr addrspace(1) %addr, half %val syncscope("block") acq_rel
ret half %retval
}
define half @fmax_acq_rel_half_global_cta(ptr addrspace(1) %addr, half %val) {
-; SM60-NOFTZ-LABEL: fmax_acq_rel_half_global_cta(
-; SM60-NOFTZ: {
-; SM60-NOFTZ-NEXT: .reg .pred %p<2>;
-; SM60-NOFTZ-NEXT: .reg .b16 %rs<4>;
-; SM60-NOFTZ-NEXT: .reg .b32 %r<18>;
-; SM60-NOFTZ-NEXT: .reg .b64 %rd<3>;
-; SM60-NOFTZ-EMPTY:
-; SM60-NOFTZ-NEXT: // %bb.0:
-; SM60-NOFTZ-NEXT: ld.param.b16 %rs1, [fmax_acq_rel_half_global_cta_param_1];
-; SM60-NOFTZ-NEXT: ld.param.b64 %rd2, [fmax_acq_rel_half_global_cta_param_0];
-; SM60-NOFTZ-NEXT: membar.cta;
-; SM60-NOFTZ-NEXT: and.b64 %rd1, %rd2, -4;
-; SM60-NOFTZ-NEXT: cvt.u32.u64 %r4, %rd2;
-; SM60-NOFTZ-NEXT: and.b32 %r5, %r4, 3;
-; SM60-NOFTZ-NEXT: shl.b32 %r1, %r5, 3;
-; SM60-NOFTZ-NEXT: mov.b32 %r6, 65535;
-; SM60-NOFTZ-NEXT: shl.b32 %r7, %r6, %r1;
-; SM60-NOFTZ-NEXT: not.b32 %r2, %r7;
-; SM60-NOFTZ-NEXT: ld.volatile.global.b32 %r17, [%rd1];
-; SM60-NOFTZ-NEXT: cvt.f32.f16 %r10, %rs1;
-; SM60-NOFTZ-NEXT: $L__BB75_1: // %atomicrmw.start
-; SM60-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NOFTZ-NEXT: shr.u32 %r8, %r17, %r1;
-; SM60-NOFTZ-NEXT: cvt.u16.u32 %rs2, %r8;
-; SM60-NOFTZ-NEXT: cvt.f32.f16 %r9, %rs2;
-; SM60-NOFTZ-NEXT: max.f32 %r11, %r9, %r10;
-; SM60-NOFTZ-NEXT: cvt.rn.f16.f32 %rs3, %r11;
-; SM60-NOFTZ-NEXT: cvt.u32.u16 %r12, %rs3;
-; SM60-NOFTZ-NEXT: shl.b32 %r13, %r12, %r1;
-; SM60-NOFTZ-NEXT: and.b32 %r14, %r17, %r2;
-; SM60-NOFTZ-NEXT: or.b32 %r15, %r14, %r13;
-; SM60-NOFTZ-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r17, %r15;
-; SM60-NOFTZ-NEXT: setp.ne.b32 %p1, %r3, %r17;
-; SM60-NOFTZ-NEXT: mov.b32 %r17, %r3;
-; SM60-NOFTZ-NEXT: @%p1 bra $L__BB75_1;
-; SM60-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end
-; SM60-NOFTZ-NEXT: shr.u32 %r16, %r3, %r1;
-; SM60-NOFTZ-NEXT: membar.cta;
-; SM60-NOFTZ-NEXT: st.param.b16 [func_retval0], %r16;
-; SM60-NOFTZ-NEXT: ret;
+; SM60-NOFTZ-DISALLOW-LABEL: fmax_acq_rel_half_global_cta(
+; SM60-NOFTZ-DISALLOW: {
+; SM60-NOFTZ-DISALLOW-NEXT: .reg .pred %p<2>;
+; SM60-NOFTZ-DISALLOW-NEXT: .reg .b16 %rs<4>;
+; SM60-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<18>;
+; SM60-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
+; SM60-NOFTZ-DISALLOW-EMPTY:
+; SM60-NOFTZ-DISALLOW-NEXT: // %bb.0:
+; SM60-NOFTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fmax_acq_rel_half_global_cta_param_1];
+; SM60-NOFTZ-DISALLOW-NEXT: ld.param.b64 %rd2, [fmax_acq_rel_half_global_cta_param_0];
+; SM60-NOFTZ-DISALLOW-NEXT: membar.cta;
+; SM60-NOFTZ-DISALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-NOFTZ-DISALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM60-NOFTZ-DISALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-NOFTZ-DISALLOW-NEXT: mov.b32 %r6, 65535;
+; SM60-NOFTZ-DISALLOW-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-NOFTZ-DISALLOW-NEXT: not.b32 %r2, %r7;
+; SM60-NOFTZ-DISALLOW-NEXT: ld.volatile.global.b32 %r17, [%rd1];
+; SM60-NOFTZ-DISALLOW-NEXT: cvt.f32.f16 %r10, %rs1;
+; SM60-NOFTZ-DISALLOW-NEXT: $L__BB75_1: // %atomicrmw.start
+; SM60-NOFTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NOFTZ-DISALLOW-NEXT: shr.u32 %r8, %r17, %r1;
+; SM60-NOFTZ-DISALLOW-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM60-NOFTZ-DISALLOW-NEXT: cvt.f32.f16 %r9, %rs2;
+; SM60-NOFTZ-DISALLOW-NEXT: max.f32 %r11, %r9, %r10;
+; SM60-NOFTZ-DISALLOW-NEXT: cvt.rn.f16.f32 %rs3, %r11;
+; SM60-NOFTZ-DISALLOW-NEXT: cvt.u32.u16 %r12, %rs3;
+; SM60-NOFTZ-DISALLOW-NEXT: shl.b32 %r13, %r12, %r1;
+; SM60-NOFTZ-DISALLOW-NEXT: and.b32 %r14, %r17, %r2;
+; SM60-NOFTZ-DISALLOW-NEXT: or.b32 %r15, %r14, %r13;
+; SM60-NOFTZ-DISALLOW-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r17, %r15;
+; SM60-NOFTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r3, %r17;
+; SM60-NOFTZ-DISALLOW-NEXT: mov.b32 %r17, %r3;
+; SM60-NOFTZ-DISALLOW-NEXT: @%p1 bra $L__BB75_1;
+; SM60-NOFTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NOFTZ-DISALLOW-NEXT: shr.u32 %r16, %r3, %r1;
+; SM60-NOFTZ-DISALLOW-NEXT: membar.cta;
+; SM60-NOFTZ-DISALLOW-NEXT: st.param.b16 [func_retval0], %r16;
+; SM60-NOFTZ-DISALLOW-NEXT: ret;
;
-; SM60-FTZ-LABEL: fmax_acq_rel_half_global_cta(
-; SM60-FTZ: {
-; SM60-FTZ-NEXT: .reg .pred %p<2>;
-; SM60-FTZ-NEXT: .reg .b16 %rs<4>;
-; SM60-FTZ-NEXT: .reg .b32 %r<18>;
-; SM60-FTZ-NEXT: .reg .b64 %rd<3>;
-; SM60-FTZ-EMPTY:
-; SM60-FTZ-NEXT: // %bb.0:
-; SM60-FTZ-NEXT: ld.param.b16 %rs1, [fmax_acq_rel_half_global_cta_param_1];
-; SM60-FTZ-NEXT: ld.param.b64 %rd2, [fmax_acq_rel_half_global_cta_param_0];
-; SM60-FTZ-NEXT: membar.cta;
-; SM60-FTZ-NEXT: and.b64 %rd1, %rd2, -4;
-; SM60-FTZ-NEXT: cvt.u32.u64 %r4, %rd2;
-; SM60-FTZ-NEXT: and.b32 %r5, %r4, 3;
-; SM60-FTZ-NEXT: shl.b32 %r1, %r5, 3;
-; SM60-FTZ-NEXT: mov.b32 %r6, 65535;
-; SM60-FTZ-NEXT: shl.b32 %r7, %r6, %r1;
-; SM60-FTZ-NEXT: not.b32 %r2, %r7;
-; SM60-FTZ-NEXT: ld.volatile.global.b32 %r17, [%rd1];
-; SM60-FTZ-NEXT: cvt.ftz.f32.f16 %r10, %rs1;
-; SM60-FTZ-NEXT: $L__BB75_1: // %atomicrmw.start
-; SM60-FTZ-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-FTZ-NEXT: shr.u32 %r8, %r17, %r1;
-; SM60-FTZ-NEXT: cvt.u16.u32 %rs2, %r8;
-; SM60-FTZ-NEXT: cvt.ftz.f32.f16 %r9, %rs2;
-; SM60-FTZ-NEXT: max.ftz.f32 %r11, %r9, %r10;
-; SM60-FTZ-NEXT: cvt.rn.f16.f32 %rs3, %r11;
-; SM60-FTZ-NEXT: cvt.u32.u16 %r12, %rs3;
-; SM60-FTZ-NEXT: shl.b32 %r13, %r12, %r1;
-; SM60-FTZ-NEXT: and.b32 %r14, %r17, %r2;
-; SM60-FTZ-NEXT: or.b32 %r15, %r14, %r13;
-; SM60-FTZ-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r17, %r15;
-; SM60-FTZ-NEXT: setp.ne.b32 %p1, %r3, %r17;
-; SM60-FTZ-NEXT: mov.b32 %r17, %r3;
-; SM60-FTZ-NEXT: @%p1 bra $L__BB75_1;
-; SM60-FTZ-NEXT: // %bb.2: // %atomicrmw.end
-; SM60-FTZ-NEXT: shr.u32 %r16, %r3, %r1;
-; SM60-FTZ-NEXT: membar.cta;
-; SM60-FTZ-NEXT: st.param.b16 [func_retval0], %r16;
-; SM60-FTZ-NEXT: ret;
+; SM60-NOFTZ-ALLOW-LABEL: fmax_acq_rel_half_global_cta(
+; SM60-NOFTZ-ALLOW: {
+; SM60-NOFTZ-ALLOW-NEXT: .reg .pred %p<2>;
+; SM60-NOFTZ-ALLOW-NEXT: .reg .b16 %rs<4>;
+; SM60-NOFTZ-ALLOW-NEXT: .reg .b32 %r<18>;
+; SM60-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
+; SM60-NOFTZ-ALLOW-EMPTY:
+; SM60-NOFTZ-ALLOW-NEXT: // %bb.0:
+; SM60-NOFTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fmax_acq_rel_half_global_cta_param_1];
+; SM60-NOFTZ-ALLOW-NEXT: ld.param.b64 %rd2, [fmax_acq_rel_half_global_cta_param_0];
+; SM60-NOFTZ-ALLOW-NEXT: membar.cta;
+; SM60-NOFTZ-ALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NOFTZ-ALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-NOFTZ-ALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM60-NOFTZ-ALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-NOFTZ-ALLOW-NEXT: mov.b32 %r6, 65535;
+; SM60-NOFTZ-ALLOW-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-NOFTZ-ALLOW-NEXT: not.b32 %r2, %r7;
+; SM60-NOFTZ-ALLOW-NEXT: ld.volatile.global.b32 %r17, [%rd1];
+; SM60-NOFTZ-ALLOW-NEXT: cvt.f32.f16 %r10, %rs1;
+; SM60-NOFTZ-ALLOW-NEXT: $L__BB75_1: // %atomicrmw.start
+; SM60-NOFTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NOFTZ-ALLOW-NEXT: shr.u32 %r8, %r17, %r1;
+; SM60-NOFTZ-ALLOW-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM60-NOFTZ-ALLOW-NEXT: cvt.f32.f16 %r9, %rs2;
+; SM60-NOFTZ-ALLOW-NEXT: max.f32 %r11, %r9, %r10;
+; SM60-NOFTZ-ALLOW-NEXT: cvt.rn.f16.f32 %rs3, %r11;
+; SM60-NOFTZ-ALLOW-NEXT: cvt.u32.u16 %r12, %rs3;
+; SM60-NOFTZ-ALLOW-NEXT: shl.b32 %r13, %r12, %r1;
+; SM60-NOFTZ-ALLOW-NEXT: and.b32 %r14, %r17, %r2;
+; SM60-NOFTZ-ALLOW-NEXT: or.b32 %r15, %r14, %r13;
+; SM60-NOFTZ-ALLOW-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r17, %r15;
+; SM60-NOFTZ-ALLOW-NEXT: setp.ne.b32 %p1, %r3, %r17;
+; SM60-NOFTZ-ALLOW-NEXT: mov.b32 %r17, %r3;
+; SM60-NOFTZ-ALLOW-NEXT: @%p1 bra $L__BB75_1;
+; SM60-NOFTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NOFTZ-ALLOW-NEXT: shr.u32 %r16, %r3, %r1;
+; SM60-NOFTZ-ALLOW-NEXT: membar.cta;
+; SM60-NOFTZ-ALLOW-NEXT: st.param.b16 [func_retval0], %r16;
+; SM60-NOFTZ-ALLOW-NEXT: ret;
+;
+; SM60-FTZ-DISALLOW-LABEL: fmax_acq_rel_half_global_cta(
+; SM60-FTZ-DISALLOW: {
+; SM60-FTZ-DISALLOW-NEXT: .reg .pred %p<2>;
+; SM60-FTZ-DISALLOW-NEXT: .reg .b16 %rs<4>;
+; SM60-FTZ-DISALLOW-NEXT: .reg .b32 %r<18>;
+; SM60-FTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
+; SM60-FTZ-DISALLOW-EMPTY:
+; SM60-FTZ-DISALLOW-NEXT: // %bb.0:
+; SM60-FTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fmax_acq_rel_half_global_cta_param_1];
+; SM60-FTZ-DISALLOW-NEXT: ld.param.b64 %rd2, [fmax_acq_rel_half_global_cta_param_0];
+; SM60-FTZ-DISALLOW-NEXT: membar.cta;
+; SM60-FTZ-DISALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-FTZ-DISALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-FTZ-DISALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM60-FTZ-DISALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-FTZ-DISALLOW-NEXT: mov.b32 %r6, 65535;
+; SM60-FTZ-DISALLOW-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-FTZ-DISALLOW-NEXT: not.b32 %r2, %r7;
+; SM60-FTZ-DISALLOW-NEXT: ld.volatile.global.b32 %r17, [%rd1];
+; SM60-FTZ-DISALLOW-NEXT: cvt.ftz.f32.f16 %r10, %rs1;
+; SM60-FTZ-DISALLOW-NEXT: $L__BB75_1: // %atomicrmw.start
+; SM60-FTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-FTZ-DISALLOW-NEXT: shr.u32 %r8, %r17, %r1;
+; SM60-FTZ-DISALLOW-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM60-FTZ-DISALLOW-NEXT: cvt.ftz.f32.f16 %r9, %rs2;
+; SM60-FTZ-DISALLOW-NEXT: max.ftz.f32 %r11, %r9, %r10;
+; SM60-FTZ-DISALLOW-NEXT: cvt.rn.f16.f32 %rs3, %r11;
+; SM60-FTZ-DISALLOW-NEXT: cvt.u32.u16 %r12, %rs3;
+; SM60-FTZ-DISALLOW-NEXT: shl.b32 %r13, %r12, %r1;
+; SM60-FTZ-DISALLOW-NEXT: and.b32 %r14, %r17, %r2;
+; SM60-FTZ-DISALLOW-NEXT: or.b32 %r15, %r14, %r13;
+; SM60-FTZ-DISALLOW-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r17, %r15;
+; SM60-FTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r3, %r17;
+; SM60-FTZ-DISALLOW-NEXT: mov.b32 %r17, %r3;
+; SM60-FTZ-DISALLOW-NEXT: @%p1 bra $L__BB75_1;
+; SM60-FTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-FTZ-DISALLOW-NEXT: shr.u32 %r16, %r3, %r1;
+; SM60-FTZ-DISALLOW-NEXT: membar.cta;
+; SM60-FTZ-DISALLOW-NEXT: st.param.b16 [func_retval0], %r16;
+; SM60-FTZ-DISALLOW-NEXT: ret;
+;
+; SM60-FTZ-ALLOW-LABEL: fmax_acq_rel_half_global_cta(
+; SM60-FTZ-ALLOW: {
+; SM60-FTZ-ALLOW-NEXT: .reg .pred %p<2>;
+; SM60-FTZ-ALLOW-NEXT: .reg .b16 %rs<4>;
+; SM60-FTZ-ALLOW-NEXT: .reg .b32 %r<18>;
+; SM60-FTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
+; SM60-FTZ-ALLOW-EMPTY:
+; SM60-FTZ-ALLOW-NEXT: // %bb.0:
+; SM60-FTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fmax_acq_rel_half_global_cta_param_1];
+; SM60-FTZ-ALLOW-NEXT: ld.param.b64 %rd2, [fmax_acq_rel_half_global_cta_param_0];
+; SM60-FTZ-ALLOW-NEXT: membar.cta;
+; SM60-FTZ-ALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-FTZ-ALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-FTZ-ALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM60-FTZ-ALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-FTZ-ALLOW-NEXT: mov.b32 %r6, 65535;
+; SM60-FTZ-ALLOW-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-FTZ-ALLOW-NEXT: not.b32 %r2, %r7;
+; SM60-FTZ-ALLOW-NEXT: ld.volatile.global.b32 %r17, [%rd1];
+; SM60-FTZ-ALLOW-NEXT: cvt.ftz.f32.f16 %r10, %rs1;
+; SM60-FTZ-ALLOW-NEXT: $L__BB75_1: // %atomicrmw.start
+; SM60-FTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-FTZ-ALLOW-NEXT: shr.u32 %r8, %r17, %r1;
+; SM60-FTZ-ALLOW-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM60-FTZ-ALLOW-NEXT: cvt.ftz.f32.f16 %r9, %rs2;
+; SM60-FTZ-ALLOW-NEXT: max.ftz.f32 %r11, %r9, %r10;
+; SM60-FTZ-ALLOW-NEXT: cvt.rn.f16.f32 %rs3, %r11;
+; SM60-FTZ-ALLOW-NEXT: cvt.u32.u16 %r12, %rs3;
+; SM60-FTZ-ALLOW-NEXT: shl.b32 %r13, %r12, %r1;
+; SM60-FTZ-ALLOW-NEXT: and.b32 %r14, %r17, %r2;
+; SM60-FTZ-ALLOW-NEXT: or.b32 %r15, %r14, %r13;
+; SM60-FTZ-ALLOW-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r17, %r15;
+; SM60-FTZ-ALLOW-NEXT: setp.ne.b32 %p1, %r3, %r17;
+; SM60-FTZ-ALLOW-NEXT: mov.b32 %r17, %r3;
+; SM60-FTZ-ALLOW-NEXT: @%p1 bra $L__BB75_1;
+; SM60-FTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-FTZ-ALLOW-NEXT: shr.u32 %r16, %r3, %r1;
+; SM60-FTZ-ALLOW-NEXT: membar.cta;
+; SM60-FTZ-ALLOW-NEXT: st.param.b16 [func_retval0], %r16;
+; SM60-FTZ-ALLOW-NEXT: ret;
%retval = atomicrmw fmax ptr addrspace(1) %addr, half %val syncscope("block") acq_rel
ret half %retval
}
define half @fminimum_acq_rel_half_global_cta(ptr addrspace(1) %addr, half %val) {
-; SM60-NOFTZ-LABEL: fminimum_acq_rel_half_global_cta(
-; SM60-NOFTZ: {
-; SM60-NOFTZ-NEXT: .reg .pred %p<6>;
-; SM60-NOFTZ-NEXT: .reg .b16 %rs<7>;
-; SM60-NOFTZ-NEXT: .reg .b32 %r<15>;
-; SM60-NOFTZ-NEXT: .reg .b64 %rd<3>;
-; SM60-NOFTZ-EMPTY:
-; SM60-NOFTZ-NEXT: // %bb.0:
-; SM60-NOFTZ-NEXT: ld.param.b16 %rs1, [fminimum_acq_rel_half_global_cta_param_1];
-; SM60-NOFTZ-NEXT: ld.param.b64 %rd2, [fminimum_acq_rel_half_global_cta_param_0];
-; SM60-NOFTZ-NEXT: membar.cta;
-; SM60-NOFTZ-NEXT: and.b64 %rd1, %rd2, -4;
-; SM60-NOFTZ-NEXT: cvt.u32.u64 %r4, %rd2;
-; SM60-NOFTZ-NEXT: and.b32 %r5, %r4, 3;
-; SM60-NOFTZ-NEXT: shl.b32 %r1, %r5, 3;
-; SM60-NOFTZ-NEXT: mov.b32 %r6, 65535;
-; SM60-NOFTZ-NEXT: shl.b32 %r7, %r6, %r1;
-; SM60-NOFTZ-NEXT: not.b32 %r2, %r7;
-; SM60-NOFTZ-NEXT: ld.volatile.global.b32 %r14, [%rd1];
-; SM60-NOFTZ-NEXT: $L__BB76_1: // %atomicrmw.start
-; SM60-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NOFTZ-NEXT: shr.u32 %r8, %r14, %r1;
-; SM60-NOFTZ-NEXT: cvt.u16.u32 %rs2, %r8;
-; SM60-NOFTZ-NEXT: setp.eq.b16 %p1, %rs2, -32768;
-; SM60-NOFTZ-NEXT: selp.b16 %rs3, %rs2, %rs1, %p1;
-; SM60-NOFTZ-NEXT: setp.lt.f16 %p2, %rs2, %rs1;
-; SM60-NOFTZ-NEXT: selp.b16 %rs4, %rs2, %rs1, %p2;
-; SM60-NOFTZ-NEXT: setp.nan.f16 %p3, %rs2, %rs1;
-; SM60-NOFTZ-NEXT: selp.b16 %rs5, 0x7E00, %rs4, %p3;
-; SM60-NOFTZ-NEXT: setp.eq.f16 %p4, %rs2, %rs1;
-; SM60-NOFTZ-NEXT: selp.b16 %rs6, %rs3, %rs5, %p4;
-; SM60-NOFTZ-NEXT: cvt.u32.u16 %r9, %rs6;
-; SM60-NOFTZ-NEXT: shl.b32 %r10, %r9, %r1;
-; SM60-NOFTZ-NEXT: and.b32 %r11, %r14, %r2;
-; SM60-NOFTZ-NEXT: or.b32 %r12, %r11, %r10;
-; SM60-NOFTZ-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
-; SM60-NOFTZ-NEXT: setp.ne.b32 %p5, %r3, %r14;
-; SM60-NOFTZ-NEXT: mov.b32 %r14, %r3;
-; SM60-NOFTZ-NEXT: @%p5 bra $L__BB76_1;
-; SM60-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end
-; SM60-NOFTZ-NEXT: shr.u32 %r13, %r3, %r1;
-; SM60-NOFTZ-NEXT: membar.cta;
-; SM60-NOFTZ-NEXT: st.param.b16 [func_retval0], %r13;
-; SM60-NOFTZ-NEXT: ret;
+; SM60-NOFTZ-DISALLOW-LABEL: fminimum_acq_rel_half_global_cta(
+; SM60-NOFTZ-DISALLOW: {
+; SM60-NOFTZ-DISALLOW-NEXT: .reg .pred %p<6>;
+; SM60-NOFTZ-DISALLOW-NEXT: .reg .b16 %rs<7>;
+; SM60-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<15>;
+; SM60-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
+; SM60-NOFTZ-DISALLOW-EMPTY:
+; SM60-NOFTZ-DISALLOW-NEXT: // %bb.0:
+; SM60-NOFTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fminimum_acq_rel_half_global_cta_param_1];
+; SM60-NOFTZ-DISALLOW-NEXT: ld.param.b64 %rd2, [fminimum_acq_rel_half_global_cta_param_0];
+; SM60-NOFTZ-DISALLOW-NEXT: membar.cta;
+; SM60-NOFTZ-DISALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-NOFTZ-DISALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM60-NOFTZ-DISALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-NOFTZ-DISALLOW-NEXT: mov.b32 %r6, 65535;
+; SM60-NOFTZ-DISALLOW-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-NOFTZ-DISALLOW-NEXT: not.b32 %r2, %r7;
+; SM60-NOFTZ-DISALLOW-NEXT: ld.volatile.global.b32 %r14, [%rd1];
+; SM60-NOFTZ-DISALLOW-NEXT: $L__BB76_1: // %atomicrmw.start
+; SM60-NOFTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NOFTZ-DISALLOW-NEXT: shr.u32 %r8, %r14, %r1;
+; SM60-NOFTZ-DISALLOW-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM60-NOFTZ-DISALLOW-NEXT: setp.eq.b16 %p1, %rs2, -32768;
+; SM60-NOFTZ-DISALLOW-NEXT: selp.b16 %rs3, %rs2, %rs1, %p1;
+; SM60-NOFTZ-DISALLOW-NEXT: setp.lt.f16 %p2, %rs2, %rs1;
+; SM60-NOFTZ-DISALLOW-NEXT: selp.b16 %rs4, %rs2, %rs1, %p2;
+; SM60-NOFTZ-DISALLOW-NEXT: setp.nan.f16 %p3, %rs2, %rs1;
+; SM60-NOFTZ-DISALLOW-NEXT: selp.b16 %rs5, 0x7E00, %rs4, %p3;
+; SM60-NOFTZ-DISALLOW-NEXT: setp.eq.f16 %p4, %rs2, %rs1;
+; SM60-NOFTZ-DISALLOW-NEXT: selp.b16 %rs6, %rs3, %rs5, %p4;
+; SM60-NOFTZ-DISALLOW-NEXT: cvt.u32.u16 %r9, %rs6;
+; SM60-NOFTZ-DISALLOW-NEXT: shl.b32 %r10, %r9, %r1;
+; SM60-NOFTZ-DISALLOW-NEXT: and.b32 %r11, %r14, %r2;
+; SM60-NOFTZ-DISALLOW-NEXT: or.b32 %r12, %r11, %r10;
+; SM60-NOFTZ-DISALLOW-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM60-NOFTZ-DISALLOW-NEXT: setp.ne.b32 %p5, %r3, %r14;
+; SM60-NOFTZ-DISALLOW-NEXT: mov.b32 %r14, %r3;
+; SM60-NOFTZ-DISALLOW-NEXT: @%p5 bra $L__BB76_1;
+; SM60-NOFTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NOFTZ-DISALLOW-NEXT: shr.u32 %r13, %r3, %r1;
+; SM60-NOFTZ-DISALLOW-NEXT: membar.cta;
+; SM60-NOFTZ-DISALLOW-NEXT: st.param.b16 [func_retval0], %r13;
+; SM60-NOFTZ-DISALLOW-NEXT: ret;
+;
+; SM60-NOFTZ-ALLOW-LABEL: fminimum_acq_rel_half_global_cta(
+; SM60-NOFTZ-ALLOW: {
+; SM60-NOFTZ-ALLOW-NEXT: .reg .pred %p<6>;
+; SM60-NOFTZ-ALLOW-NEXT: .reg .b16 %rs<7>;
+; SM60-NOFTZ-ALLOW-NEXT: .reg .b32 %r<15>;
+; SM60-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
+; SM60-NOFTZ-ALLOW-EMPTY:
+; SM60-NOFTZ-ALLOW-NEXT: // %bb.0:
+; SM60-NOFTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fminimum_acq_rel_half_global_cta_param_1];
+; SM60-NOFTZ-ALLOW-NEXT: ld.param.b64 %rd2, [fminimum_acq_rel_half_global_cta_param_0];
+; SM60-NOFTZ-ALLOW-NEXT: membar.cta;
+; SM60-NOFTZ-ALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NOFTZ-ALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-NOFTZ-ALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM60-NOFTZ-ALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-NOFTZ-ALLOW-NEXT: mov.b32 %r6, 65535;
+; SM60-NOFTZ-ALLOW-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-NOFTZ-ALLOW-NEXT: not.b32 %r2, %r7;
+; SM60-NOFTZ-ALLOW-NEXT: ld.volatile.global.b32 %r14, [%rd1];
+; SM60-NOFTZ-ALLOW-NEXT: $L__BB76_1: // %atomicrmw.start
+; SM60-NOFTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NOFTZ-ALLOW-NEXT: shr.u32 %r8, %r14, %r1;
+; SM60-NOFTZ-ALLOW-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM60-NOFTZ-ALLOW-NEXT: setp.eq.b16 %p1, %rs2, -32768;
+; SM60-NOFTZ-ALLOW-NEXT: selp.b16 %rs3, %rs2, %rs1, %p1;
+; SM60-NOFTZ-ALLOW-NEXT: setp.lt.f16 %p2, %rs2, %rs1;
+; SM60-NOFTZ-ALLOW-NEXT: selp.b16 %rs4, %rs2, %rs1, %p2;
+; SM60-NOFTZ-ALLOW-NEXT: setp.nan.f16 %p3, %rs2, %rs1;
+; SM60-NOFTZ-ALLOW-NEXT: selp.b16 %rs5, 0x7E00, %rs4, %p3;
+; SM60-NOFTZ-ALLOW-NEXT: setp.eq.f16 %p4, %rs2, %rs1;
+; SM60-NOFTZ-ALLOW-NEXT: selp.b16 %rs6, %rs3, %rs5, %p4;
+; SM60-NOFTZ-ALLOW-NEXT: cvt.u32.u16 %r9, %rs6;
+; SM60-NOFTZ-ALLOW-NEXT: shl.b32 %r10, %r9, %r1;
+; SM60-NOFTZ-ALLOW-NEXT: and.b32 %r11, %r14, %r2;
+; SM60-NOFTZ-ALLOW-NEXT: or.b32 %r12, %r11, %r10;
+; SM60-NOFTZ-ALLOW-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM60-NOFTZ-ALLOW-NEXT: setp.ne.b32 %p5, %r3, %r14;
+; SM60-NOFTZ-ALLOW-NEXT: mov.b32 %r14, %r3;
+; SM60-NOFTZ-ALLOW-NEXT: @%p5 bra $L__BB76_1;
+; SM60-NOFTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NOFTZ-ALLOW-NEXT: shr.u32 %r13, %r3, %r1;
+; SM60-NOFTZ-ALLOW-NEXT: membar.cta;
+; SM60-NOFTZ-ALLOW-NEXT: st.param.b16 [func_retval0], %r13;
+; SM60-NOFTZ-ALLOW-NEXT: ret;
;
-; SM60-FTZ-LABEL: fminimum_acq_rel_half_global_cta(
-; SM60-FTZ: {
-; SM60-FTZ-NEXT: .reg .pred %p<6>;
-; SM60-FTZ-NEXT: .reg .b16 %rs<7>;
-; SM60-FTZ-NEXT: .reg .b32 %r<15>;
-; SM60-FTZ-NEXT: .reg .b64 %rd<3>;
-; SM60-FTZ-EMPTY:
-; SM60-FTZ-NEXT: // %bb.0:
-; SM60-FTZ-NEXT: ld.param.b16 %rs1, [fminimum_acq_rel_half_global_cta_param_1];
-; SM60-FTZ-NEXT: ld.param.b64 %rd2, [fminimum_acq_rel_half_global_cta_param_0];
-; SM60-FTZ-NEXT: membar.cta;
-; SM60-FTZ-NEXT: and.b64 %rd1, %rd2, -4;
-; SM60-FTZ-NEXT: cvt.u32.u64 %r4, %rd2;
-; SM60-FTZ-NEXT: and.b32 %r5, %r4, 3;
-; SM60-FTZ-NEXT: shl.b32 %r1, %r5, 3;
-; SM60-FTZ-NEXT: mov.b32 %r6, 65535;
-; SM60-FTZ-NEXT: shl.b32 %r7, %r6, %r1;
-; SM60-FTZ-NEXT: not.b32 %r2, %r7;
-; SM60-FTZ-NEXT: ld.volatile.global.b32 %r14, [%rd1];
-; SM60-FTZ-NEXT: $L__BB76_1: // %atomicrmw.start
-; SM60-FTZ-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-FTZ-NEXT: shr.u32 %r8, %r14, %r1;
-; SM60-FTZ-NEXT: cvt.u16.u32 %rs2, %r8;
-; SM60-FTZ-NEXT: setp.eq.b16 %p1, %rs2, -32768;
-; SM60-FTZ-NEXT: selp.b16 %rs3, %rs2, %rs1, %p1;
-; SM60-FTZ-NEXT: setp.lt.ftz.f16 %p2, %rs2, %rs1;
-; SM60-FTZ-NEXT: selp.b16 %rs4, %rs2, %rs1, %p2;
-; SM60-FTZ-NEXT: setp.nan.ftz.f16 %p3, %rs2, %rs1;
-; SM60-FTZ-NEXT: selp.b16 %rs5, 0x7E00, %rs4, %p3;
-; SM60-FTZ-NEXT: setp.eq.ftz.f16 %p4, %rs2, %rs1;
-; SM60-FTZ-NEXT: selp.b16 %rs6, %rs3, %rs5, %p4;
-; SM60-FTZ-NEXT: cvt.u32.u16 %r9, %rs6;
-; SM60-FTZ-NEXT: shl.b32 %r10, %r9, %r1;
-; SM60-FTZ-NEXT: and.b32 %r11, %r14, %r2;
-; SM60-FTZ-NEXT: or.b32 %r12, %r11, %r10;
-; SM60-FTZ-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
-; SM60-FTZ-NEXT: setp.ne.b32 %p5, %r3, %r14;
-; SM60-FTZ-NEXT: mov.b32 %r14, %r3;
-; SM60-FTZ-NEXT: @%p5 bra $L__BB76_1;
-; SM60-FTZ-NEXT: // %bb.2: // %atomicrmw.end
-; SM60-FTZ-NEXT: shr.u32 %r13, %r3, %r1;
-; SM60-FTZ-NEXT: membar.cta;
-; SM60-FTZ-NEXT: st.param.b16 [func_retval0], %r13;
-; SM60-FTZ-NEXT: ret;
+; SM60-FTZ-DISALLOW-LABEL: fminimum_acq_rel_half_global_cta(
+; SM60-FTZ-DISALLOW: {
+; SM60-FTZ-DISALLOW-NEXT: .reg .pred %p<6>;
+; SM60-FTZ-DISALLOW-NEXT: .reg .b16 %rs<7>;
+; SM60-FTZ-DISALLOW-NEXT: .reg .b32 %r<15>;
+; SM60-FTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
+; SM60-FTZ-DISALLOW-EMPTY:
+; SM60-FTZ-DISALLOW-NEXT: // %bb.0:
+; SM60-FTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fminimum_acq_rel_half_global_cta_param_1];
+; SM60-FTZ-DISALLOW-NEXT: ld.param.b64 %rd2, [fminimum_acq_rel_half_global_cta_param_0];
+; SM60-FTZ-DISALLOW-NEXT: membar.cta;
+; SM60-FTZ-DISALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-FTZ-DISALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-FTZ-DISALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM60-FTZ-DISALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-FTZ-DISALLOW-NEXT: mov.b32 %r6, 65535;
+; SM60-FTZ-DISALLOW-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-FTZ-DISALLOW-NEXT: not.b32 %r2, %r7;
+; SM60-FTZ-DISALLOW-NEXT: ld.volatile.global.b32 %r14, [%rd1];
+; SM60-FTZ-DISALLOW-NEXT: $L__BB76_1: // %atomicrmw.start
+; SM60-FTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-FTZ-DISALLOW-NEXT: shr.u32 %r8, %r14, %r1;
+; SM60-FTZ-DISALLOW-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM60-FTZ-DISALLOW-NEXT: setp.eq.b16 %p1, %rs2, -32768;
+; SM60-FTZ-DISALLOW-NEXT: selp.b16 %rs3, %rs2, %rs1, %p1;
+; SM60-FTZ-DISALLOW-NEXT: setp.lt.ftz.f16 %p2, %rs2, %rs1;
+; SM60-FTZ-DISALLOW-NEXT: selp.b16 %rs4, %rs2, %rs1, %p2;
+; SM60-FTZ-DISALLOW-NEXT: setp.nan.ftz.f16 %p3, %rs2, %rs1;
+; SM60-FTZ-DISALLOW-NEXT: selp.b16 %rs5, 0x7E00, %rs4, %p3;
+; SM60-FTZ-DISALLOW-NEXT: setp.eq.ftz.f16 %p4, %rs2, %rs1;
+; SM60-FTZ-DISALLOW-NEXT: selp.b16 %rs6, %rs3, %rs5, %p4;
+; SM60-FTZ-DISALLOW-NEXT: cvt.u32.u16 %r9, %rs6;
+; SM60-FTZ-DISALLOW-NEXT: shl.b32 %r10, %r9, %r1;
+; SM60-FTZ-DISALLOW-NEXT: and.b32 %r11, %r14, %r2;
+; SM60-FTZ-DISALLOW-NEXT: or.b32 %r12, %r11, %r10;
+; SM60-FTZ-DISALLOW-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM60-FTZ-DISALLOW-NEXT: setp.ne.b32 %p5, %r3, %r14;
+; SM60-FTZ-DISALLOW-NEXT: mov.b32 %r14, %r3;
+; SM60-FTZ-DISALLOW-NEXT: @%p5 bra $L__BB76_1;
+; SM60-FTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-FTZ-DISALLOW-NEXT: shr.u32 %r13, %r3, %r1;
+; SM60-FTZ-DISALLOW-NEXT: membar.cta;
+; SM60-FTZ-DISALLOW-NEXT: st.param.b16 [func_retval0], %r13;
+; SM60-FTZ-DISALLOW-NEXT: ret;
+;
+; SM60-FTZ-ALLOW-LABEL: fminimum_acq_rel_half_global_cta(
+; SM60-FTZ-ALLOW: {
+; SM60-FTZ-ALLOW-NEXT: .reg .pred %p<6>;
+; SM60-FTZ-ALLOW-NEXT: .reg .b16 %rs<7>;
+; SM60-FTZ-ALLOW-NEXT: .reg .b32 %r<15>;
+; SM60-FTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
+; SM60-FTZ-ALLOW-EMPTY:
+; SM60-FTZ-ALLOW-NEXT: // %bb.0:
+; SM60-FTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fminimum_acq_rel_half_global_cta_param_1];
+; SM60-FTZ-ALLOW-NEXT: ld.param.b64 %rd2, [fminimum_acq_rel_half_global_cta_param_0];
+; SM60-FTZ-ALLOW-NEXT: membar.cta;
+; SM60-FTZ-ALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-FTZ-ALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-FTZ-ALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM60-FTZ-ALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-FTZ-ALLOW-NEXT: mov.b32 %r6, 65535;
+; SM60-FTZ-ALLOW-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-FTZ-ALLOW-NEXT: not.b32 %r2, %r7;
+; SM60-FTZ-ALLOW-NEXT: ld.volatile.global.b32 %r14, [%rd1];
+; SM60-FTZ-ALLOW-NEXT: $L__BB76_1: // %atomicrmw.start
+; SM60-FTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-FTZ-ALLOW-NEXT: shr.u32 %r8, %r14, %r1;
+; SM60-FTZ-ALLOW-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM60-FTZ-ALLOW-NEXT: setp.eq.b16 %p1, %rs2, -32768;
+; SM60-FTZ-ALLOW-NEXT: selp.b16 %rs3, %rs2, %rs1, %p1;
+; SM60-FTZ-ALLOW-NEXT: setp.lt.ftz.f16 %p2, %rs2, %rs1;
+; SM60-FTZ-ALLOW-NEXT: selp.b16 %rs4, %rs2, %rs1, %p2;
+; SM60-FTZ-ALLOW-NEXT: setp.nan.ftz.f16 %p3, %rs2, %rs1;
+; SM60-FTZ-ALLOW-NEXT: selp.b16 %rs5, 0x7E00, %rs4, %p3;
+; SM60-FTZ-ALLOW-NEXT: setp.eq.ftz.f16 %p4, %rs2, %rs1;
+; SM60-FTZ-ALLOW-NEXT: selp.b16 %rs6, %rs3, %rs5, %p4;
+; SM60-FTZ-ALLOW-NEXT: cvt.u32.u16 %r9, %rs6;
+; SM60-FTZ-ALLOW-NEXT: shl.b32 %r10, %r9, %r1;
+; SM60-FTZ-ALLOW-NEXT: and.b32 %r11, %r14, %r2;
+; SM60-FTZ-ALLOW-NEXT: or.b32 %r12, %r11, %r10;
+; SM60-FTZ-ALLOW-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM60-FTZ-ALLOW-NEXT: setp.ne.b32 %p5, %r3, %r14;
+; SM60-FTZ-ALLOW-NEXT: mov.b32 %r14, %r3;
+; SM60-FTZ-ALLOW-NEXT: @%p5 bra $L__BB76_1;
+; SM60-FTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-FTZ-ALLOW-NEXT: shr.u32 %r13, %r3, %r1;
+; SM60-FTZ-ALLOW-NEXT: membar.cta;
+; SM60-FTZ-ALLOW-NEXT: st.param.b16 [func_retval0], %r13;
+; SM60-FTZ-ALLOW-NEXT: ret;
%retval = atomicrmw fminimum ptr addrspace(1) %addr, half %val syncscope("block") acq_rel
ret half %retval
}
define half @fmaximum_acq_rel_half_global_cta(ptr addrspace(1) %addr, half %val) {
-; SM60-NOFTZ-LABEL: fmaximum_acq_rel_half_global_cta(
-; SM60-NOFTZ: {
-; SM60-NOFTZ-NEXT: .reg .pred %p<6>;
-; SM60-NOFTZ-NEXT: .reg .b16 %rs<7>;
-; SM60-NOFTZ-NEXT: .reg .b32 %r<15>;
-; SM60-NOFTZ-NEXT: .reg .b64 %rd<3>;
-; SM60-NOFTZ-EMPTY:
-; SM60-NOFTZ-NEXT: // %bb.0:
-; SM60-NOFTZ-NEXT: ld.param.b16 %rs1, [fmaximum_acq_rel_half_global_cta_param_1];
-; SM60-NOFTZ-NEXT: ld.param.b64 %rd2, [fmaximum_acq_rel_half_global_cta_param_0];
-; SM60-NOFTZ-NEXT: membar.cta;
-; SM60-NOFTZ-NEXT: and.b64 %rd1, %rd2, -4;
-; SM60-NOFTZ-NEXT: cvt.u32.u64 %r4, %rd2;
-; SM60-NOFTZ-NEXT: and.b32 %r5, %r4, 3;
-; SM60-NOFTZ-NEXT: shl.b32 %r1, %r5, 3;
-; SM60-NOFTZ-NEXT: mov.b32 %r6, 65535;
-; SM60-NOFTZ-NEXT: shl.b32 %r7, %r6, %r1;
-; SM60-NOFTZ-NEXT: not.b32 %r2, %r7;
-; SM60-NOFTZ-NEXT: ld.volatile.global.b32 %r14, [%rd1];
-; SM60-NOFTZ-NEXT: $L__BB77_1: // %atomicrmw.start
-; SM60-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NOFTZ-NEXT: shr.u32 %r8, %r14, %r1;
-; SM60-NOFTZ-NEXT: cvt.u16.u32 %rs2, %r8;
-; SM60-NOFTZ-NEXT: setp.eq.b16 %p1, %rs2, 0;
-; SM60-NOFTZ-NEXT: selp.b16 %rs3, %rs2, %rs1, %p1;
-; SM60-NOFTZ-NEXT: setp.gt.f16 %p2, %rs2, %rs1;
-; SM60-NOFTZ-NEXT: selp.b16 %rs4, %rs2, %rs1, %p2;
-; SM60-NOFTZ-NEXT: setp.nan.f16 %p3, %rs2, %rs1;
-; SM60-NOFTZ-NEXT: selp.b16 %rs5, 0x7E00, %rs4, %p3;
-; SM60-NOFTZ-NEXT: setp.eq.f16 %p4, %rs2, %rs1;
-; SM60-NOFTZ-NEXT: selp.b16 %rs6, %rs3, %rs5, %p4;
-; SM60-NOFTZ-NEXT: cvt.u32.u16 %r9, %rs6;
-; SM60-NOFTZ-NEXT: shl.b32 %r10, %r9, %r1;
-; SM60-NOFTZ-NEXT: and.b32 %r11, %r14, %r2;
-; SM60-NOFTZ-NEXT: or.b32 %r12, %r11, %r10;
-; SM60-NOFTZ-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
-; SM60-NOFTZ-NEXT: setp.ne.b32 %p5, %r3, %r14;
-; SM60-NOFTZ-NEXT: mov.b32 %r14, %r3;
-; SM60-NOFTZ-NEXT: @%p5 bra $L__BB77_1;
-; SM60-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end
-; SM60-NOFTZ-NEXT: shr.u32 %r13, %r3, %r1;
-; SM60-NOFTZ-NEXT: membar.cta;
-; SM60-NOFTZ-NEXT: st.param.b16 [func_retval0], %r13;
-; SM60-NOFTZ-NEXT: ret;
+; SM60-NOFTZ-DISALLOW-LABEL: fmaximum_acq_rel_half_global_cta(
+; SM60-NOFTZ-DISALLOW: {
+; SM60-NOFTZ-DISALLOW-NEXT: .reg .pred %p<6>;
+; SM60-NOFTZ-DISALLOW-NEXT: .reg .b16 %rs<7>;
+; SM60-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<15>;
+; SM60-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
+; SM60-NOFTZ-DISALLOW-EMPTY:
+; SM60-NOFTZ-DISALLOW-NEXT: // %bb.0:
+; SM60-NOFTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fmaximum_acq_rel_half_global_cta_param_1];
+; SM60-NOFTZ-DISALLOW-NEXT: ld.param.b64 %rd2, [fmaximum_acq_rel_half_global_cta_param_0];
+; SM60-NOFTZ-DISALLOW-NEXT: membar.cta;
+; SM60-NOFTZ-DISALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-NOFTZ-DISALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM60-NOFTZ-DISALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-NOFTZ-DISALLOW-NEXT: mov.b32 %r6, 65535;
+; SM60-NOFTZ-DISALLOW-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-NOFTZ-DISALLOW-NEXT: not.b32 %r2, %r7;
+; SM60-NOFTZ-DISALLOW-NEXT: ld.volatile.global.b32 %r14, [%rd1];
+; SM60-NOFTZ-DISALLOW-NEXT: $L__BB77_1: // %atomicrmw.start
+; SM60-NOFTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NOFTZ-DISALLOW-NEXT: shr.u32 %r8, %r14, %r1;
+; SM60-NOFTZ-DISALLOW-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM60-NOFTZ-DISALLOW-NEXT: setp.eq.b16 %p1, %rs2, 0;
+; SM60-NOFTZ-DISALLOW-NEXT: selp.b16 %rs3, %rs2, %rs1, %p1;
+; SM60-NOFTZ-DISALLOW-NEXT: setp.gt.f16 %p2, %rs2, %rs1;
+; SM60-NOFTZ-DISALLOW-NEXT: selp.b16 %rs4, %rs2, %rs1, %p2;
+; SM60-NOFTZ-DISALLOW-NEXT: setp.nan.f16 %p3, %rs2, %rs1;
+; SM60-NOFTZ-DISALLOW-NEXT: selp.b16 %rs5, 0x7E00, %rs4, %p3;
+; SM60-NOFTZ-DISALLOW-NEXT: setp.eq.f16 %p4, %rs2, %rs1;
+; SM60-NOFTZ-DISALLOW-NEXT: selp.b16 %rs6, %rs3, %rs5, %p4;
+; SM60-NOFTZ-DISALLOW-NEXT: cvt.u32.u16 %r9, %rs6;
+; SM60-NOFTZ-DISALLOW-NEXT: shl.b32 %r10, %r9, %r1;
+; SM60-NOFTZ-DISALLOW-NEXT: and.b32 %r11, %r14, %r2;
+; SM60-NOFTZ-DISALLOW-NEXT: or.b32 %r12, %r11, %r10;
+; SM60-NOFTZ-DISALLOW-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM60-NOFTZ-DISALLOW-NEXT: setp.ne.b32 %p5, %r3, %r14;
+; SM60-NOFTZ-DISALLOW-NEXT: mov.b32 %r14, %r3;
+; SM60-NOFTZ-DISALLOW-NEXT: @%p5 bra $L__BB77_1;
+; SM60-NOFTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NOFTZ-DISALLOW-NEXT: shr.u32 %r13, %r3, %r1;
+; SM60-NOFTZ-DISALLOW-NEXT: membar.cta;
+; SM60-NOFTZ-DISALLOW-NEXT: st.param.b16 [func_retval0], %r13;
+; SM60-NOFTZ-DISALLOW-NEXT: ret;
+;
+; SM60-NOFTZ-ALLOW-LABEL: fmaximum_acq_rel_half_global_cta(
+; SM60-NOFTZ-ALLOW: {
+; SM60-NOFTZ-ALLOW-NEXT: .reg .pred %p<6>;
+; SM60-NOFTZ-ALLOW-NEXT: .reg .b16 %rs<7>;
+; SM60-NOFTZ-ALLOW-NEXT: .reg .b32 %r<15>;
+; SM60-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
+; SM60-NOFTZ-ALLOW-EMPTY:
+; SM60-NOFTZ-ALLOW-NEXT: // %bb.0:
+; SM60-NOFTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fmaximum_acq_rel_half_global_cta_param_1];
+; SM60-NOFTZ-ALLOW-NEXT: ld.param.b64 %rd2, [fmaximum_acq_rel_half_global_cta_param_0];
+; SM60-NOFTZ-ALLOW-NEXT: membar.cta;
+; SM60-NOFTZ-ALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NOFTZ-ALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-NOFTZ-ALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM60-NOFTZ-ALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-NOFTZ-ALLOW-NEXT: mov.b32 %r6, 65535;
+; SM60-NOFTZ-ALLOW-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-NOFTZ-ALLOW-NEXT: not.b32 %r2, %r7;
+; SM60-NOFTZ-ALLOW-NEXT: ld.volatile.global.b32 %r14, [%rd1];
+; SM60-NOFTZ-ALLOW-NEXT: $L__BB77_1: // %atomicrmw.start
+; SM60-NOFTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NOFTZ-ALLOW-NEXT: shr.u32 %r8, %r14, %r1;
+; SM60-NOFTZ-ALLOW-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM60-NOFTZ-ALLOW-NEXT: setp.eq.b16 %p1, %rs2, 0;
+; SM60-NOFTZ-ALLOW-NEXT: selp.b16 %rs3, %rs2, %rs1, %p1;
+; SM60-NOFTZ-ALLOW-NEXT: setp.gt.f16 %p2, %rs2, %rs1;
+; SM60-NOFTZ-ALLOW-NEXT: selp.b16 %rs4, %rs2, %rs1, %p2;
+; SM60-NOFTZ-ALLOW-NEXT: setp.nan.f16 %p3, %rs2, %rs1;
+; SM60-NOFTZ-ALLOW-NEXT: selp.b16 %rs5, 0x7E00, %rs4, %p3;
+; SM60-NOFTZ-ALLOW-NEXT: setp.eq.f16 %p4, %rs2, %rs1;
+; SM60-NOFTZ-ALLOW-NEXT: selp.b16 %rs6, %rs3, %rs5, %p4;
+; SM60-NOFTZ-ALLOW-NEXT: cvt.u32.u16 %r9, %rs6;
+; SM60-NOFTZ-ALLOW-NEXT: shl.b32 %r10, %r9, %r1;
+; SM60-NOFTZ-ALLOW-NEXT: and.b32 %r11, %r14, %r2;
+; SM60-NOFTZ-ALLOW-NEXT: or.b32 %r12, %r11, %r10;
+; SM60-NOFTZ-ALLOW-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM60-NOFTZ-ALLOW-NEXT: setp.ne.b32 %p5, %r3, %r14;
+; SM60-NOFTZ-ALLOW-NEXT: mov.b32 %r14, %r3;
+; SM60-NOFTZ-ALLOW-NEXT: @%p5 bra $L__BB77_1;
+; SM60-NOFTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NOFTZ-ALLOW-NEXT: shr.u32 %r13, %r3, %r1;
+; SM60-NOFTZ-ALLOW-NEXT: membar.cta;
+; SM60-NOFTZ-ALLOW-NEXT: st.param.b16 [func_retval0], %r13;
+; SM60-NOFTZ-ALLOW-NEXT: ret;
;
-; SM60-FTZ-LABEL: fmaximum_acq_rel_half_global_cta(
-; SM60-FTZ: {
-; SM60-FTZ-NEXT: .reg .pred %p<6>;
-; SM60-FTZ-NEXT: .reg .b16 %rs<7>;
-; SM60-FTZ-NEXT: .reg .b32 %r<15>;
-; SM60-FTZ-NEXT: .reg .b64 %rd<3>;
-; SM60-FTZ-EMPTY:
-; SM60-FTZ-NEXT: // %bb.0:
-; SM60-FTZ-NEXT: ld.param.b16 %rs1, [fmaximum_acq_rel_half_global_cta_param_1];
-; SM60-FTZ-NEXT: ld.param.b64 %rd2, [fmaximum_acq_rel_half_global_cta_param_0];
-; SM60-FTZ-NEXT: membar.cta;
-; SM60-FTZ-NEXT: and.b64 %rd1, %rd2, -4;
-; SM60-FTZ-NEXT: cvt.u32.u64 %r4, %rd2;
-; SM60-FTZ-NEXT: and.b32 %r5, %r4, 3;
-; SM60-FTZ-NEXT: shl.b32 %r1, %r5, 3;
-; SM60-FTZ-NEXT: mov.b32 %r6, 65535;
-; SM60-FTZ-NEXT: shl.b32 %r7, %r6, %r1;
-; SM60-FTZ-NEXT: not.b32 %r2, %r7;
-; SM60-FTZ-NEXT: ld.volatile.global.b32 %r14, [%rd1];
-; SM60-FTZ-NEXT: $L__BB77_1: // %atomicrmw.start
-; SM60-FTZ-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-FTZ-NEXT: shr.u32 %r8, %r14, %r1;
-; SM60-FTZ-NEXT: cvt.u16.u32 %rs2, %r8;
-; SM60-FTZ-NEXT: setp.eq.b16 %p1, %rs2, 0;
-; SM60-FTZ-NEXT: selp.b16 %rs3, %rs2, %rs1, %p1;
-; SM60-FTZ-NEXT: setp.gt.ftz.f16 %p2, %rs2, %rs1;
-; SM60-FTZ-NEXT: selp.b16 %rs4, %rs2, %rs1, %p2;
-; SM60-FTZ-NEXT: setp.nan.ftz.f16 %p3, %rs2, %rs1;
-; SM60-FTZ-NEXT: selp.b16 %rs5, 0x7E00, %rs4, %p3;
-; SM60-FTZ-NEXT: setp.eq.ftz.f16 %p4, %rs2, %rs1;
-; SM60-FTZ-NEXT: selp.b16 %rs6, %rs3, %rs5, %p4;
-; SM60-FTZ-NEXT: cvt.u32.u16 %r9, %rs6;
-; SM60-FTZ-NEXT: shl.b32 %r10, %r9, %r1;
-; SM60-FTZ-NEXT: and.b32 %r11, %r14, %r2;
-; SM60-FTZ-NEXT: or.b32 %r12, %r11, %r10;
-; SM60-FTZ-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
-; SM60-FTZ-NEXT: setp.ne.b32 %p5, %r3, %r14;
-; SM60-FTZ-NEXT: mov.b32 %r14, %r3;
-; SM60-FTZ-NEXT: @%p5 bra $L__BB77_1;
-; SM60-FTZ-NEXT: // %bb.2: // %atomicrmw.end
-; SM60-FTZ-NEXT: shr.u32 %r13, %r3, %r1;
-; SM60-FTZ-NEXT: membar.cta;
-; SM60-FTZ-NEXT: st.param.b16 [func_retval0], %r13;
-; SM60-FTZ-NEXT: ret;
+; SM60-FTZ-DISALLOW-LABEL: fmaximum_acq_rel_half_global_cta(
+; SM60-FTZ-DISALLOW: {
+; SM60-FTZ-DISALLOW-NEXT: .reg .pred %p<6>;
+; SM60-FTZ-DISALLOW-NEXT: .reg .b16 %rs<7>;
+; SM60-FTZ-DISALLOW-NEXT: .reg .b32 %r<15>;
+; SM60-FTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
+; SM60-FTZ-DISALLOW-EMPTY:
+; SM60-FTZ-DISALLOW-NEXT: // %bb.0:
+; SM60-FTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fmaximum_acq_rel_half_global_cta_param_1];
+; SM60-FTZ-DISALLOW-NEXT: ld.param.b64 %rd2, [fmaximum_acq_rel_half_global_cta_param_0];
+; SM60-FTZ-DISALLOW-NEXT: membar.cta;
+; SM60-FTZ-DISALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-FTZ-DISALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-FTZ-DISALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM60-FTZ-DISALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-FTZ-DISALLOW-NEXT: mov.b32 %r6, 65535;
+; SM60-FTZ-DISALLOW-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-FTZ-DISALLOW-NEXT: not.b32 %r2, %r7;
+; SM60-FTZ-DISALLOW-NEXT: ld.volatile.global.b32 %r14, [%rd1];
+; SM60-FTZ-DISALLOW-NEXT: $L__BB77_1: // %atomicrmw.start
+; SM60-FTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-FTZ-DISALLOW-NEXT: shr.u32 %r8, %r14, %r1;
+; SM60-FTZ-DISALLOW-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM60-FTZ-DISALLOW-NEXT: setp.eq.b16 %p1, %rs2, 0;
+; SM60-FTZ-DISALLOW-NEXT: selp.b16 %rs3, %rs2, %rs1, %p1;
+; SM60-FTZ-DISALLOW-NEXT: setp.gt.ftz.f16 %p2, %rs2, %rs1;
+; SM60-FTZ-DISALLOW-NEXT: selp.b16 %rs4, %rs2, %rs1, %p2;
+; SM60-FTZ-DISALLOW-NEXT: setp.nan.ftz.f16 %p3, %rs2, %rs1;
+; SM60-FTZ-DISALLOW-NEXT: selp.b16 %rs5, 0x7E00, %rs4, %p3;
+; SM60-FTZ-DISALLOW-NEXT: setp.eq.ftz.f16 %p4, %rs2, %rs1;
+; SM60-FTZ-DISALLOW-NEXT: selp.b16 %rs6, %rs3, %rs5, %p4;
+; SM60-FTZ-DISALLOW-NEXT: cvt.u32.u16 %r9, %rs6;
+; SM60-FTZ-DISALLOW-NEXT: shl.b32 %r10, %r9, %r1;
+; SM60-FTZ-DISALLOW-NEXT: and.b32 %r11, %r14, %r2;
+; SM60-FTZ-DISALLOW-NEXT: or.b32 %r12, %r11, %r10;
+; SM60-FTZ-DISALLOW-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM60-FTZ-DISALLOW-NEXT: setp.ne.b32 %p5, %r3, %r14;
+; SM60-FTZ-DISALLOW-NEXT: mov.b32 %r14, %r3;
+; SM60-FTZ-DISALLOW-NEXT: @%p5 bra $L__BB77_1;
+; SM60-FTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-FTZ-DISALLOW-NEXT: shr.u32 %r13, %r3, %r1;
+; SM60-FTZ-DISALLOW-NEXT: membar.cta;
+; SM60-FTZ-DISALLOW-NEXT: st.param.b16 [func_retval0], %r13;
+; SM60-FTZ-DISALLOW-NEXT: ret;
+;
+; SM60-FTZ-ALLOW-LABEL: fmaximum_acq_rel_half_global_cta(
+; SM60-FTZ-ALLOW: {
+; SM60-FTZ-ALLOW-NEXT: .reg .pred %p<6>;
+; SM60-FTZ-ALLOW-NEXT: .reg .b16 %rs<7>;
+; SM60-FTZ-ALLOW-NEXT: .reg .b32 %r<15>;
+; SM60-FTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
+; SM60-FTZ-ALLOW-EMPTY:
+; SM60-FTZ-ALLOW-NEXT: // %bb.0:
+; SM60-FTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fmaximum_acq_rel_half_global_cta_param_1];
+; SM60-FTZ-ALLOW-NEXT: ld.param.b64 %rd2, [fmaximum_acq_rel_half_global_cta_param_0];
+; SM60-FTZ-ALLOW-NEXT: membar.cta;
+; SM60-FTZ-ALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-FTZ-ALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-FTZ-ALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM60-FTZ-ALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-FTZ-ALLOW-NEXT: mov.b32 %r6, 65535;
+; SM60-FTZ-ALLOW-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-FTZ-ALLOW-NEXT: not.b32 %r2, %r7;
+; SM60-FTZ-ALLOW-NEXT: ld.volatile.global.b32 %r14, [%rd1];
+; SM60-FTZ-ALLOW-NEXT: $L__BB77_1: // %atomicrmw.start
+; SM60-FTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-FTZ-ALLOW-NEXT: shr.u32 %r8, %r14, %r1;
+; SM60-FTZ-ALLOW-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM60-FTZ-ALLOW-NEXT: setp.eq.b16 %p1, %rs2, 0;
+; SM60-FTZ-ALLOW-NEXT: selp.b16 %rs3, %rs2, %rs1, %p1;
+; SM60-FTZ-ALLOW-NEXT: setp.gt.ftz.f16 %p2, %rs2, %rs1;
+; SM60-FTZ-ALLOW-NEXT: selp.b16 %rs4, %rs2, %rs1, %p2;
+; SM60-FTZ-ALLOW-NEXT: setp.nan.ftz.f16 %p3, %rs2, %rs1;
+; SM60-FTZ-ALLOW-NEXT: selp.b16 %rs5, 0x7E00, %rs4, %p3;
+; SM60-FTZ-ALLOW-NEXT: setp.eq.ftz.f16 %p4, %rs2, %rs1;
+; SM60-FTZ-ALLOW-NEXT: selp.b16 %rs6, %rs3, %rs5, %p4;
+; SM60-FTZ-ALLOW-NEXT: cvt.u32.u16 %r9, %rs6;
+; SM60-FTZ-ALLOW-NEXT: shl.b32 %r10, %r9, %r1;
+; SM60-FTZ-ALLOW-NEXT: and.b32 %r11, %r14, %r2;
+; SM60-FTZ-ALLOW-NEXT: or.b32 %r12, %r11, %r10;
+; SM60-FTZ-ALLOW-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM60-FTZ-ALLOW-NEXT: setp.ne.b32 %p5, %r3, %r14;
+; SM60-FTZ-ALLOW-NEXT: mov.b32 %r14, %r3;
+; SM60-FTZ-ALLOW-NEXT: @%p5 bra $L__BB77_1;
+; SM60-FTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-FTZ-ALLOW-NEXT: shr.u32 %r13, %r3, %r1;
+; SM60-FTZ-ALLOW-NEXT: membar.cta;
+; SM60-FTZ-ALLOW-NEXT: st.param.b16 [func_retval0], %r13;
+; SM60-FTZ-ALLOW-NEXT: ret;
%retval = atomicrmw fmaximum ptr addrspace(1) %addr, half %val syncscope("block") acq_rel
ret half %retval
}
define bfloat @fadd_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat %val) {
-; SM60-NOFTZ-LABEL: fadd_acq_rel_bfloat_global_cta(
-; SM60-NOFTZ: {
-; SM60-NOFTZ-NEXT: .reg .pred %p<3>;
-; SM60-NOFTZ-NEXT: .reg .b16 %rs<2>;
-; SM60-NOFTZ-NEXT: .reg .b32 %r<24>;
-; SM60-NOFTZ-NEXT: .reg .b64 %rd<3>;
-; SM60-NOFTZ-EMPTY:
-; SM60-NOFTZ-NEXT: // %bb.0:
-; SM60-NOFTZ-NEXT: ld.param.b16 %rs1, [fadd_acq_rel_bfloat_global_cta_param_1];
-; SM60-NOFTZ-NEXT: ld.param.b64 %rd2, [fadd_acq_rel_bfloat_global_cta_param_0];
-; SM60-NOFTZ-NEXT: membar.cta;
-; SM60-NOFTZ-NEXT: and.b64 %rd1, %rd2, -4;
-; SM60-NOFTZ-NEXT: cvt.u32.u64 %r4, %rd2;
-; SM60-NOFTZ-NEXT: and.b32 %r5, %r4, 3;
-; SM60-NOFTZ-NEXT: shl.b32 %r1, %r5, 3;
-; SM60-NOFTZ-NEXT: mov.b32 %r6, 65535;
-; SM60-NOFTZ-NEXT: shl.b32 %r7, %r6, %r1;
-; SM60-NOFTZ-NEXT: not.b32 %r2, %r7;
-; SM60-NOFTZ-NEXT: ld.volatile.global.b32 %r23, [%rd1];
-; SM60-NOFTZ-NEXT: cvt.u32.u16 %r10, %rs1;
-; SM60-NOFTZ-NEXT: shl.b32 %r11, %r10, 16;
-; SM60-NOFTZ-NEXT: $L__BB78_1: // %atomicrmw.start
-; SM60-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NOFTZ-NEXT: shr.u32 %r8, %r23, %r1;
-; SM60-NOFTZ-NEXT: shl.b32 %r9, %r8, 16;
-; SM60-NOFTZ-NEXT: add.rn.f32 %r12, %r9, %r11;
-; SM60-NOFTZ-NEXT: bfe.u32 %r13, %r12, 16, 1;
-; SM60-NOFTZ-NEXT: add.s32 %r14, %r13, %r12;
-; SM60-NOFTZ-NEXT: add.s32 %r15, %r14, 32767;
-; SM60-NOFTZ-NEXT: setp.nan.f32 %p1, %r12, %r12;
-; SM60-NOFTZ-NEXT: or.b32 %r16, %r12, 4194304;
-; SM60-NOFTZ-NEXT: selp.b32 %r17, %r16, %r15, %p1;
-; SM60-NOFTZ-NEXT: shr.u32 %r18, %r17, 16;
-; SM60-NOFTZ-NEXT: shl.b32 %r19, %r18, %r1;
-; SM60-NOFTZ-NEXT: and.b32 %r20, %r23, %r2;
-; SM60-NOFTZ-NEXT: or.b32 %r21, %r20, %r19;
-; SM60-NOFTZ-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r23, %r21;
-; SM60-NOFTZ-NEXT: setp.ne.b32 %p2, %r3, %r23;
-; SM60-NOFTZ-NEXT: mov.b32 %r23, %r3;
-; SM60-NOFTZ-NEXT: @%p2 bra $L__BB78_1;
-; SM60-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end
-; SM60-NOFTZ-NEXT: shr.u32 %r22, %r3, %r1;
-; SM60-NOFTZ-NEXT: membar.cta;
-; SM60-NOFTZ-NEXT: st.param.b16 [func_retval0], %r22;
-; SM60-NOFTZ-NEXT: ret;
+; SM60-NOFTZ-DISALLOW-LABEL: fadd_acq_rel_bfloat_global_cta(
+; SM60-NOFTZ-DISALLOW: {
+; SM60-NOFTZ-DISALLOW-NEXT: .reg .pred %p<3>;
+; SM60-NOFTZ-DISALLOW-NEXT: .reg .b16 %rs<2>;
+; SM60-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<24>;
+; SM60-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
+; SM60-NOFTZ-DISALLOW-EMPTY:
+; SM60-NOFTZ-DISALLOW-NEXT: // %bb.0:
+; SM60-NOFTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fadd_acq_rel_bfloat_global_cta_param_1];
+; SM60-NOFTZ-DISALLOW-NEXT: ld.param.b64 %rd2, [fadd_acq_rel_bfloat_global_cta_param_0];
+; SM60-NOFTZ-DISALLOW-NEXT: membar.cta;
+; SM60-NOFTZ-DISALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-NOFTZ-DISALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM60-NOFTZ-DISALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-NOFTZ-DISALLOW-NEXT: mov.b32 %r6, 65535;
+; SM60-NOFTZ-DISALLOW-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-NOFTZ-DISALLOW-NEXT: not.b32 %r2, %r7;
+; SM60-NOFTZ-DISALLOW-NEXT: ld.volatile.global.b32 %r23, [%rd1];
+; SM60-NOFTZ-DISALLOW-NEXT: cvt.u32.u16 %r10, %rs1;
+; SM60-NOFTZ-DISALLOW-NEXT: shl.b32 %r11, %r10, 16;
+; SM60-NOFTZ-DISALLOW-NEXT: $L__BB78_1: // %atomicrmw.start
+; SM60-NOFTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NOFTZ-DISALLOW-NEXT: shr.u32 %r8, %r23, %r1;
+; SM60-NOFTZ-DISALLOW-NEXT: shl.b32 %r9, %r8, 16;
+; SM60-NOFTZ-DISALLOW-NEXT: add.rn.f32 %r12, %r9, %r11;
+; SM60-NOFTZ-DISALLOW-NEXT: bfe.u32 %r13, %r12, 16, 1;
+; SM60-NOFTZ-DISALLOW-NEXT: add.s32 %r14, %r13, %r12;
+; SM60-NOFTZ-DISALLOW-NEXT: add.s32 %r15, %r14, 32767;
+; SM60-NOFTZ-DISALLOW-NEXT: setp.nan.f32 %p1, %r12, %r12;
+; SM60-NOFTZ-DISALLOW-NEXT: or.b32 %r16, %r12, 4194304;
+; SM60-NOFTZ-DISALLOW-NEXT: selp.b32 %r17, %r16, %r15, %p1;
+; SM60-NOFTZ-DISALLOW-NEXT: shr.u32 %r18, %r17, 16;
+; SM60-NOFTZ-DISALLOW-NEXT: shl.b32 %r19, %r18, %r1;
+; SM60-NOFTZ-DISALLOW-NEXT: and.b32 %r20, %r23, %r2;
+; SM60-NOFTZ-DISALLOW-NEXT: or.b32 %r21, %r20, %r19;
+; SM60-NOFTZ-DISALLOW-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r23, %r21;
+; SM60-NOFTZ-DISALLOW-NEXT: setp.ne.b32 %p2, %r3, %r23;
+; SM60-NOFTZ-DISALLOW-NEXT: mov.b32 %r23, %r3;
+; SM60-NOFTZ-DISALLOW-NEXT: @%p2 bra $L__BB78_1;
+; SM60-NOFTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NOFTZ-DISALLOW-NEXT: shr.u32 %r22, %r3, %r1;
+; SM60-NOFTZ-DISALLOW-NEXT: membar.cta;
+; SM60-NOFTZ-DISALLOW-NEXT: st.param.b16 [func_retval0], %r22;
+; SM60-NOFTZ-DISALLOW-NEXT: ret;
+;
+; SM60-NOFTZ-ALLOW-LABEL: fadd_acq_rel_bfloat_global_cta(
+; SM60-NOFTZ-ALLOW: {
+; SM60-NOFTZ-ALLOW-NEXT: .reg .pred %p<3>;
+; SM60-NOFTZ-ALLOW-NEXT: .reg .b16 %rs<2>;
+; SM60-NOFTZ-ALLOW-NEXT: .reg .b32 %r<24>;
+; SM60-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
+; SM60-NOFTZ-ALLOW-EMPTY:
+; SM60-NOFTZ-ALLOW-NEXT: // %bb.0:
+; SM60-NOFTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fadd_acq_rel_bfloat_global_cta_param_1];
+; SM60-NOFTZ-ALLOW-NEXT: ld.param.b64 %rd2, [fadd_acq_rel_bfloat_global_cta_param_0];
+; SM60-NOFTZ-ALLOW-NEXT: membar.cta;
+; SM60-NOFTZ-ALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NOFTZ-ALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-NOFTZ-ALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM60-NOFTZ-ALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-NOFTZ-ALLOW-NEXT: mov.b32 %r6, 65535;
+; SM60-NOFTZ-ALLOW-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-NOFTZ-ALLOW-NEXT: not.b32 %r2, %r7;
+; SM60-NOFTZ-ALLOW-NEXT: ld.volatile.global.b32 %r23, [%rd1];
+; SM60-NOFTZ-ALLOW-NEXT: cvt.u32.u16 %r10, %rs1;
+; SM60-NOFTZ-ALLOW-NEXT: shl.b32 %r11, %r10, 16;
+; SM60-NOFTZ-ALLOW-NEXT: $L__BB78_1: // %atomicrmw.start
+; SM60-NOFTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NOFTZ-ALLOW-NEXT: shr.u32 %r8, %r23, %r1;
+; SM60-NOFTZ-ALLOW-NEXT: shl.b32 %r9, %r8, 16;
+; SM60-NOFTZ-ALLOW-NEXT: add.rn.f32 %r12, %r9, %r11;
+; SM60-NOFTZ-ALLOW-NEXT: bfe.u32 %r13, %r12, 16, 1;
+; SM60-NOFTZ-ALLOW-NEXT: add.s32 %r14, %r13, %r12;
+; SM60-NOFTZ-ALLOW-NEXT: add.s32 %r15, %r14, 32767;
+; SM60-NOFTZ-ALLOW-NEXT: setp.nan.f32 %p1, %r12, %r12;
+; SM60-NOFTZ-ALLOW-NEXT: or.b32 %r16, %r12, 4194304;
+; SM60-NOFTZ-ALLOW-NEXT: selp.b32 %r17, %r16, %r15, %p1;
+; SM60-NOFTZ-ALLOW-NEXT: shr.u32 %r18, %r17, 16;
+; SM60-NOFTZ-ALLOW-NEXT: shl.b32 %r19, %r18, %r1;
+; SM60-NOFTZ-ALLOW-NEXT: and.b32 %r20, %r23, %r2;
+; SM60-NOFTZ-ALLOW-NEXT: or.b32 %r21, %r20, %r19;
+; SM60-NOFTZ-ALLOW-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r23, %r21;
+; SM60-NOFTZ-ALLOW-NEXT: setp.ne.b32 %p2, %r3, %r23;
+; SM60-NOFTZ-ALLOW-NEXT: mov.b32 %r23, %r3;
+; SM60-NOFTZ-ALLOW-NEXT: @%p2 bra $L__BB78_1;
+; SM60-NOFTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NOFTZ-ALLOW-NEXT: shr.u32 %r22, %r3, %r1;
+; SM60-NOFTZ-ALLOW-NEXT: membar.cta;
+; SM60-NOFTZ-ALLOW-NEXT: st.param.b16 [func_retval0], %r22;
+; SM60-NOFTZ-ALLOW-NEXT: ret;
+;
+; SM60-FTZ-DISALLOW-LABEL: fadd_acq_rel_bfloat_global_cta(
+; SM60-FTZ-DISALLOW: {
+; SM60-FTZ-DISALLOW-NEXT: .reg .pred %p<3>;
+; SM60-FTZ-DISALLOW-NEXT: .reg .b16 %rs<2>;
+; SM60-FTZ-DISALLOW-NEXT: .reg .b32 %r<24>;
+; SM60-FTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
+; SM60-FTZ-DISALLOW-EMPTY:
+; SM60-FTZ-DISALLOW-NEXT: // %bb.0:
+; SM60-FTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fadd_acq_rel_bfloat_global_cta_param_1];
+; SM60-FTZ-DISALLOW-NEXT: ld.param.b64 %rd2, [fadd_acq_rel_bfloat_global_cta_param_0];
+; SM60-FTZ-DISALLOW-NEXT: membar.cta;
+; SM60-FTZ-DISALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-FTZ-DISALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-FTZ-DISALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM60-FTZ-DISALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-FTZ-DISALLOW-NEXT: mov.b32 %r6, 65535;
+; SM60-FTZ-DISALLOW-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-FTZ-DISALLOW-NEXT: not.b32 %r2, %r7;
+; SM60-FTZ-DISALLOW-NEXT: ld.volatile.global.b32 %r23, [%rd1];
+; SM60-FTZ-DISALLOW-NEXT: cvt.u32.u16 %r10, %rs1;
+; SM60-FTZ-DISALLOW-NEXT: shl.b32 %r11, %r10, 16;
+; SM60-FTZ-DISALLOW-NEXT: $L__BB78_1: // %atomicrmw.start
+; SM60-FTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-FTZ-DISALLOW-NEXT: shr.u32 %r8, %r23, %r1;
+; SM60-FTZ-DISALLOW-NEXT: shl.b32 %r9, %r8, 16;
+; SM60-FTZ-DISALLOW-NEXT: add.rn.ftz.f32 %r12, %r9, %r11;
+; SM60-FTZ-DISALLOW-NEXT: bfe.u32 %r13, %r12, 16, 1;
+; SM60-FTZ-DISALLOW-NEXT: add.s32 %r14, %r13, %r12;
+; SM60-FTZ-DISALLOW-NEXT: add.s32 %r15, %r14, 32767;
+; SM60-FTZ-DISALLOW-NEXT: setp.nan.ftz.f32 %p1, %r12, %r12;
+; SM60-FTZ-DISALLOW-NEXT: or.b32 %r16, %r12, 4194304;
+; SM60-FTZ-DISALLOW-NEXT: selp.b32 %r17, %r16, %r15, %p1;
+; SM60-FTZ-DISALLOW-NEXT: shr.u32 %r18, %r17, 16;
+; SM60-FTZ-DISALLOW-NEXT: shl.b32 %r19, %r18, %r1;
+; SM60-FTZ-DISALLOW-NEXT: and.b32 %r20, %r23, %r2;
+; SM60-FTZ-DISALLOW-NEXT: or.b32 %r21, %r20, %r19;
+; SM60-FTZ-DISALLOW-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r23, %r21;
+; SM60-FTZ-DISALLOW-NEXT: setp.ne.b32 %p2, %r3, %r23;
+; SM60-FTZ-DISALLOW-NEXT: mov.b32 %r23, %r3;
+; SM60-FTZ-DISALLOW-NEXT: @%p2 bra $L__BB78_1;
+; SM60-FTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-FTZ-DISALLOW-NEXT: shr.u32 %r22, %r3, %r1;
+; SM60-FTZ-DISALLOW-NEXT: membar.cta;
+; SM60-FTZ-DISALLOW-NEXT: st.param.b16 [func_retval0], %r22;
+; SM60-FTZ-DISALLOW-NEXT: ret;
;
-; SM60-FTZ-LABEL: fadd_acq_rel_bfloat_global_cta(
-; SM60-FTZ: {
-; SM60-FTZ-NEXT: .reg .pred %p<3>;
-; SM60-FTZ-NEXT: .reg .b16 %rs<2>;
-; SM60-FTZ-NEXT: .reg .b32 %r<24>;
-; SM60-FTZ-NEXT: .reg .b64 %rd<3>;
-; SM60-FTZ-EMPTY:
-; SM60-FTZ-NEXT: // %bb.0:
-; SM60-FTZ-NEXT: ld.param.b16 %rs1, [fadd_acq_rel_bfloat_global_cta_param_1];
-; SM60-FTZ-NEXT: ld.param.b64 %rd2, [fadd_acq_rel_bfloat_global_cta_param_0];
-; SM60-FTZ-NEXT: membar.cta;
-; SM60-FTZ-NEXT: and.b64 %rd1, %rd2, -4;
-; SM60-FTZ-NEXT: cvt.u32.u64 %r4, %rd2;
-; SM60-FTZ-NEXT: and.b32 %r5, %r4, 3;
-; SM60-FTZ-NEXT: shl.b32 %r1, %r5, 3;
-; SM60-FTZ-NEXT: mov.b32 %r6, 65535;
-; SM60-FTZ-NEXT: shl.b32 %r7, %r6, %r1;
-; SM60-FTZ-NEXT: not.b32 %r2, %r7;
-; SM60-FTZ-NEXT: ld.volatile.global.b32 %r23, [%rd1];
-; SM60-FTZ-NEXT: cvt.u32.u16 %r10, %rs1;
-; SM60-FTZ-NEXT: shl.b32 %r11, %r10, 16;
-; SM60-FTZ-NEXT: $L__BB78_1: // %atomicrmw.start
-; SM60-FTZ-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-FTZ-NEXT: shr.u32 %r8, %r23, %r1;
-; SM60-FTZ-NEXT: shl.b32 %r9, %r8, 16;
-; SM60-FTZ-NEXT: add.rn.ftz.f32 %r12, %r9, %r11;
-; SM60-FTZ-NEXT: bfe.u32 %r13, %r12, 16, 1;
-; SM60-FTZ-NEXT: add.s32 %r14, %r13, %r12;
-; SM60-FTZ-NEXT: add.s32 %r15, %r14, 32767;
-; SM60-FTZ-NEXT: setp.nan.ftz.f32 %p1, %r12, %r12;
-; SM60-FTZ-NEXT: or.b32 %r16, %r12, 4194304;
-; SM60-FTZ-NEXT: selp.b32 %r17, %r16, %r15, %p1;
-; SM60-FTZ-NEXT: shr.u32 %r18, %r17, 16;
-; SM60-FTZ-NEXT: shl.b32 %r19, %r18, %r1;
-; SM60-FTZ-NEXT: and.b32 %r20, %r23, %r2;
-; SM60-FTZ-NEXT: or.b32 %r21, %r20, %r19;
-; SM60-FTZ-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r23, %r21;
-; SM60-FTZ-NEXT: setp.ne.b32 %p2, %r3, %r23;
-; SM60-FTZ-NEXT: mov.b32 %r23, %r3;
-; SM60-FTZ-NEXT: @%p2 bra $L__BB78_1;
-; SM60-FTZ-NEXT: // %bb.2: // %atomicrmw.end
-; SM60-FTZ-NEXT: shr.u32 %r22, %r3, %r1;
-; SM60-FTZ-NEXT: membar.cta;
-; SM60-FTZ-NEXT: st.param.b16 [func_retval0], %r22;
-; SM60-FTZ-NEXT: ret;
+; SM60-FTZ-ALLOW-LABEL: fadd_acq_rel_bfloat_global_cta(
+; SM60-FTZ-ALLOW: {
+; SM60-FTZ-ALLOW-NEXT: .reg .pred %p<3>;
+; SM60-FTZ-ALLOW-NEXT: .reg .b16 %rs<2>;
+; SM60-FTZ-ALLOW-NEXT: .reg .b32 %r<24>;
+; SM60-FTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
+; SM60-FTZ-ALLOW-EMPTY:
+; SM60-FTZ-ALLOW-NEXT: // %bb.0:
+; SM60-FTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fadd_acq_rel_bfloat_global_cta_param_1];
+; SM60-FTZ-ALLOW-NEXT: ld.param.b64 %rd2, [fadd_acq_rel_bfloat_global_cta_param_0];
+; SM60-FTZ-ALLOW-NEXT: membar.cta;
+; SM60-FTZ-ALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-FTZ-ALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-FTZ-ALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM60-FTZ-ALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-FTZ-ALLOW-NEXT: mov.b32 %r6, 65535;
+; SM60-FTZ-ALLOW-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-FTZ-ALLOW-NEXT: not.b32 %r2, %r7;
+; SM60-FTZ-ALLOW-NEXT: ld.volatile.global.b32 %r23, [%rd1];
+; SM60-FTZ-ALLOW-NEXT: cvt.u32.u16 %r10, %rs1;
+; SM60-FTZ-ALLOW-NEXT: shl.b32 %r11, %r10, 16;
+; SM60-FTZ-ALLOW-NEXT: $L__BB78_1: // %atomicrmw.start
+; SM60-FTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-FTZ-ALLOW-NEXT: shr.u32 %r8, %r23, %r1;
+; SM60-FTZ-ALLOW-NEXT: shl.b32 %r9, %r8, 16;
+; SM60-FTZ-ALLOW-NEXT: add.rn.ftz.f32 %r12, %r9, %r11;
+; SM60-FTZ-ALLOW-NEXT: bfe.u32 %r13, %r12, 16, 1;
+; SM60-FTZ-ALLOW-NEXT: add.s32 %r14, %r13, %r12;
+; SM60-FTZ-ALLOW-NEXT: add.s32 %r15, %r14, 32767;
+; SM60-FTZ-ALLOW-NEXT: setp.nan.ftz.f32 %p1, %r12, %r12;
+; SM60-FTZ-ALLOW-NEXT: or.b32 %r16, %r12, 4194304;
+; SM60-FTZ-ALLOW-NEXT: selp.b32 %r17, %r16, %r15, %p1;
+; SM60-FTZ-ALLOW-NEXT: shr.u32 %r18, %r17, 16;
+; SM60-FTZ-ALLOW-NEXT: shl.b32 %r19, %r18, %r1;
+; SM60-FTZ-ALLOW-NEXT: and.b32 %r20, %r23, %r2;
+; SM60-FTZ-ALLOW-NEXT: or.b32 %r21, %r20, %r19;
+; SM60-FTZ-ALLOW-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r23, %r21;
+; SM60-FTZ-ALLOW-NEXT: setp.ne.b32 %p2, %r3, %r23;
+; SM60-FTZ-ALLOW-NEXT: mov.b32 %r23, %r3;
+; SM60-FTZ-ALLOW-NEXT: @%p2 bra $L__BB78_1;
+; SM60-FTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-FTZ-ALLOW-NEXT: shr.u32 %r22, %r3, %r1;
+; SM60-FTZ-ALLOW-NEXT: membar.cta;
+; SM60-FTZ-ALLOW-NEXT: st.param.b16 [func_retval0], %r22;
+; SM60-FTZ-ALLOW-NEXT: ret;
%retval = atomicrmw fadd ptr addrspace(1) %addr, bfloat %val syncscope("block") acq_rel
ret bfloat %retval
}
define bfloat @fsub_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat %val) {
-; SM60-NOFTZ-LABEL: fsub_acq_rel_bfloat_global_cta(
-; SM60-NOFTZ: {
-; SM60-NOFTZ-NEXT: .reg .pred %p<3>;
-; SM60-NOFTZ-NEXT: .reg .b16 %rs<2>;
-; SM60-NOFTZ-NEXT: .reg .b32 %r<24>;
-; SM60-NOFTZ-NEXT: .reg .b64 %rd<3>;
-; SM60-NOFTZ-EMPTY:
-; SM60-NOFTZ-NEXT: // %bb.0:
-; SM60-NOFTZ-NEXT: ld.param.b16 %rs1, [fsub_acq_rel_bfloat_global_cta_param_1];
-; SM60-NOFTZ-NEXT: ld.param.b64 %rd2, [fsub_acq_rel_bfloat_global_cta_param_0];
-; SM60-NOFTZ-NEXT: membar.cta;
-; SM60-NOFTZ-NEXT: and.b64 %rd1, %rd2, -4;
-; SM60-NOFTZ-NEXT: cvt.u32.u64 %r4, %rd2;
-; SM60-NOFTZ-NEXT: and.b32 %r5, %r4, 3;
-; SM60-NOFTZ-NEXT: shl.b32 %r1, %r5, 3;
-; SM60-NOFTZ-NEXT: mov.b32 %r6, 65535;
-; SM60-NOFTZ-NEXT: shl.b32 %r7, %r6, %r1;
-; SM60-NOFTZ-NEXT: not.b32 %r2, %r7;
-; SM60-NOFTZ-NEXT: ld.volatile.global.b32 %r23, [%rd1];
-; SM60-NOFTZ-NEXT: cvt.u32.u16 %r10, %rs1;
-; SM60-NOFTZ-NEXT: shl.b32 %r11, %r10, 16;
-; SM60-NOFTZ-NEXT: $L__BB79_1: // %atomicrmw.start
-; SM60-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NOFTZ-NEXT: shr.u32 %r8, %r23, %r1;
-; SM60-NOFTZ-NEXT: shl.b32 %r9, %r8, 16;
-; SM60-NOFTZ-NEXT: sub.rn.f32 %r12, %r9, %r11;
-; SM60-NOFTZ-NEXT: bfe.u32 %r13, %r12, 16, 1;
-; SM60-NOFTZ-NEXT: add.s32 %r14, %r13, %r12;
-; SM60-NOFTZ-NEXT: add.s32 %r15, %r14, 32767;
-; SM60-NOFTZ-NEXT: setp.nan.f32 %p1, %r12, %r12;
-; SM60-NOFTZ-NEXT: or.b32 %r16, %r12, 4194304;
-; SM60-NOFTZ-NEXT: selp.b32 %r17, %r16, %r15, %p1;
-; SM60-NOFTZ-NEXT: shr.u32 %r18, %r17, 16;
-; SM60-NOFTZ-NEXT: shl.b32 %r19, %r18, %r1;
-; SM60-NOFTZ-NEXT: and.b32 %r20, %r23, %r2;
-; SM60-NOFTZ-NEXT: or.b32 %r21, %r20, %r19;
-; SM60-NOFTZ-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r23, %r21;
-; SM60-NOFTZ-NEXT: setp.ne.b32 %p2, %r3, %r23;
-; SM60-NOFTZ-NEXT: mov.b32 %r23, %r3;
-; SM60-NOFTZ-NEXT: @%p2 bra $L__BB79_1;
-; SM60-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end
-; SM60-NOFTZ-NEXT: shr.u32 %r22, %r3, %r1;
-; SM60-NOFTZ-NEXT: membar.cta;
-; SM60-NOFTZ-NEXT: st.param.b16 [func_retval0], %r22;
-; SM60-NOFTZ-NEXT: ret;
+; SM60-NOFTZ-DISALLOW-LABEL: fsub_acq_rel_bfloat_global_cta(
+; SM60-NOFTZ-DISALLOW: {
+; SM60-NOFTZ-DISALLOW-NEXT: .reg .pred %p<3>;
+; SM60-NOFTZ-DISALLOW-NEXT: .reg .b16 %rs<2>;
+; SM60-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<24>;
+; SM60-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
+; SM60-NOFTZ-DISALLOW-EMPTY:
+; SM60-NOFTZ-DISALLOW-NEXT: // %bb.0:
+; SM60-NOFTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fsub_acq_rel_bfloat_global_cta_param_1];
+; SM60-NOFTZ-DISALLOW-NEXT: ld.param.b64 %rd2, [fsub_acq_rel_bfloat_global_cta_param_0];
+; SM60-NOFTZ-DISALLOW-NEXT: membar.cta;
+; SM60-NOFTZ-DISALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-NOFTZ-DISALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM60-NOFTZ-DISALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-NOFTZ-DISALLOW-NEXT: mov.b32 %r6, 65535;
+; SM60-NOFTZ-DISALLOW-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-NOFTZ-DISALLOW-NEXT: not.b32 %r2, %r7;
+; SM60-NOFTZ-DISALLOW-NEXT: ld.volatile.global.b32 %r23, [%rd1];
+; SM60-NOFTZ-DISALLOW-NEXT: cvt.u32.u16 %r10, %rs1;
+; SM60-NOFTZ-DISALLOW-NEXT: shl.b32 %r11, %r10, 16;
+; SM60-NOFTZ-DISALLOW-NEXT: $L__BB79_1: // %atomicrmw.start
+; SM60-NOFTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NOFTZ-DISALLOW-NEXT: shr.u32 %r8, %r23, %r1;
+; SM60-NOFTZ-DISALLOW-NEXT: shl.b32 %r9, %r8, 16;
+; SM60-NOFTZ-DISALLOW-NEXT: sub.rn.f32 %r12, %r9, %r11;
+; SM60-NOFTZ-DISALLOW-NEXT: bfe.u32 %r13, %r12, 16, 1;
+; SM60-NOFTZ-DISALLOW-NEXT: add.s32 %r14, %r13, %r12;
+; SM60-NOFTZ-DISALLOW-NEXT: add.s32 %r15, %r14, 32767;
+; SM60-NOFTZ-DISALLOW-NEXT: setp.nan.f32 %p1, %r12, %r12;
+; SM60-NOFTZ-DISALLOW-NEXT: or.b32 %r16, %r12, 4194304;
+; SM60-NOFTZ-DISALLOW-NEXT: selp.b32 %r17, %r16, %r15, %p1;
+; SM60-NOFTZ-DISALLOW-NEXT: shr.u32 %r18, %r17, 16;
+; SM60-NOFTZ-DISALLOW-NEXT: shl.b32 %r19, %r18, %r1;
+; SM60-NOFTZ-DISALLOW-NEXT: and.b32 %r20, %r23, %r2;
+; SM60-NOFTZ-DISALLOW-NEXT: or.b32 %r21, %r20, %r19;
+; SM60-NOFTZ-DISALLOW-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r23, %r21;
+; SM60-NOFTZ-DISALLOW-NEXT: setp.ne.b32 %p2, %r3, %r23;
+; SM60-NOFTZ-DISALLOW-NEXT: mov.b32 %r23, %r3;
+; SM60-NOFTZ-DISALLOW-NEXT: @%p2 bra $L__BB79_1;
+; SM60-NOFTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NOFTZ-DISALLOW-NEXT: shr.u32 %r22, %r3, %r1;
+; SM60-NOFTZ-DISALLOW-NEXT: membar.cta;
+; SM60-NOFTZ-DISALLOW-NEXT: st.param.b16 [func_retval0], %r22;
+; SM60-NOFTZ-DISALLOW-NEXT: ret;
;
-; SM60-FTZ-LABEL: fsub_acq_rel_bfloat_global_cta(
-; SM60-FTZ: {
-; SM60-FTZ-NEXT: .reg .pred %p<3>;
-; SM60-FTZ-NEXT: .reg .b16 %rs<2>;
-; SM60-FTZ-NEXT: .reg .b32 %r<24>;
-; SM60-FTZ-NEXT: .reg .b64 %rd<3>;
-; SM60-FTZ-EMPTY:
-; SM60-FTZ-NEXT: // %bb.0:
-; SM60-FTZ-NEXT: ld.param.b16 %rs1, [fsub_acq_rel_bfloat_global_cta_param_1];
-; SM60-FTZ-NEXT: ld.param.b64 %rd2, [fsub_acq_rel_bfloat_global_cta_param_0];
-; SM60-FTZ-NEXT: membar.cta;
-; SM60-FTZ-NEXT: and.b64 %rd1, %rd2, -4;
-; SM60-FTZ-NEXT: cvt.u32.u64 %r4, %rd2;
-; SM60-FTZ-NEXT: and.b32 %r5, %r4, 3;
-; SM60-FTZ-NEXT: shl.b32 %r1, %r5, 3;
-; SM60-FTZ-NEXT: mov.b32 %r6, 65535;
-; SM60-FTZ-NEXT: shl.b32 %r7, %r6, %r1;
-; SM60-FTZ-NEXT: not.b32 %r2, %r7;
-; SM60-FTZ-NEXT: ld.volatile.global.b32 %r23, [%rd1];
-; SM60-FTZ-NEXT: cvt.u32.u16 %r10, %rs1;
-; SM60-FTZ-NEXT: shl.b32 %r11, %r10, 16;
-; SM60-FTZ-NEXT: $L__BB79_1: // %atomicrmw.start
-; SM60-FTZ-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-FTZ-NEXT: shr.u32 %r8, %r23, %r1;
-; SM60-FTZ-NEXT: shl.b32 %r9, %r8, 16;
-; SM60-FTZ-NEXT: sub.rn.ftz.f32 %r12, %r9, %r11;
-; SM60-FTZ-NEXT: bfe.u32 %r13, %r12, 16, 1;
-; SM60-FTZ-NEXT: add.s32 %r14, %r13, %r12;
-; SM60-FTZ-NEXT: add.s32 %r15, %r14, 32767;
-; SM60-FTZ-NEXT: setp.nan.ftz.f32 %p1, %r12, %r12;
-; SM60-FTZ-NEXT: or.b32 %r16, %r12, 4194304;
-; SM60-FTZ-NEXT: selp.b32 %r17, %r16, %r15, %p1;
-; SM60-FTZ-NEXT: shr.u32 %r18, %r17, 16;
-; SM60-FTZ-NEXT: shl.b32 %r19, %r18, %r1;
-; SM60-FTZ-NEXT: and.b32 %r20, %r23, %r2;
-; SM60-FTZ-NEXT: or.b32 %r21, %r20, %r19;
-; SM60-FTZ-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r23, %r21;
-; SM60-FTZ-NEXT: setp.ne.b32 %p2, %r3, %r23;
-; SM60-FTZ-NEXT: mov.b32 %r23, %r3;
-; SM60-FTZ-NEXT: @%p2 bra $L__BB79_1;
-; SM60-FTZ-NEXT: // %bb.2: // %atomicrmw.end
-; SM60-FTZ-NEXT: shr.u32 %r22, %r3, %r1;
-; SM60-FTZ-NEXT: membar.cta;
-; SM60-FTZ-NEXT: st.param.b16 [func_retval0], %r22;
-; SM60-FTZ-NEXT: ret;
+; SM60-NOFTZ-ALLOW-LABEL: fsub_acq_rel_bfloat_global_cta(
+; SM60-NOFTZ-ALLOW: {
+; SM60-NOFTZ-ALLOW-NEXT: .reg .pred %p<3>;
+; SM60-NOFTZ-ALLOW-NEXT: .reg .b16 %rs<2>;
+; SM60-NOFTZ-ALLOW-NEXT: .reg .b32 %r<24>;
+; SM60-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
+; SM60-NOFTZ-ALLOW-EMPTY:
+; SM60-NOFTZ-ALLOW-NEXT: // %bb.0:
+; SM60-NOFTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fsub_acq_rel_bfloat_global_cta_param_1];
+; SM60-NOFTZ-ALLOW-NEXT: ld.param.b64 %rd2, [fsub_acq_rel_bfloat_global_cta_param_0];
+; SM60-NOFTZ-ALLOW-NEXT: membar.cta;
+; SM60-NOFTZ-ALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NOFTZ-ALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-NOFTZ-ALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM60-NOFTZ-ALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-NOFTZ-ALLOW-NEXT: mov.b32 %r6, 65535;
+; SM60-NOFTZ-ALLOW-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-NOFTZ-ALLOW-NEXT: not.b32 %r2, %r7;
+; SM60-NOFTZ-ALLOW-NEXT: ld.volatile.global.b32 %r23, [%rd1];
+; SM60-NOFTZ-ALLOW-NEXT: cvt.u32.u16 %r10, %rs1;
+; SM60-NOFTZ-ALLOW-NEXT: shl.b32 %r11, %r10, 16;
+; SM60-NOFTZ-ALLOW-NEXT: $L__BB79_1: // %atomicrmw.start
+; SM60-NOFTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NOFTZ-ALLOW-NEXT: shr.u32 %r8, %r23, %r1;
+; SM60-NOFTZ-ALLOW-NEXT: shl.b32 %r9, %r8, 16;
+; SM60-NOFTZ-ALLOW-NEXT: sub.rn.f32 %r12, %r9, %r11;
+; SM60-NOFTZ-ALLOW-NEXT: bfe.u32 %r13, %r12, 16, 1;
+; SM60-NOFTZ-ALLOW-NEXT: add.s32 %r14, %r13, %r12;
+; SM60-NOFTZ-ALLOW-NEXT: add.s32 %r15, %r14, 32767;
+; SM60-NOFTZ-ALLOW-NEXT: setp.nan.f32 %p1, %r12, %r12;
+; SM60-NOFTZ-ALLOW-NEXT: or.b32 %r16, %r12, 4194304;
+; SM60-NOFTZ-ALLOW-NEXT: selp.b32 %r17, %r16, %r15, %p1;
+; SM60-NOFTZ-ALLOW-NEXT: shr.u32 %r18, %r17, 16;
+; SM60-NOFTZ-ALLOW-NEXT: shl.b32 %r19, %r18, %r1;
+; SM60-NOFTZ-ALLOW-NEXT: and.b32 %r20, %r23, %r2;
+; SM60-NOFTZ-ALLOW-NEXT: or.b32 %r21, %r20, %r19;
+; SM60-NOFTZ-ALLOW-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r23, %r21;
+; SM60-NOFTZ-ALLOW-NEXT: setp.ne.b32 %p2, %r3, %r23;
+; SM60-NOFTZ-ALLOW-NEXT: mov.b32 %r23, %r3;
+; SM60-NOFTZ-ALLOW-NEXT: @%p2 bra $L__BB79_1;
+; SM60-NOFTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NOFTZ-ALLOW-NEXT: shr.u32 %r22, %r3, %r1;
+; SM60-NOFTZ-ALLOW-NEXT: membar.cta;
+; SM60-NOFTZ-ALLOW-NEXT: st.param.b16 [func_retval0], %r22;
+; SM60-NOFTZ-ALLOW-NEXT: ret;
+;
+; SM60-FTZ-DISALLOW-LABEL: fsub_acq_rel_bfloat_global_cta(
+; SM60-FTZ-DISALLOW: {
+; SM60-FTZ-DISALLOW-NEXT: .reg .pred %p<3>;
+; SM60-FTZ-DISALLOW-NEXT: .reg .b16 %rs<2>;
+; SM60-FTZ-DISALLOW-NEXT: .reg .b32 %r<24>;
+; SM60-FTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
+; SM60-FTZ-DISALLOW-EMPTY:
+; SM60-FTZ-DISALLOW-NEXT: // %bb.0:
+; SM60-FTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fsub_acq_rel_bfloat_global_cta_param_1];
+; SM60-FTZ-DISALLOW-NEXT: ld.param.b64 %rd2, [fsub_acq_rel_bfloat_global_cta_param_0];
+; SM60-FTZ-DISALLOW-NEXT: membar.cta;
+; SM60-FTZ-DISALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-FTZ-DISALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-FTZ-DISALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM60-FTZ-DISALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-FTZ-DISALLOW-NEXT: mov.b32 %r6, 65535;
+; SM60-FTZ-DISALLOW-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-FTZ-DISALLOW-NEXT: not.b32 %r2, %r7;
+; SM60-FTZ-DISALLOW-NEXT: ld.volatile.global.b32 %r23, [%rd1];
+; SM60-FTZ-DISALLOW-NEXT: cvt.u32.u16 %r10, %rs1;
+; SM60-FTZ-DISALLOW-NEXT: shl.b32 %r11, %r10, 16;
+; SM60-FTZ-DISALLOW-NEXT: $L__BB79_1: // %atomicrmw.start
+; SM60-FTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-FTZ-DISALLOW-NEXT: shr.u32 %r8, %r23, %r1;
+; SM60-FTZ-DISALLOW-NEXT: shl.b32 %r9, %r8, 16;
+; SM60-FTZ-DISALLOW-NEXT: sub.rn.ftz.f32 %r12, %r9, %r11;
+; SM60-FTZ-DISALLOW-NEXT: bfe.u32 %r13, %r12, 16, 1;
+; SM60-FTZ-DISALLOW-NEXT: add.s32 %r14, %r13, %r12;
+; SM60-FTZ-DISALLOW-NEXT: add.s32 %r15, %r14, 32767;
+; SM60-FTZ-DISALLOW-NEXT: setp.nan.ftz.f32 %p1, %r12, %r12;
+; SM60-FTZ-DISALLOW-NEXT: or.b32 %r16, %r12, 4194304;
+; SM60-FTZ-DISALLOW-NEXT: selp.b32 %r17, %r16, %r15, %p1;
+; SM60-FTZ-DISALLOW-NEXT: shr.u32 %r18, %r17, 16;
+; SM60-FTZ-DISALLOW-NEXT: shl.b32 %r19, %r18, %r1;
+; SM60-FTZ-DISALLOW-NEXT: and.b32 %r20, %r23, %r2;
+; SM60-FTZ-DISALLOW-NEXT: or.b32 %r21, %r20, %r19;
+; SM60-FTZ-DISALLOW-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r23, %r21;
+; SM60-FTZ-DISALLOW-NEXT: setp.ne.b32 %p2, %r3, %r23;
+; SM60-FTZ-DISALLOW-NEXT: mov.b32 %r23, %r3;
+; SM60-FTZ-DISALLOW-NEXT: @%p2 bra $L__BB79_1;
+; SM60-FTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-FTZ-DISALLOW-NEXT: shr.u32 %r22, %r3, %r1;
+; SM60-FTZ-DISALLOW-NEXT: membar.cta;
+; SM60-FTZ-DISALLOW-NEXT: st.param.b16 [func_retval0], %r22;
+; SM60-FTZ-DISALLOW-NEXT: ret;
+;
+; SM60-FTZ-ALLOW-LABEL: fsub_acq_rel_bfloat_global_cta(
+; SM60-FTZ-ALLOW: {
+; SM60-FTZ-ALLOW-NEXT: .reg .pred %p<3>;
+; SM60-FTZ-ALLOW-NEXT: .reg .b16 %rs<2>;
+; SM60-FTZ-ALLOW-NEXT: .reg .b32 %r<24>;
+; SM60-FTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
+; SM60-FTZ-ALLOW-EMPTY:
+; SM60-FTZ-ALLOW-NEXT: // %bb.0:
+; SM60-FTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fsub_acq_rel_bfloat_global_cta_param_1];
+; SM60-FTZ-ALLOW-NEXT: ld.param.b64 %rd2, [fsub_acq_rel_bfloat_global_cta_param_0];
+; SM60-FTZ-ALLOW-NEXT: membar.cta;
+; SM60-FTZ-ALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-FTZ-ALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-FTZ-ALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM60-FTZ-ALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-FTZ-ALLOW-NEXT: mov.b32 %r6, 65535;
+; SM60-FTZ-ALLOW-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-FTZ-ALLOW-NEXT: not.b32 %r2, %r7;
+; SM60-FTZ-ALLOW-NEXT: ld.volatile.global.b32 %r23, [%rd1];
+; SM60-FTZ-ALLOW-NEXT: cvt.u32.u16 %r10, %rs1;
+; SM60-FTZ-ALLOW-NEXT: shl.b32 %r11, %r10, 16;
+; SM60-FTZ-ALLOW-NEXT: $L__BB79_1: // %atomicrmw.start
+; SM60-FTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-FTZ-ALLOW-NEXT: shr.u32 %r8, %r23, %r1;
+; SM60-FTZ-ALLOW-NEXT: shl.b32 %r9, %r8, 16;
+; SM60-FTZ-ALLOW-NEXT: sub.rn.ftz.f32 %r12, %r9, %r11;
+; SM60-FTZ-ALLOW-NEXT: bfe.u32 %r13, %r12, 16, 1;
+; SM60-FTZ-ALLOW-NEXT: add.s32 %r14, %r13, %r12;
+; SM60-FTZ-ALLOW-NEXT: add.s32 %r15, %r14, 32767;
+; SM60-FTZ-ALLOW-NEXT: setp.nan.ftz.f32 %p1, %r12, %r12;
+; SM60-FTZ-ALLOW-NEXT: or.b32 %r16, %r12, 4194304;
+; SM60-FTZ-ALLOW-NEXT: selp.b32 %r17, %r16, %r15, %p1;
+; SM60-FTZ-ALLOW-NEXT: shr.u32 %r18, %r17, 16;
+; SM60-FTZ-ALLOW-NEXT: shl.b32 %r19, %r18, %r1;
+; SM60-FTZ-ALLOW-NEXT: and.b32 %r20, %r23, %r2;
+; SM60-FTZ-ALLOW-NEXT: or.b32 %r21, %r20, %r19;
+; SM60-FTZ-ALLOW-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r23, %r21;
+; SM60-FTZ-ALLOW-NEXT: setp.ne.b32 %p2, %r3, %r23;
+; SM60-FTZ-ALLOW-NEXT: mov.b32 %r23, %r3;
+; SM60-FTZ-ALLOW-NEXT: @%p2 bra $L__BB79_1;
+; SM60-FTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-FTZ-ALLOW-NEXT: shr.u32 %r22, %r3, %r1;
+; SM60-FTZ-ALLOW-NEXT: membar.cta;
+; SM60-FTZ-ALLOW-NEXT: st.param.b16 [func_retval0], %r22;
+; SM60-FTZ-ALLOW-NEXT: ret;
%retval = atomicrmw fsub ptr addrspace(1) %addr, bfloat %val syncscope("block") acq_rel
ret bfloat %retval
}
define bfloat @fmin_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat %val) {
-; SM60-NOFTZ-LABEL: fmin_acq_rel_bfloat_global_cta(
-; SM60-NOFTZ: {
-; SM60-NOFTZ-NEXT: .reg .pred %p<3>;
-; SM60-NOFTZ-NEXT: .reg .b16 %rs<2>;
-; SM60-NOFTZ-NEXT: .reg .b32 %r<24>;
-; SM60-NOFTZ-NEXT: .reg .b64 %rd<3>;
-; SM60-NOFTZ-EMPTY:
-; SM60-NOFTZ-NEXT: // %bb.0:
-; SM60-NOFTZ-NEXT: ld.param.b16 %rs1, [fmin_acq_rel_bfloat_global_cta_param_1];
-; SM60-NOFTZ-NEXT: ld.param.b64 %rd2, [fmin_acq_rel_bfloat_global_cta_param_0];
-; SM60-NOFTZ-NEXT: membar.cta;
-; SM60-NOFTZ-NEXT: and.b64 %rd1, %rd2, -4;
-; SM60-NOFTZ-NEXT: cvt.u32.u64 %r4, %rd2;
-; SM60-NOFTZ-NEXT: and.b32 %r5, %r4, 3;
-; SM60-NOFTZ-NEXT: shl.b32 %r1, %r5, 3;
-; SM60-NOFTZ-NEXT: mov.b32 %r6, 65535;
-; SM60-NOFTZ-NEXT: shl.b32 %r7, %r6, %r1;
-; SM60-NOFTZ-NEXT: not.b32 %r2, %r7;
-; SM60-NOFTZ-NEXT: ld.volatile.global.b32 %r23, [%rd1];
-; SM60-NOFTZ-NEXT: cvt.u32.u16 %r10, %rs1;
-; SM60-NOFTZ-NEXT: shl.b32 %r11, %r10, 16;
-; SM60-NOFTZ-NEXT: $L__BB80_1: // %atomicrmw.start
-; SM60-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NOFTZ-NEXT: shr.u32 %r8, %r23, %r1;
-; SM60-NOFTZ-NEXT: shl.b32 %r9, %r8, 16;
-; SM60-NOFTZ-NEXT: min.f32 %r12, %r9, %r11;
-; SM60-NOFTZ-NEXT: bfe.u32 %r13, %r12, 16, 1;
-; SM60-NOFTZ-NEXT: add.s32 %r14, %r13, %r12;
-; SM60-NOFTZ-NEXT: add.s32 %r15, %r14, 32767;
-; SM60-NOFTZ-NEXT: setp.nan.f32 %p1, %r12, %r12;
-; SM60-NOFTZ-NEXT: or.b32 %r16, %r12, 4194304;
-; SM60-NOFTZ-NEXT: selp.b32 %r17, %r16, %r15, %p1;
-; SM60-NOFTZ-NEXT: shr.u32 %r18, %r17, 16;
-; SM60-NOFTZ-NEXT: shl.b32 %r19, %r18, %r1;
-; SM60-NOFTZ-NEXT: and.b32 %r20, %r23, %r2;
-; SM60-NOFTZ-NEXT: or.b32 %r21, %r20, %r19;
-; SM60-NOFTZ-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r23, %r21;
-; SM60-NOFTZ-NEXT: setp.ne.b32 %p2, %r3, %r23;
-; SM60-NOFTZ-NEXT: mov.b32 %r23, %r3;
-; SM60-NOFTZ-NEXT: @%p2 bra $L__BB80_1;
-; SM60-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end
-; SM60-NOFTZ-NEXT: shr.u32 %r22, %r3, %r1;
-; SM60-NOFTZ-NEXT: membar.cta;
-; SM60-NOFTZ-NEXT: st.param.b16 [func_retval0], %r22;
-; SM60-NOFTZ-NEXT: ret;
+; SM60-NOFTZ-DISALLOW-LABEL: fmin_acq_rel_bfloat_global_cta(
+; SM60-NOFTZ-DISALLOW: {
+; SM60-NOFTZ-DISALLOW-NEXT: .reg .pred %p<3>;
+; SM60-NOFTZ-DISALLOW-NEXT: .reg .b16 %rs<2>;
+; SM60-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<24>;
+; SM60-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
+; SM60-NOFTZ-DISALLOW-EMPTY:
+; SM60-NOFTZ-DISALLOW-NEXT: // %bb.0:
+; SM60-NOFTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fmin_acq_rel_bfloat_global_cta_param_1];
+; SM60-NOFTZ-DISALLOW-NEXT: ld.param.b64 %rd2, [fmin_acq_rel_bfloat_global_cta_param_0];
+; SM60-NOFTZ-DISALLOW-NEXT: membar.cta;
+; SM60-NOFTZ-DISALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-NOFTZ-DISALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM60-NOFTZ-DISALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-NOFTZ-DISALLOW-NEXT: mov.b32 %r6, 65535;
+; SM60-NOFTZ-DISALLOW-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-NOFTZ-DISALLOW-NEXT: not.b32 %r2, %r7;
+; SM60-NOFTZ-DISALLOW-NEXT: ld.volatile.global.b32 %r23, [%rd1];
+; SM60-NOFTZ-DISALLOW-NEXT: cvt.u32.u16 %r10, %rs1;
+; SM60-NOFTZ-DISALLOW-NEXT: shl.b32 %r11, %r10, 16;
+; SM60-NOFTZ-DISALLOW-NEXT: $L__BB80_1: // %atomicrmw.start
+; SM60-NOFTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NOFTZ-DISALLOW-NEXT: shr.u32 %r8, %r23, %r1;
+; SM60-NOFTZ-DISALLOW-NEXT: shl.b32 %r9, %r8, 16;
+; SM60-NOFTZ-DISALLOW-NEXT: min.f32 %r12, %r9, %r11;
+; SM60-NOFTZ-DISALLOW-NEXT: bfe.u32 %r13, %r12, 16, 1;
+; SM60-NOFTZ-DISALLOW-NEXT: add.s32 %r14, %r13, %r12;
+; SM60-NOFTZ-DISALLOW-NEXT: add.s32 %r15, %r14, 32767;
+; SM60-NOFTZ-DISALLOW-NEXT: setp.nan.f32 %p1, %r12, %r12;
+; SM60-NOFTZ-DISALLOW-NEXT: or.b32 %r16, %r12, 4194304;
+; SM60-NOFTZ-DISALLOW-NEXT: selp.b32 %r17, %r16, %r15, %p1;
+; SM60-NOFTZ-DISALLOW-NEXT: shr.u32 %r18, %r17, 16;
+; SM60-NOFTZ-DISALLOW-NEXT: shl.b32 %r19, %r18, %r1;
+; SM60-NOFTZ-DISALLOW-NEXT: and.b32 %r20, %r23, %r2;
+; SM60-NOFTZ-DISALLOW-NEXT: or.b32 %r21, %r20, %r19;
+; SM60-NOFTZ-DISALLOW-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r23, %r21;
+; SM60-NOFTZ-DISALLOW-NEXT: setp.ne.b32 %p2, %r3, %r23;
+; SM60-NOFTZ-DISALLOW-NEXT: mov.b32 %r23, %r3;
+; SM60-NOFTZ-DISALLOW-NEXT: @%p2 bra $L__BB80_1;
+; SM60-NOFTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NOFTZ-DISALLOW-NEXT: shr.u32 %r22, %r3, %r1;
+; SM60-NOFTZ-DISALLOW-NEXT: membar.cta;
+; SM60-NOFTZ-DISALLOW-NEXT: st.param.b16 [func_retval0], %r22;
+; SM60-NOFTZ-DISALLOW-NEXT: ret;
+;
+; SM60-NOFTZ-ALLOW-LABEL: fmin_acq_rel_bfloat_global_cta(
+; SM60-NOFTZ-ALLOW: {
+; SM60-NOFTZ-ALLOW-NEXT: .reg .pred %p<3>;
+; SM60-NOFTZ-ALLOW-NEXT: .reg .b16 %rs<2>;
+; SM60-NOFTZ-ALLOW-NEXT: .reg .b32 %r<24>;
+; SM60-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
+; SM60-NOFTZ-ALLOW-EMPTY:
+; SM60-NOFTZ-ALLOW-NEXT: // %bb.0:
+; SM60-NOFTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fmin_acq_rel_bfloat_global_cta_param_1];
+; SM60-NOFTZ-ALLOW-NEXT: ld.param.b64 %rd2, [fmin_acq_rel_bfloat_global_cta_param_0];
+; SM60-NOFTZ-ALLOW-NEXT: membar.cta;
+; SM60-NOFTZ-ALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NOFTZ-ALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-NOFTZ-ALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM60-NOFTZ-ALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-NOFTZ-ALLOW-NEXT: mov.b32 %r6, 65535;
+; SM60-NOFTZ-ALLOW-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-NOFTZ-ALLOW-NEXT: not.b32 %r2, %r7;
+; SM60-NOFTZ-ALLOW-NEXT: ld.volatile.global.b32 %r23, [%rd1];
+; SM60-NOFTZ-ALLOW-NEXT: cvt.u32.u16 %r10, %rs1;
+; SM60-NOFTZ-ALLOW-NEXT: shl.b32 %r11, %r10, 16;
+; SM60-NOFTZ-ALLOW-NEXT: $L__BB80_1: // %atomicrmw.start
+; SM60-NOFTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NOFTZ-ALLOW-NEXT: shr.u32 %r8, %r23, %r1;
+; SM60-NOFTZ-ALLOW-NEXT: shl.b32 %r9, %r8, 16;
+; SM60-NOFTZ-ALLOW-NEXT: min.f32 %r12, %r9, %r11;
+; SM60-NOFTZ-ALLOW-NEXT: bfe.u32 %r13, %r12, 16, 1;
+; SM60-NOFTZ-ALLOW-NEXT: add.s32 %r14, %r13, %r12;
+; SM60-NOFTZ-ALLOW-NEXT: add.s32 %r15, %r14, 32767;
+; SM60-NOFTZ-ALLOW-NEXT: setp.nan.f32 %p1, %r12, %r12;
+; SM60-NOFTZ-ALLOW-NEXT: or.b32 %r16, %r12, 4194304;
+; SM60-NOFTZ-ALLOW-NEXT: selp.b32 %r17, %r16, %r15, %p1;
+; SM60-NOFTZ-ALLOW-NEXT: shr.u32 %r18, %r17, 16;
+; SM60-NOFTZ-ALLOW-NEXT: shl.b32 %r19, %r18, %r1;
+; SM60-NOFTZ-ALLOW-NEXT: and.b32 %r20, %r23, %r2;
+; SM60-NOFTZ-ALLOW-NEXT: or.b32 %r21, %r20, %r19;
+; SM60-NOFTZ-ALLOW-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r23, %r21;
+; SM60-NOFTZ-ALLOW-NEXT: setp.ne.b32 %p2, %r3, %r23;
+; SM60-NOFTZ-ALLOW-NEXT: mov.b32 %r23, %r3;
+; SM60-NOFTZ-ALLOW-NEXT: @%p2 bra $L__BB80_1;
+; SM60-NOFTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NOFTZ-ALLOW-NEXT: shr.u32 %r22, %r3, %r1;
+; SM60-NOFTZ-ALLOW-NEXT: membar.cta;
+; SM60-NOFTZ-ALLOW-NEXT: st.param.b16 [func_retval0], %r22;
+; SM60-NOFTZ-ALLOW-NEXT: ret;
;
-; SM60-FTZ-LABEL: fmin_acq_rel_bfloat_global_cta(
-; SM60-FTZ: {
-; SM60-FTZ-NEXT: .reg .pred %p<3>;
-; SM60-FTZ-NEXT: .reg .b16 %rs<2>;
-; SM60-FTZ-NEXT: .reg .b32 %r<24>;
-; SM60-FTZ-NEXT: .reg .b64 %rd<3>;
-; SM60-FTZ-EMPTY:
-; SM60-FTZ-NEXT: // %bb.0:
-; SM60-FTZ-NEXT: ld.param.b16 %rs1, [fmin_acq_rel_bfloat_global_cta_param_1];
-; SM60-FTZ-NEXT: ld.param.b64 %rd2, [fmin_acq_rel_bfloat_global_cta_param_0];
-; SM60-FTZ-NEXT: membar.cta;
-; SM60-FTZ-NEXT: and.b64 %rd1, %rd2, -4;
-; SM60-FTZ-NEXT: cvt.u32.u64 %r4, %rd2;
-; SM60-FTZ-NEXT: and.b32 %r5, %r4, 3;
-; SM60-FTZ-NEXT: shl.b32 %r1, %r5, 3;
-; SM60-FTZ-NEXT: mov.b32 %r6, 65535;
-; SM60-FTZ-NEXT: shl.b32 %r7, %r6, %r1;
-; SM60-FTZ-NEXT: not.b32 %r2, %r7;
-; SM60-FTZ-NEXT: ld.volatile.global.b32 %r23, [%rd1];
-; SM60-FTZ-NEXT: cvt.u32.u16 %r10, %rs1;
-; SM60-FTZ-NEXT: shl.b32 %r11, %r10, 16;
-; SM60-FTZ-NEXT: $L__BB80_1: // %atomicrmw.start
-; SM60-FTZ-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-FTZ-NEXT: shr.u32 %r8, %r23, %r1;
-; SM60-FTZ-NEXT: shl.b32 %r9, %r8, 16;
-; SM60-FTZ-NEXT: min.ftz.f32 %r12, %r9, %r11;
-; SM60-FTZ-NEXT: bfe.u32 %r13, %r12, 16, 1;
-; SM60-FTZ-NEXT: add.s32 %r14, %r13, %r12;
-; SM60-FTZ-NEXT: add.s32 %r15, %r14, 32767;
-; SM60-FTZ-NEXT: setp.nan.ftz.f32 %p1, %r12, %r12;
-; SM60-FTZ-NEXT: or.b32 %r16, %r12, 4194304;
-; SM60-FTZ-NEXT: selp.b32 %r17, %r16, %r15, %p1;
-; SM60-FTZ-NEXT: shr.u32 %r18, %r17, 16;
-; SM60-FTZ-NEXT: shl.b32 %r19, %r18, %r1;
-; SM60-FTZ-NEXT: and.b32 %r20, %r23, %r2;
-; SM60-FTZ-NEXT: or.b32 %r21, %r20, %r19;
-; SM60-FTZ-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r23, %r21;
-; SM60-FTZ-NEXT: setp.ne.b32 %p2, %r3, %r23;
-; SM60-FTZ-NEXT: mov.b32 %r23, %r3;
-; SM60-FTZ-NEXT: @%p2 bra $L__BB80_1;
-; SM60-FTZ-NEXT: // %bb.2: // %atomicrmw.end
-; SM60-FTZ-NEXT: shr.u32 %r22, %r3, %r1;
-; SM60-FTZ-NEXT: membar.cta;
-; SM60-FTZ-NEXT: st.param.b16 [func_retval0], %r22;
-; SM60-FTZ-NEXT: ret;
+; SM60-FTZ-DISALLOW-LABEL: fmin_acq_rel_bfloat_global_cta(
+; SM60-FTZ-DISALLOW: {
+; SM60-FTZ-DISALLOW-NEXT: .reg .pred %p<3>;
+; SM60-FTZ-DISALLOW-NEXT: .reg .b16 %rs<2>;
+; SM60-FTZ-DISALLOW-NEXT: .reg .b32 %r<24>;
+; SM60-FTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
+; SM60-FTZ-DISALLOW-EMPTY:
+; SM60-FTZ-DISALLOW-NEXT: // %bb.0:
+; SM60-FTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fmin_acq_rel_bfloat_global_cta_param_1];
+; SM60-FTZ-DISALLOW-NEXT: ld.param.b64 %rd2, [fmin_acq_rel_bfloat_global_cta_param_0];
+; SM60-FTZ-DISALLOW-NEXT: membar.cta;
+; SM60-FTZ-DISALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-FTZ-DISALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-FTZ-DISALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM60-FTZ-DISALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-FTZ-DISALLOW-NEXT: mov.b32 %r6, 65535;
+; SM60-FTZ-DISALLOW-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-FTZ-DISALLOW-NEXT: not.b32 %r2, %r7;
+; SM60-FTZ-DISALLOW-NEXT: ld.volatile.global.b32 %r23, [%rd1];
+; SM60-FTZ-DISALLOW-NEXT: cvt.u32.u16 %r10, %rs1;
+; SM60-FTZ-DISALLOW-NEXT: shl.b32 %r11, %r10, 16;
+; SM60-FTZ-DISALLOW-NEXT: $L__BB80_1: // %atomicrmw.start
+; SM60-FTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-FTZ-DISALLOW-NEXT: shr.u32 %r8, %r23, %r1;
+; SM60-FTZ-DISALLOW-NEXT: shl.b32 %r9, %r8, 16;
+; SM60-FTZ-DISALLOW-NEXT: min.ftz.f32 %r12, %r9, %r11;
+; SM60-FTZ-DISALLOW-NEXT: bfe.u32 %r13, %r12, 16, 1;
+; SM60-FTZ-DISALLOW-NEXT: add.s32 %r14, %r13, %r12;
+; SM60-FTZ-DISALLOW-NEXT: add.s32 %r15, %r14, 32767;
+; SM60-FTZ-DISALLOW-NEXT: setp.nan.ftz.f32 %p1, %r12, %r12;
+; SM60-FTZ-DISALLOW-NEXT: or.b32 %r16, %r12, 4194304;
+; SM60-FTZ-DISALLOW-NEXT: selp.b32 %r17, %r16, %r15, %p1;
+; SM60-FTZ-DISALLOW-NEXT: shr.u32 %r18, %r17, 16;
+; SM60-FTZ-DISALLOW-NEXT: shl.b32 %r19, %r18, %r1;
+; SM60-FTZ-DISALLOW-NEXT: and.b32 %r20, %r23, %r2;
+; SM60-FTZ-DISALLOW-NEXT: or.b32 %r21, %r20, %r19;
+; SM60-FTZ-DISALLOW-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r23, %r21;
+; SM60-FTZ-DISALLOW-NEXT: setp.ne.b32 %p2, %r3, %r23;
+; SM60-FTZ-DISALLOW-NEXT: mov.b32 %r23, %r3;
+; SM60-FTZ-DISALLOW-NEXT: @%p2 bra $L__BB80_1;
+; SM60-FTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-FTZ-DISALLOW-NEXT: shr.u32 %r22, %r3, %r1;
+; SM60-FTZ-DISALLOW-NEXT: membar.cta;
+; SM60-FTZ-DISALLOW-NEXT: st.param.b16 [func_retval0], %r22;
+; SM60-FTZ-DISALLOW-NEXT: ret;
+;
+; SM60-FTZ-ALLOW-LABEL: fmin_acq_rel_bfloat_global_cta(
+; SM60-FTZ-ALLOW: {
+; SM60-FTZ-ALLOW-NEXT: .reg .pred %p<3>;
+; SM60-FTZ-ALLOW-NEXT: .reg .b16 %rs<2>;
+; SM60-FTZ-ALLOW-NEXT: .reg .b32 %r<24>;
+; SM60-FTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
+; SM60-FTZ-ALLOW-EMPTY:
+; SM60-FTZ-ALLOW-NEXT: // %bb.0:
+; SM60-FTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fmin_acq_rel_bfloat_global_cta_param_1];
+; SM60-FTZ-ALLOW-NEXT: ld.param.b64 %rd2, [fmin_acq_rel_bfloat_global_cta_param_0];
+; SM60-FTZ-ALLOW-NEXT: membar.cta;
+; SM60-FTZ-ALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-FTZ-ALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-FTZ-ALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM60-FTZ-ALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-FTZ-ALLOW-NEXT: mov.b32 %r6, 65535;
+; SM60-FTZ-ALLOW-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-FTZ-ALLOW-NEXT: not.b32 %r2, %r7;
+; SM60-FTZ-ALLOW-NEXT: ld.volatile.global.b32 %r23, [%rd1];
+; SM60-FTZ-ALLOW-NEXT: cvt.u32.u16 %r10, %rs1;
+; SM60-FTZ-ALLOW-NEXT: shl.b32 %r11, %r10, 16;
+; SM60-FTZ-ALLOW-NEXT: $L__BB80_1: // %atomicrmw.start
+; SM60-FTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-FTZ-ALLOW-NEXT: shr.u32 %r8, %r23, %r1;
+; SM60-FTZ-ALLOW-NEXT: shl.b32 %r9, %r8, 16;
+; SM60-FTZ-ALLOW-NEXT: min.ftz.f32 %r12, %r9, %r11;
+; SM60-FTZ-ALLOW-NEXT: bfe.u32 %r13, %r12, 16, 1;
+; SM60-FTZ-ALLOW-NEXT: add.s32 %r14, %r13, %r12;
+; SM60-FTZ-ALLOW-NEXT: add.s32 %r15, %r14, 32767;
+; SM60-FTZ-ALLOW-NEXT: setp.nan.ftz.f32 %p1, %r12, %r12;
+; SM60-FTZ-ALLOW-NEXT: or.b32 %r16, %r12, 4194304;
+; SM60-FTZ-ALLOW-NEXT: selp.b32 %r17, %r16, %r15, %p1;
+; SM60-FTZ-ALLOW-NEXT: shr.u32 %r18, %r17, 16;
+; SM60-FTZ-ALLOW-NEXT: shl.b32 %r19, %r18, %r1;
+; SM60-FTZ-ALLOW-NEXT: and.b32 %r20, %r23, %r2;
+; SM60-FTZ-ALLOW-NEXT: or.b32 %r21, %r20, %r19;
+; SM60-FTZ-ALLOW-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r23, %r21;
+; SM60-FTZ-ALLOW-NEXT: setp.ne.b32 %p2, %r3, %r23;
+; SM60-FTZ-ALLOW-NEXT: mov.b32 %r23, %r3;
+; SM60-FTZ-ALLOW-NEXT: @%p2 bra $L__BB80_1;
+; SM60-FTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-FTZ-ALLOW-NEXT: shr.u32 %r22, %r3, %r1;
+; SM60-FTZ-ALLOW-NEXT: membar.cta;
+; SM60-FTZ-ALLOW-NEXT: st.param.b16 [func_retval0], %r22;
+; SM60-FTZ-ALLOW-NEXT: ret;
%retval = atomicrmw fmin ptr addrspace(1) %addr, bfloat %val syncscope("block") acq_rel
ret bfloat %retval
}
define bfloat @fmax_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat %val) {
-; SM60-NOFTZ-LABEL: fmax_acq_rel_bfloat_global_cta(
-; SM60-NOFTZ: {
-; SM60-NOFTZ-NEXT: .reg .pred %p<3>;
-; SM60-NOFTZ-NEXT: .reg .b16 %rs<2>;
-; SM60-NOFTZ-NEXT: .reg .b32 %r<24>;
-; SM60-NOFTZ-NEXT: .reg .b64 %rd<3>;
-; SM60-NOFTZ-EMPTY:
-; SM60-NOFTZ-NEXT: // %bb.0:
-; SM60-NOFTZ-NEXT: ld.param.b16 %rs1, [fmax_acq_rel_bfloat_global_cta_param_1];
-; SM60-NOFTZ-NEXT: ld.param.b64 %rd2, [fmax_acq_rel_bfloat_global_cta_param_0];
-; SM60-NOFTZ-NEXT: membar.cta;
-; SM60-NOFTZ-NEXT: and.b64 %rd1, %rd2, -4;
-; SM60-NOFTZ-NEXT: cvt.u32.u64 %r4, %rd2;
-; SM60-NOFTZ-NEXT: and.b32 %r5, %r4, 3;
-; SM60-NOFTZ-NEXT: shl.b32 %r1, %r5, 3;
-; SM60-NOFTZ-NEXT: mov.b32 %r6, 65535;
-; SM60-NOFTZ-NEXT: shl.b32 %r7, %r6, %r1;
-; SM60-NOFTZ-NEXT: not.b32 %r2, %r7;
-; SM60-NOFTZ-NEXT: ld.volatile.global.b32 %r23, [%rd1];
-; SM60-NOFTZ-NEXT: cvt.u32.u16 %r10, %rs1;
-; SM60-NOFTZ-NEXT: shl.b32 %r11, %r10, 16;
-; SM60-NOFTZ-NEXT: $L__BB81_1: // %atomicrmw.start
-; SM60-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NOFTZ-NEXT: shr.u32 %r8, %r23, %r1;
-; SM60-NOFTZ-NEXT: shl.b32 %r9, %r8, 16;
-; SM60-NOFTZ-NEXT: max.f32 %r12, %r9, %r11;
-; SM60-NOFTZ-NEXT: bfe.u32 %r13, %r12, 16, 1;
-; SM60-NOFTZ-NEXT: add.s32 %r14, %r13, %r12;
-; SM60-NOFTZ-NEXT: add.s32 %r15, %r14, 32767;
-; SM60-NOFTZ-NEXT: setp.nan.f32 %p1, %r12, %r12;
-; SM60-NOFTZ-NEXT: or.b32 %r16, %r12, 4194304;
-; SM60-NOFTZ-NEXT: selp.b32 %r17, %r16, %r15, %p1;
-; SM60-NOFTZ-NEXT: shr.u32 %r18, %r17, 16;
-; SM60-NOFTZ-NEXT: shl.b32 %r19, %r18, %r1;
-; SM60-NOFTZ-NEXT: and.b32 %r20, %r23, %r2;
-; SM60-NOFTZ-NEXT: or.b32 %r21, %r20, %r19;
-; SM60-NOFTZ-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r23, %r21;
-; SM60-NOFTZ-NEXT: setp.ne.b32 %p2, %r3, %r23;
-; SM60-NOFTZ-NEXT: mov.b32 %r23, %r3;
-; SM60-NOFTZ-NEXT: @%p2 bra $L__BB81_1;
-; SM60-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end
-; SM60-NOFTZ-NEXT: shr.u32 %r22, %r3, %r1;
-; SM60-NOFTZ-NEXT: membar.cta;
-; SM60-NOFTZ-NEXT: st.param.b16 [func_retval0], %r22;
-; SM60-NOFTZ-NEXT: ret;
+; SM60-NOFTZ-DISALLOW-LABEL: fmax_acq_rel_bfloat_global_cta(
+; SM60-NOFTZ-DISALLOW: {
+; SM60-NOFTZ-DISALLOW-NEXT: .reg .pred %p<3>;
+; SM60-NOFTZ-DISALLOW-NEXT: .reg .b16 %rs<2>;
+; SM60-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<24>;
+; SM60-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
+; SM60-NOFTZ-DISALLOW-EMPTY:
+; SM60-NOFTZ-DISALLOW-NEXT: // %bb.0:
+; SM60-NOFTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fmax_acq_rel_bfloat_global_cta_param_1];
+; SM60-NOFTZ-DISALLOW-NEXT: ld.param.b64 %rd2, [fmax_acq_rel_bfloat_global_cta_param_0];
+; SM60-NOFTZ-DISALLOW-NEXT: membar.cta;
+; SM60-NOFTZ-DISALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-NOFTZ-DISALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM60-NOFTZ-DISALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-NOFTZ-DISALLOW-NEXT: mov.b32 %r6, 65535;
+; SM60-NOFTZ-DISALLOW-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-NOFTZ-DISALLOW-NEXT: not.b32 %r2, %r7;
+; SM60-NOFTZ-DISALLOW-NEXT: ld.volatile.global.b32 %r23, [%rd1];
+; SM60-NOFTZ-DISALLOW-NEXT: cvt.u32.u16 %r10, %rs1;
+; SM60-NOFTZ-DISALLOW-NEXT: shl.b32 %r11, %r10, 16;
+; SM60-NOFTZ-DISALLOW-NEXT: $L__BB81_1: // %atomicrmw.start
+; SM60-NOFTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NOFTZ-DISALLOW-NEXT: shr.u32 %r8, %r23, %r1;
+; SM60-NOFTZ-DISALLOW-NEXT: shl.b32 %r9, %r8, 16;
+; SM60-NOFTZ-DISALLOW-NEXT: max.f32 %r12, %r9, %r11;
+; SM60-NOFTZ-DISALLOW-NEXT: bfe.u32 %r13, %r12, 16, 1;
+; SM60-NOFTZ-DISALLOW-NEXT: add.s32 %r14, %r13, %r12;
+; SM60-NOFTZ-DISALLOW-NEXT: add.s32 %r15, %r14, 32767;
+; SM60-NOFTZ-DISALLOW-NEXT: setp.nan.f32 %p1, %r12, %r12;
+; SM60-NOFTZ-DISALLOW-NEXT: or.b32 %r16, %r12, 4194304;
+; SM60-NOFTZ-DISALLOW-NEXT: selp.b32 %r17, %r16, %r15, %p1;
+; SM60-NOFTZ-DISALLOW-NEXT: shr.u32 %r18, %r17, 16;
+; SM60-NOFTZ-DISALLOW-NEXT: shl.b32 %r19, %r18, %r1;
+; SM60-NOFTZ-DISALLOW-NEXT: and.b32 %r20, %r23, %r2;
+; SM60-NOFTZ-DISALLOW-NEXT: or.b32 %r21, %r20, %r19;
+; SM60-NOFTZ-DISALLOW-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r23, %r21;
+; SM60-NOFTZ-DISALLOW-NEXT: setp.ne.b32 %p2, %r3, %r23;
+; SM60-NOFTZ-DISALLOW-NEXT: mov.b32 %r23, %r3;
+; SM60-NOFTZ-DISALLOW-NEXT: @%p2 bra $L__BB81_1;
+; SM60-NOFTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NOFTZ-DISALLOW-NEXT: shr.u32 %r22, %r3, %r1;
+; SM60-NOFTZ-DISALLOW-NEXT: membar.cta;
+; SM60-NOFTZ-DISALLOW-NEXT: st.param.b16 [func_retval0], %r22;
+; SM60-NOFTZ-DISALLOW-NEXT: ret;
+;
+; SM60-NOFTZ-ALLOW-LABEL: fmax_acq_rel_bfloat_global_cta(
+; SM60-NOFTZ-ALLOW: {
+; SM60-NOFTZ-ALLOW-NEXT: .reg .pred %p<3>;
+; SM60-NOFTZ-ALLOW-NEXT: .reg .b16 %rs<2>;
+; SM60-NOFTZ-ALLOW-NEXT: .reg .b32 %r<24>;
+; SM60-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
+; SM60-NOFTZ-ALLOW-EMPTY:
+; SM60-NOFTZ-ALLOW-NEXT: // %bb.0:
+; SM60-NOFTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fmax_acq_rel_bfloat_global_cta_param_1];
+; SM60-NOFTZ-ALLOW-NEXT: ld.param.b64 %rd2, [fmax_acq_rel_bfloat_global_cta_param_0];
+; SM60-NOFTZ-ALLOW-NEXT: membar.cta;
+; SM60-NOFTZ-ALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NOFTZ-ALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-NOFTZ-ALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM60-NOFTZ-ALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-NOFTZ-ALLOW-NEXT: mov.b32 %r6, 65535;
+; SM60-NOFTZ-ALLOW-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-NOFTZ-ALLOW-NEXT: not.b32 %r2, %r7;
+; SM60-NOFTZ-ALLOW-NEXT: ld.volatile.global.b32 %r23, [%rd1];
+; SM60-NOFTZ-ALLOW-NEXT: cvt.u32.u16 %r10, %rs1;
+; SM60-NOFTZ-ALLOW-NEXT: shl.b32 %r11, %r10, 16;
+; SM60-NOFTZ-ALLOW-NEXT: $L__BB81_1: // %atomicrmw.start
+; SM60-NOFTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NOFTZ-ALLOW-NEXT: shr.u32 %r8, %r23, %r1;
+; SM60-NOFTZ-ALLOW-NEXT: shl.b32 %r9, %r8, 16;
+; SM60-NOFTZ-ALLOW-NEXT: max.f32 %r12, %r9, %r11;
+; SM60-NOFTZ-ALLOW-NEXT: bfe.u32 %r13, %r12, 16, 1;
+; SM60-NOFTZ-ALLOW-NEXT: add.s32 %r14, %r13, %r12;
+; SM60-NOFTZ-ALLOW-NEXT: add.s32 %r15, %r14, 32767;
+; SM60-NOFTZ-ALLOW-NEXT: setp.nan.f32 %p1, %r12, %r12;
+; SM60-NOFTZ-ALLOW-NEXT: or.b32 %r16, %r12, 4194304;
+; SM60-NOFTZ-ALLOW-NEXT: selp.b32 %r17, %r16, %r15, %p1;
+; SM60-NOFTZ-ALLOW-NEXT: shr.u32 %r18, %r17, 16;
+; SM60-NOFTZ-ALLOW-NEXT: shl.b32 %r19, %r18, %r1;
+; SM60-NOFTZ-ALLOW-NEXT: and.b32 %r20, %r23, %r2;
+; SM60-NOFTZ-ALLOW-NEXT: or.b32 %r21, %r20, %r19;
+; SM60-NOFTZ-ALLOW-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r23, %r21;
+; SM60-NOFTZ-ALLOW-NEXT: setp.ne.b32 %p2, %r3, %r23;
+; SM60-NOFTZ-ALLOW-NEXT: mov.b32 %r23, %r3;
+; SM60-NOFTZ-ALLOW-NEXT: @%p2 bra $L__BB81_1;
+; SM60-NOFTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NOFTZ-ALLOW-NEXT: shr.u32 %r22, %r3, %r1;
+; SM60-NOFTZ-ALLOW-NEXT: membar.cta;
+; SM60-NOFTZ-ALLOW-NEXT: st.param.b16 [func_retval0], %r22;
+; SM60-NOFTZ-ALLOW-NEXT: ret;
+;
+; SM60-FTZ-DISALLOW-LABEL: fmax_acq_rel_bfloat_global_cta(
+; SM60-FTZ-DISALLOW: {
+; SM60-FTZ-DISALLOW-NEXT: .reg .pred %p<3>;
+; SM60-FTZ-DISALLOW-NEXT: .reg .b16 %rs<2>;
+; SM60-FTZ-DISALLOW-NEXT: .reg .b32 %r<24>;
+; SM60-FTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
+; SM60-FTZ-DISALLOW-EMPTY:
+; SM60-FTZ-DISALLOW-NEXT: // %bb.0:
+; SM60-FTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fmax_acq_rel_bfloat_global_cta_param_1];
+; SM60-FTZ-DISALLOW-NEXT: ld.param.b64 %rd2, [fmax_acq_rel_bfloat_global_cta_param_0];
+; SM60-FTZ-DISALLOW-NEXT: membar.cta;
+; SM60-FTZ-DISALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-FTZ-DISALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-FTZ-DISALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM60-FTZ-DISALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-FTZ-DISALLOW-NEXT: mov.b32 %r6, 65535;
+; SM60-FTZ-DISALLOW-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-FTZ-DISALLOW-NEXT: not.b32 %r2, %r7;
+; SM60-FTZ-DISALLOW-NEXT: ld.volatile.global.b32 %r23, [%rd1];
+; SM60-FTZ-DISALLOW-NEXT: cvt.u32.u16 %r10, %rs1;
+; SM60-FTZ-DISALLOW-NEXT: shl.b32 %r11, %r10, 16;
+; SM60-FTZ-DISALLOW-NEXT: $L__BB81_1: // %atomicrmw.start
+; SM60-FTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-FTZ-DISALLOW-NEXT: shr.u32 %r8, %r23, %r1;
+; SM60-FTZ-DISALLOW-NEXT: shl.b32 %r9, %r8, 16;
+; SM60-FTZ-DISALLOW-NEXT: max.ftz.f32 %r12, %r9, %r11;
+; SM60-FTZ-DISALLOW-NEXT: bfe.u32 %r13, %r12, 16, 1;
+; SM60-FTZ-DISALLOW-NEXT: add.s32 %r14, %r13, %r12;
+; SM60-FTZ-DISALLOW-NEXT: add.s32 %r15, %r14, 32767;
+; SM60-FTZ-DISALLOW-NEXT: setp.nan.ftz.f32 %p1, %r12, %r12;
+; SM60-FTZ-DISALLOW-NEXT: or.b32 %r16, %r12, 4194304;
+; SM60-FTZ-DISALLOW-NEXT: selp.b32 %r17, %r16, %r15, %p1;
+; SM60-FTZ-DISALLOW-NEXT: shr.u32 %r18, %r17, 16;
+; SM60-FTZ-DISALLOW-NEXT: shl.b32 %r19, %r18, %r1;
+; SM60-FTZ-DISALLOW-NEXT: and.b32 %r20, %r23, %r2;
+; SM60-FTZ-DISALLOW-NEXT: or.b32 %r21, %r20, %r19;
+; SM60-FTZ-DISALLOW-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r23, %r21;
+; SM60-FTZ-DISALLOW-NEXT: setp.ne.b32 %p2, %r3, %r23;
+; SM60-FTZ-DISALLOW-NEXT: mov.b32 %r23, %r3;
+; SM60-FTZ-DISALLOW-NEXT: @%p2 bra $L__BB81_1;
+; SM60-FTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-FTZ-DISALLOW-NEXT: shr.u32 %r22, %r3, %r1;
+; SM60-FTZ-DISALLOW-NEXT: membar.cta;
+; SM60-FTZ-DISALLOW-NEXT: st.param.b16 [func_retval0], %r22;
+; SM60-FTZ-DISALLOW-NEXT: ret;
;
-; SM60-FTZ-LABEL: fmax_acq_rel_bfloat_global_cta(
-; SM60-FTZ: {
-; SM60-FTZ-NEXT: .reg .pred %p<3>;
-; SM60-FTZ-NEXT: .reg .b16 %rs<2>;
-; SM60-FTZ-NEXT: .reg .b32 %r<24>;
-; SM60-FTZ-NEXT: .reg .b64 %rd<3>;
-; SM60-FTZ-EMPTY:
-; SM60-FTZ-NEXT: // %bb.0:
-; SM60-FTZ-NEXT: ld.param.b16 %rs1, [fmax_acq_rel_bfloat_global_cta_param_1];
-; SM60-FTZ-NEXT: ld.param.b64 %rd2, [fmax_acq_rel_bfloat_global_cta_param_0];
-; SM60-FTZ-NEXT: membar.cta;
-; SM60-FTZ-NEXT: and.b64 %rd1, %rd2, -4;
-; SM60-FTZ-NEXT: cvt.u32.u64 %r4, %rd2;
-; SM60-FTZ-NEXT: and.b32 %r5, %r4, 3;
-; SM60-FTZ-NEXT: shl.b32 %r1, %r5, 3;
-; SM60-FTZ-NEXT: mov.b32 %r6, 65535;
-; SM60-FTZ-NEXT: shl.b32 %r7, %r6, %r1;
-; SM60-FTZ-NEXT: not.b32 %r2, %r7;
-; SM60-FTZ-NEXT: ld.volatile.global.b32 %r23, [%rd1];
-; SM60-FTZ-NEXT: cvt.u32.u16 %r10, %rs1;
-; SM60-FTZ-NEXT: shl.b32 %r11, %r10, 16;
-; SM60-FTZ-NEXT: $L__BB81_1: // %atomicrmw.start
-; SM60-FTZ-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-FTZ-NEXT: shr.u32 %r8, %r23, %r1;
-; SM60-FTZ-NEXT: shl.b32 %r9, %r8, 16;
-; SM60-FTZ-NEXT: max.ftz.f32 %r12, %r9, %r11;
-; SM60-FTZ-NEXT: bfe.u32 %r13, %r12, 16, 1;
-; SM60-FTZ-NEXT: add.s32 %r14, %r13, %r12;
-; SM60-FTZ-NEXT: add.s32 %r15, %r14, 32767;
-; SM60-FTZ-NEXT: setp.nan.ftz.f32 %p1, %r12, %r12;
-; SM60-FTZ-NEXT: or.b32 %r16, %r12, 4194304;
-; SM60-FTZ-NEXT: selp.b32 %r17, %r16, %r15, %p1;
-; SM60-FTZ-NEXT: shr.u32 %r18, %r17, 16;
-; SM60-FTZ-NEXT: shl.b32 %r19, %r18, %r1;
-; SM60-FTZ-NEXT: and.b32 %r20, %r23, %r2;
-; SM60-FTZ-NEXT: or.b32 %r21, %r20, %r19;
-; SM60-FTZ-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r23, %r21;
-; SM60-FTZ-NEXT: setp.ne.b32 %p2, %r3, %r23;
-; SM60-FTZ-NEXT: mov.b32 %r23, %r3;
-; SM60-FTZ-NEXT: @%p2 bra $L__BB81_1;
-; SM60-FTZ-NEXT: // %bb.2: // %atomicrmw.end
-; SM60-FTZ-NEXT: shr.u32 %r22, %r3, %r1;
-; SM60-FTZ-NEXT: membar.cta;
-; SM60-FTZ-NEXT: st.param.b16 [func_retval0], %r22;
-; SM60-FTZ-NEXT: ret;
+; SM60-FTZ-ALLOW-LABEL: fmax_acq_rel_bfloat_global_cta(
+; SM60-FTZ-ALLOW: {
+; SM60-FTZ-ALLOW-NEXT: .reg .pred %p<3>;
+; SM60-FTZ-ALLOW-NEXT: .reg .b16 %rs<2>;
+; SM60-FTZ-ALLOW-NEXT: .reg .b32 %r<24>;
+; SM60-FTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
+; SM60-FTZ-ALLOW-EMPTY:
+; SM60-FTZ-ALLOW-NEXT: // %bb.0:
+; SM60-FTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fmax_acq_rel_bfloat_global_cta_param_1];
+; SM60-FTZ-ALLOW-NEXT: ld.param.b64 %rd2, [fmax_acq_rel_bfloat_global_cta_param_0];
+; SM60-FTZ-ALLOW-NEXT: membar.cta;
+; SM60-FTZ-ALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-FTZ-ALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-FTZ-ALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM60-FTZ-ALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-FTZ-ALLOW-NEXT: mov.b32 %r6, 65535;
+; SM60-FTZ-ALLOW-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-FTZ-ALLOW-NEXT: not.b32 %r2, %r7;
+; SM60-FTZ-ALLOW-NEXT: ld.volatile.global.b32 %r23, [%rd1];
+; SM60-FTZ-ALLOW-NEXT: cvt.u32.u16 %r10, %rs1;
+; SM60-FTZ-ALLOW-NEXT: shl.b32 %r11, %r10, 16;
+; SM60-FTZ-ALLOW-NEXT: $L__BB81_1: // %atomicrmw.start
+; SM60-FTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-FTZ-ALLOW-NEXT: shr.u32 %r8, %r23, %r1;
+; SM60-FTZ-ALLOW-NEXT: shl.b32 %r9, %r8, 16;
+; SM60-FTZ-ALLOW-NEXT: max.ftz.f32 %r12, %r9, %r11;
+; SM60-FTZ-ALLOW-NEXT: bfe.u32 %r13, %r12, 16, 1;
+; SM60-FTZ-ALLOW-NEXT: add.s32 %r14, %r13, %r12;
+; SM60-FTZ-ALLOW-NEXT: add.s32 %r15, %r14, 32767;
+; SM60-FTZ-ALLOW-NEXT: setp.nan.ftz.f32 %p1, %r12, %r12;
+; SM60-FTZ-ALLOW-NEXT: or.b32 %r16, %r12, 4194304;
+; SM60-FTZ-ALLOW-NEXT: selp.b32 %r17, %r16, %r15, %p1;
+; SM60-FTZ-ALLOW-NEXT: shr.u32 %r18, %r17, 16;
+; SM60-FTZ-ALLOW-NEXT: shl.b32 %r19, %r18, %r1;
+; SM60-FTZ-ALLOW-NEXT: and.b32 %r20, %r23, %r2;
+; SM60-FTZ-ALLOW-NEXT: or.b32 %r21, %r20, %r19;
+; SM60-FTZ-ALLOW-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r23, %r21;
+; SM60-FTZ-ALLOW-NEXT: setp.ne.b32 %p2, %r3, %r23;
+; SM60-FTZ-ALLOW-NEXT: mov.b32 %r23, %r3;
+; SM60-FTZ-ALLOW-NEXT: @%p2 bra $L__BB81_1;
+; SM60-FTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-FTZ-ALLOW-NEXT: shr.u32 %r22, %r3, %r1;
+; SM60-FTZ-ALLOW-NEXT: membar.cta;
+; SM60-FTZ-ALLOW-NEXT: st.param.b16 [func_retval0], %r22;
+; SM60-FTZ-ALLOW-NEXT: ret;
%retval = atomicrmw fmax ptr addrspace(1) %addr, bfloat %val syncscope("block") acq_rel
ret bfloat %retval
}
define bfloat @fminimum_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat %val) {
-; SM60-NOFTZ-LABEL: fminimum_acq_rel_bfloat_global_cta(
-; SM60-NOFTZ: {
-; SM60-NOFTZ-NEXT: .reg .pred %p<6>;
-; SM60-NOFTZ-NEXT: .reg .b16 %rs<7>;
-; SM60-NOFTZ-NEXT: .reg .b32 %r<18>;
-; SM60-NOFTZ-NEXT: .reg .b64 %rd<3>;
-; SM60-NOFTZ-EMPTY:
-; SM60-NOFTZ-NEXT: // %bb.0:
-; SM60-NOFTZ-NEXT: ld.param.b16 %rs1, [fminimum_acq_rel_bfloat_global_cta_param_1];
-; SM60-NOFTZ-NEXT: ld.param.b64 %rd2, [fminimum_acq_rel_bfloat_global_cta_param_0];
-; SM60-NOFTZ-NEXT: membar.cta;
-; SM60-NOFTZ-NEXT: and.b64 %rd1, %rd2, -4;
-; SM60-NOFTZ-NEXT: cvt.u32.u64 %r4, %rd2;
-; SM60-NOFTZ-NEXT: and.b32 %r5, %r4, 3;
-; SM60-NOFTZ-NEXT: shl.b32 %r1, %r5, 3;
-; SM60-NOFTZ-NEXT: mov.b32 %r6, 65535;
-; SM60-NOFTZ-NEXT: shl.b32 %r7, %r6, %r1;
-; SM60-NOFTZ-NEXT: not.b32 %r2, %r7;
-; SM60-NOFTZ-NEXT: ld.volatile.global.b32 %r17, [%rd1];
-; SM60-NOFTZ-NEXT: cvt.u32.u16 %r10, %rs1;
-; SM60-NOFTZ-NEXT: shl.b32 %r11, %r10, 16;
-; SM60-NOFTZ-NEXT: $L__BB82_1: // %atomicrmw.start
-; SM60-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NOFTZ-NEXT: shr.u32 %r8, %r17, %r1;
-; SM60-NOFTZ-NEXT: cvt.u16.u32 %rs2, %r8;
-; SM60-NOFTZ-NEXT: setp.eq.b16 %p1, %rs2, -32768;
-; SM60-NOFTZ-NEXT: selp.b16 %rs3, %rs2, %rs1, %p1;
-; SM60-NOFTZ-NEXT: shl.b32 %r9, %r8, 16;
-; SM60-NOFTZ-NEXT: setp.lt.f32 %p2, %r9, %r11;
-; SM60-NOFTZ-NEXT: selp.b16 %rs4, %rs2, %rs1, %p2;
-; SM60-NOFTZ-NEXT: setp.nan.f32 %p3, %r9, %r11;
-; SM60-NOFTZ-NEXT: selp.b16 %rs5, 0x7FC0, %rs4, %p3;
-; SM60-NOFTZ-NEXT: setp.eq.f32 %p4, %r9, %r11;
-; SM60-NOFTZ-NEXT: selp.b16 %rs6, %rs3, %rs5, %p4;
-; SM60-NOFTZ-NEXT: cvt.u32.u16 %r12, %rs6;
-; SM60-NOFTZ-NEXT: shl.b32 %r13, %r12, %r1;
-; SM60-NOFTZ-NEXT: and.b32 %r14, %r17, %r2;
-; SM60-NOFTZ-NEXT: or.b32 %r15, %r14, %r13;
-; SM60-NOFTZ-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r17, %r15;
-; SM60-NOFTZ-NEXT: setp.ne.b32 %p5, %r3, %r17;
-; SM60-NOFTZ-NEXT: mov.b32 %r17, %r3;
-; SM60-NOFTZ-NEXT: @%p5 bra $L__BB82_1;
-; SM60-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end
-; SM60-NOFTZ-NEXT: shr.u32 %r16, %r3, %r1;
-; SM60-NOFTZ-NEXT: membar.cta;
-; SM60-NOFTZ-NEXT: st.param.b16 [func_retval0], %r16;
-; SM60-NOFTZ-NEXT: ret;
+; SM60-NOFTZ-DISALLOW-LABEL: fminimum_acq_rel_bfloat_global_cta(
+; SM60-NOFTZ-DISALLOW: {
+; SM60-NOFTZ-DISALLOW-NEXT: .reg .pred %p<6>;
+; SM60-NOFTZ-DISALLOW-NEXT: .reg .b16 %rs<7>;
+; SM60-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<18>;
+; SM60-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
+; SM60-NOFTZ-DISALLOW-EMPTY:
+; SM60-NOFTZ-DISALLOW-NEXT: // %bb.0:
+; SM60-NOFTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fminimum_acq_rel_bfloat_global_cta_param_1];
+; SM60-NOFTZ-DISALLOW-NEXT: ld.param.b64 %rd2, [fminimum_acq_rel_bfloat_global_cta_param_0];
+; SM60-NOFTZ-DISALLOW-NEXT: membar.cta;
+; SM60-NOFTZ-DISALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-NOFTZ-DISALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM60-NOFTZ-DISALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-NOFTZ-DISALLOW-NEXT: mov.b32 %r6, 65535;
+; SM60-NOFTZ-DISALLOW-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-NOFTZ-DISALLOW-NEXT: not.b32 %r2, %r7;
+; SM60-NOFTZ-DISALLOW-NEXT: ld.volatile.global.b32 %r17, [%rd1];
+; SM60-NOFTZ-DISALLOW-NEXT: cvt.u32.u16 %r10, %rs1;
+; SM60-NOFTZ-DISALLOW-NEXT: shl.b32 %r11, %r10, 16;
+; SM60-NOFTZ-DISALLOW-NEXT: $L__BB82_1: // %atomicrmw.start
+; SM60-NOFTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NOFTZ-DISALLOW-NEXT: shr.u32 %r8, %r17, %r1;
+; SM60-NOFTZ-DISALLOW-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM60-NOFTZ-DISALLOW-NEXT: setp.eq.b16 %p1, %rs2, -32768;
+; SM60-NOFTZ-DISALLOW-NEXT: selp.b16 %rs3, %rs2, %rs1, %p1;
+; SM60-NOFTZ-DISALLOW-NEXT: shl.b32 %r9, %r8, 16;
+; SM60-NOFTZ-DISALLOW-NEXT: setp.lt.f32 %p2, %r9, %r11;
+; SM60-NOFTZ-DISALLOW-NEXT: selp.b16 %rs4, %rs2, %rs1, %p2;
+; SM60-NOFTZ-DISALLOW-NEXT: setp.nan.f32 %p3, %r9, %r11;
+; SM60-NOFTZ-DISALLOW-NEXT: selp.b16 %rs5, 0x7FC0, %rs4, %p3;
+; SM60-NOFTZ-DISALLOW-NEXT: setp.eq.f32 %p4, %r9, %r11;
+; SM60-NOFTZ-DISALLOW-NEXT: selp.b16 %rs6, %rs3, %rs5, %p4;
+; SM60-NOFTZ-DISALLOW-NEXT: cvt.u32.u16 %r12, %rs6;
+; SM60-NOFTZ-DISALLOW-NEXT: shl.b32 %r13, %r12, %r1;
+; SM60-NOFTZ-DISALLOW-NEXT: and.b32 %r14, %r17, %r2;
+; SM60-NOFTZ-DISALLOW-NEXT: or.b32 %r15, %r14, %r13;
+; SM60-NOFTZ-DISALLOW-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r17, %r15;
+; SM60-NOFTZ-DISALLOW-NEXT: setp.ne.b32 %p5, %r3, %r17;
+; SM60-NOFTZ-DISALLOW-NEXT: mov.b32 %r17, %r3;
+; SM60-NOFTZ-DISALLOW-NEXT: @%p5 bra $L__BB82_1;
+; SM60-NOFTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NOFTZ-DISALLOW-NEXT: shr.u32 %r16, %r3, %r1;
+; SM60-NOFTZ-DISALLOW-NEXT: membar.cta;
+; SM60-NOFTZ-DISALLOW-NEXT: st.param.b16 [func_retval0], %r16;
+; SM60-NOFTZ-DISALLOW-NEXT: ret;
;
-; SM60-FTZ-LABEL: fminimum_acq_rel_bfloat_global_cta(
-; SM60-FTZ: {
-; SM60-FTZ-NEXT: .reg .pred %p<6>;
-; SM60-FTZ-NEXT: .reg .b16 %rs<7>;
-; SM60-FTZ-NEXT: .reg .b32 %r<18>;
-; SM60-FTZ-NEXT: .reg .b64 %rd<3>;
-; SM60-FTZ-EMPTY:
-; SM60-FTZ-NEXT: // %bb.0:
-; SM60-FTZ-NEXT: ld.param.b16 %rs1, [fminimum_acq_rel_bfloat_global_cta_param_1];
-; SM60-FTZ-NEXT: ld.param.b64 %rd2, [fminimum_acq_rel_bfloat_global_cta_param_0];
-; SM60-FTZ-NEXT: membar.cta;
-; SM60-FTZ-NEXT: and.b64 %rd1, %rd2, -4;
-; SM60-FTZ-NEXT: cvt.u32.u64 %r4, %rd2;
-; SM60-FTZ-NEXT: and.b32 %r5, %r4, 3;
-; SM60-FTZ-NEXT: shl.b32 %r1, %r5, 3;
-; SM60-FTZ-NEXT: mov.b32 %r6, 65535;
-; SM60-FTZ-NEXT: shl.b32 %r7, %r6, %r1;
-; SM60-FTZ-NEXT: not.b32 %r2, %r7;
-; SM60-FTZ-NEXT: ld.volatile.global.b32 %r17, [%rd1];
-; SM60-FTZ-NEXT: cvt.u32.u16 %r10, %rs1;
-; SM60-FTZ-NEXT: shl.b32 %r11, %r10, 16;
-; SM60-FTZ-NEXT: $L__BB82_1: // %atomicrmw.start
-; SM60-FTZ-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-FTZ-NEXT: shr.u32 %r8, %r17, %r1;
-; SM60-FTZ-NEXT: cvt.u16.u32 %rs2, %r8;
-; SM60-FTZ-NEXT: setp.eq.b16 %p1, %rs2, -32768;
-; SM60-FTZ-NEXT: selp.b16 %rs3, %rs2, %rs1, %p1;
-; SM60-FTZ-NEXT: shl.b32 %r9, %r8, 16;
-; SM60-FTZ-NEXT: setp.lt.ftz.f32 %p2, %r9, %r11;
-; SM60-FTZ-NEXT: selp.b16 %rs4, %rs2, %rs1, %p2;
-; SM60-FTZ-NEXT: setp.nan.ftz.f32 %p3, %r9, %r11;
-; SM60-FTZ-NEXT: selp.b16 %rs5, 0x7FC0, %rs4, %p3;
-; SM60-FTZ-NEXT: setp.eq.ftz.f32 %p4, %r9, %r11;
-; SM60-FTZ-NEXT: selp.b16 %rs6, %rs3, %rs5, %p4;
-; SM60-FTZ-NEXT: cvt.u32.u16 %r12, %rs6;
-; SM60-FTZ-NEXT: shl.b32 %r13, %r12, %r1;
-; SM60-FTZ-NEXT: and.b32 %r14, %r17, %r2;
-; SM60-FTZ-NEXT: or.b32 %r15, %r14, %r13;
-; SM60-FTZ-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r17, %r15;
-; SM60-FTZ-NEXT: setp.ne.b32 %p5, %r3, %r17;
-; SM60-FTZ-NEXT: mov.b32 %r17, %r3;
-; SM60-FTZ-NEXT: @%p5 bra $L__BB82_1;
-; SM60-FTZ-NEXT: // %bb.2: // %atomicrmw.end
-; SM60-FTZ-NEXT: shr.u32 %r16, %r3, %r1;
-; SM60-FTZ-NEXT: membar.cta;
-; SM60-FTZ-NEXT: st.param.b16 [func_retval0], %r16;
-; SM60-FTZ-NEXT: ret;
+; SM60-NOFTZ-ALLOW-LABEL: fminimum_acq_rel_bfloat_global_cta(
+; SM60-NOFTZ-ALLOW: {
+; SM60-NOFTZ-ALLOW-NEXT: .reg .pred %p<6>;
+; SM60-NOFTZ-ALLOW-NEXT: .reg .b16 %rs<7>;
+; SM60-NOFTZ-ALLOW-NEXT: .reg .b32 %r<18>;
+; SM60-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
+; SM60-NOFTZ-ALLOW-EMPTY:
+; SM60-NOFTZ-ALLOW-NEXT: // %bb.0:
+; SM60-NOFTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fminimum_acq_rel_bfloat_global_cta_param_1];
+; SM60-NOFTZ-ALLOW-NEXT: ld.param.b64 %rd2, [fminimum_acq_rel_bfloat_global_cta_param_0];
+; SM60-NOFTZ-ALLOW-NEXT: membar.cta;
+; SM60-NOFTZ-ALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NOFTZ-ALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-NOFTZ-ALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM60-NOFTZ-ALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-NOFTZ-ALLOW-NEXT: mov.b32 %r6, 65535;
+; SM60-NOFTZ-ALLOW-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-NOFTZ-ALLOW-NEXT: not.b32 %r2, %r7;
+; SM60-NOFTZ-ALLOW-NEXT: ld.volatile.global.b32 %r17, [%rd1];
+; SM60-NOFTZ-ALLOW-NEXT: cvt.u32.u16 %r10, %rs1;
+; SM60-NOFTZ-ALLOW-NEXT: shl.b32 %r11, %r10, 16;
+; SM60-NOFTZ-ALLOW-NEXT: $L__BB82_1: // %atomicrmw.start
+; SM60-NOFTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NOFTZ-ALLOW-NEXT: shr.u32 %r8, %r17, %r1;
+; SM60-NOFTZ-ALLOW-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM60-NOFTZ-ALLOW-NEXT: setp.eq.b16 %p1, %rs2, -32768;
+; SM60-NOFTZ-ALLOW-NEXT: selp.b16 %rs3, %rs2, %rs1, %p1;
+; SM60-NOFTZ-ALLOW-NEXT: shl.b32 %r9, %r8, 16;
+; SM60-NOFTZ-ALLOW-NEXT: setp.lt.f32 %p2, %r9, %r11;
+; SM60-NOFTZ-ALLOW-NEXT: selp.b16 %rs4, %rs2, %rs1, %p2;
+; SM60-NOFTZ-ALLOW-NEXT: setp.nan.f32 %p3, %r9, %r11;
+; SM60-NOFTZ-ALLOW-NEXT: selp.b16 %rs5, 0x7FC0, %rs4, %p3;
+; SM60-NOFTZ-ALLOW-NEXT: setp.eq.f32 %p4, %r9, %r11;
+; SM60-NOFTZ-ALLOW-NEXT: selp.b16 %rs6, %rs3, %rs5, %p4;
+; SM60-NOFTZ-ALLOW-NEXT: cvt.u32.u16 %r12, %rs6;
+; SM60-NOFTZ-ALLOW-NEXT: shl.b32 %r13, %r12, %r1;
+; SM60-NOFTZ-ALLOW-NEXT: and.b32 %r14, %r17, %r2;
+; SM60-NOFTZ-ALLOW-NEXT: or.b32 %r15, %r14, %r13;
+; SM60-NOFTZ-ALLOW-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r17, %r15;
+; SM60-NOFTZ-ALLOW-NEXT: setp.ne.b32 %p5, %r3, %r17;
+; SM60-NOFTZ-ALLOW-NEXT: mov.b32 %r17, %r3;
+; SM60-NOFTZ-ALLOW-NEXT: @%p5 bra $L__BB82_1;
+; SM60-NOFTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NOFTZ-ALLOW-NEXT: shr.u32 %r16, %r3, %r1;
+; SM60-NOFTZ-ALLOW-NEXT: membar.cta;
+; SM60-NOFTZ-ALLOW-NEXT: st.param.b16 [func_retval0], %r16;
+; SM60-NOFTZ-ALLOW-NEXT: ret;
+;
+; SM60-FTZ-DISALLOW-LABEL: fminimum_acq_rel_bfloat_global_cta(
+; SM60-FTZ-DISALLOW: {
+; SM60-FTZ-DISALLOW-NEXT: .reg .pred %p<6>;
+; SM60-FTZ-DISALLOW-NEXT: .reg .b16 %rs<7>;
+; SM60-FTZ-DISALLOW-NEXT: .reg .b32 %r<18>;
+; SM60-FTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
+; SM60-FTZ-DISALLOW-EMPTY:
+; SM60-FTZ-DISALLOW-NEXT: // %bb.0:
+; SM60-FTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fminimum_acq_rel_bfloat_global_cta_param_1];
+; SM60-FTZ-DISALLOW-NEXT: ld.param.b64 %rd2, [fminimum_acq_rel_bfloat_global_cta_param_0];
+; SM60-FTZ-DISALLOW-NEXT: membar.cta;
+; SM60-FTZ-DISALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-FTZ-DISALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-FTZ-DISALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM60-FTZ-DISALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-FTZ-DISALLOW-NEXT: mov.b32 %r6, 65535;
+; SM60-FTZ-DISALLOW-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-FTZ-DISALLOW-NEXT: not.b32 %r2, %r7;
+; SM60-FTZ-DISALLOW-NEXT: ld.volatile.global.b32 %r17, [%rd1];
+; SM60-FTZ-DISALLOW-NEXT: cvt.u32.u16 %r10, %rs1;
+; SM60-FTZ-DISALLOW-NEXT: shl.b32 %r11, %r10, 16;
+; SM60-FTZ-DISALLOW-NEXT: $L__BB82_1: // %atomicrmw.start
+; SM60-FTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-FTZ-DISALLOW-NEXT: shr.u32 %r8, %r17, %r1;
+; SM60-FTZ-DISALLOW-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM60-FTZ-DISALLOW-NEXT: setp.eq.b16 %p1, %rs2, -32768;
+; SM60-FTZ-DISALLOW-NEXT: selp.b16 %rs3, %rs2, %rs1, %p1;
+; SM60-FTZ-DISALLOW-NEXT: shl.b32 %r9, %r8, 16;
+; SM60-FTZ-DISALLOW-NEXT: setp.lt.ftz.f32 %p2, %r9, %r11;
+; SM60-FTZ-DISALLOW-NEXT: selp.b16 %rs4, %rs2, %rs1, %p2;
+; SM60-FTZ-DISALLOW-NEXT: setp.nan.ftz.f32 %p3, %r9, %r11;
+; SM60-FTZ-DISALLOW-NEXT: selp.b16 %rs5, 0x7FC0, %rs4, %p3;
+; SM60-FTZ-DISALLOW-NEXT: setp.eq.ftz.f32 %p4, %r9, %r11;
+; SM60-FTZ-DISALLOW-NEXT: selp.b16 %rs6, %rs3, %rs5, %p4;
+; SM60-FTZ-DISALLOW-NEXT: cvt.u32.u16 %r12, %rs6;
+; SM60-FTZ-DISALLOW-NEXT: shl.b32 %r13, %r12, %r1;
+; SM60-FTZ-DISALLOW-NEXT: and.b32 %r14, %r17, %r2;
+; SM60-FTZ-DISALLOW-NEXT: or.b32 %r15, %r14, %r13;
+; SM60-FTZ-DISALLOW-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r17, %r15;
+; SM60-FTZ-DISALLOW-NEXT: setp.ne.b32 %p5, %r3, %r17;
+; SM60-FTZ-DISALLOW-NEXT: mov.b32 %r17, %r3;
+; SM60-FTZ-DISALLOW-NEXT: @%p5 bra $L__BB82_1;
+; SM60-FTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-FTZ-DISALLOW-NEXT: shr.u32 %r16, %r3, %r1;
+; SM60-FTZ-DISALLOW-NEXT: membar.cta;
+; SM60-FTZ-DISALLOW-NEXT: st.param.b16 [func_retval0], %r16;
+; SM60-FTZ-DISALLOW-NEXT: ret;
+;
+; SM60-FTZ-ALLOW-LABEL: fminimum_acq_rel_bfloat_global_cta(
+; SM60-FTZ-ALLOW: {
+; SM60-FTZ-ALLOW-NEXT: .reg .pred %p<6>;
+; SM60-FTZ-ALLOW-NEXT: .reg .b16 %rs<7>;
+; SM60-FTZ-ALLOW-NEXT: .reg .b32 %r<18>;
+; SM60-FTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
+; SM60-FTZ-ALLOW-EMPTY:
+; SM60-FTZ-ALLOW-NEXT: // %bb.0:
+; SM60-FTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fminimum_acq_rel_bfloat_global_cta_param_1];
+; SM60-FTZ-ALLOW-NEXT: ld.param.b64 %rd2, [fminimum_acq_rel_bfloat_global_cta_param_0];
+; SM60-FTZ-ALLOW-NEXT: membar.cta;
+; SM60-FTZ-ALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-FTZ-ALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-FTZ-ALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM60-FTZ-ALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-FTZ-ALLOW-NEXT: mov.b32 %r6, 65535;
+; SM60-FTZ-ALLOW-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-FTZ-ALLOW-NEXT: not.b32 %r2, %r7;
+; SM60-FTZ-ALLOW-NEXT: ld.volatile.global.b32 %r17, [%rd1];
+; SM60-FTZ-ALLOW-NEXT: cvt.u32.u16 %r10, %rs1;
+; SM60-FTZ-ALLOW-NEXT: shl.b32 %r11, %r10, 16;
+; SM60-FTZ-ALLOW-NEXT: $L__BB82_1: // %atomicrmw.start
+; SM60-FTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-FTZ-ALLOW-NEXT: shr.u32 %r8, %r17, %r1;
+; SM60-FTZ-ALLOW-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM60-FTZ-ALLOW-NEXT: setp.eq.b16 %p1, %rs2, -32768;
+; SM60-FTZ-ALLOW-NEXT: selp.b16 %rs3, %rs2, %rs1, %p1;
+; SM60-FTZ-ALLOW-NEXT: shl.b32 %r9, %r8, 16;
+; SM60-FTZ-ALLOW-NEXT: setp.lt.ftz.f32 %p2, %r9, %r11;
+; SM60-FTZ-ALLOW-NEXT: selp.b16 %rs4, %rs2, %rs1, %p2;
+; SM60-FTZ-ALLOW-NEXT: setp.nan.ftz.f32 %p3, %r9, %r11;
+; SM60-FTZ-ALLOW-NEXT: selp.b16 %rs5, 0x7FC0, %rs4, %p3;
+; SM60-FTZ-ALLOW-NEXT: setp.eq.ftz.f32 %p4, %r9, %r11;
+; SM60-FTZ-ALLOW-NEXT: selp.b16 %rs6, %rs3, %rs5, %p4;
+; SM60-FTZ-ALLOW-NEXT: cvt.u32.u16 %r12, %rs6;
+; SM60-FTZ-ALLOW-NEXT: shl.b32 %r13, %r12, %r1;
+; SM60-FTZ-ALLOW-NEXT: and.b32 %r14, %r17, %r2;
+; SM60-FTZ-ALLOW-NEXT: or.b32 %r15, %r14, %r13;
+; SM60-FTZ-ALLOW-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r17, %r15;
+; SM60-FTZ-ALLOW-NEXT: setp.ne.b32 %p5, %r3, %r17;
+; SM60-FTZ-ALLOW-NEXT: mov.b32 %r17, %r3;
+; SM60-FTZ-ALLOW-NEXT: @%p5 bra $L__BB82_1;
+; SM60-FTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-FTZ-ALLOW-NEXT: shr.u32 %r16, %r3, %r1;
+; SM60-FTZ-ALLOW-NEXT: membar.cta;
+; SM60-FTZ-ALLOW-NEXT: st.param.b16 [func_retval0], %r16;
+; SM60-FTZ-ALLOW-NEXT: ret;
%retval = atomicrmw fminimum ptr addrspace(1) %addr, bfloat %val syncscope("block") acq_rel
ret bfloat %retval
}
define bfloat @fmaximum_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat %val) {
-; SM60-NOFTZ-LABEL: fmaximum_acq_rel_bfloat_global_cta(
-; SM60-NOFTZ: {
-; SM60-NOFTZ-NEXT: .reg .pred %p<6>;
-; SM60-NOFTZ-NEXT: .reg .b16 %rs<7>;
-; SM60-NOFTZ-NEXT: .reg .b32 %r<18>;
-; SM60-NOFTZ-NEXT: .reg .b64 %rd<3>;
-; SM60-NOFTZ-EMPTY:
-; SM60-NOFTZ-NEXT: // %bb.0:
-; SM60-NOFTZ-NEXT: ld.param.b16 %rs1, [fmaximum_acq_rel_bfloat_global_cta_param_1];
-; SM60-NOFTZ-NEXT: ld.param.b64 %rd2, [fmaximum_acq_rel_bfloat_global_cta_param_0];
-; SM60-NOFTZ-NEXT: membar.cta;
-; SM60-NOFTZ-NEXT: and.b64 %rd1, %rd2, -4;
-; SM60-NOFTZ-NEXT: cvt.u32.u64 %r4, %rd2;
-; SM60-NOFTZ-NEXT: and.b32 %r5, %r4, 3;
-; SM60-NOFTZ-NEXT: shl.b32 %r1, %r5, 3;
-; SM60-NOFTZ-NEXT: mov.b32 %r6, 65535;
-; SM60-NOFTZ-NEXT: shl.b32 %r7, %r6, %r1;
-; SM60-NOFTZ-NEXT: not.b32 %r2, %r7;
-; SM60-NOFTZ-NEXT: ld.volatile.global.b32 %r17, [%rd1];
-; SM60-NOFTZ-NEXT: cvt.u32.u16 %r10, %rs1;
-; SM60-NOFTZ-NEXT: shl.b32 %r11, %r10, 16;
-; SM60-NOFTZ-NEXT: $L__BB83_1: // %atomicrmw.start
-; SM60-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NOFTZ-NEXT: shr.u32 %r8, %r17, %r1;
-; SM60-NOFTZ-NEXT: cvt.u16.u32 %rs2, %r8;
-; SM60-NOFTZ-NEXT: setp.eq.b16 %p1, %rs2, 0;
-; SM60-NOFTZ-NEXT: selp.b16 %rs3, %rs2, %rs1, %p1;
-; SM60-NOFTZ-NEXT: shl.b32 %r9, %r8, 16;
-; SM60-NOFTZ-NEXT: setp.gt.f32 %p2, %r9, %r11;
-; SM60-NOFTZ-NEXT: selp.b16 %rs4, %rs2, %rs1, %p2;
-; SM60-NOFTZ-NEXT: setp.nan.f32 %p3, %r9, %r11;
-; SM60-NOFTZ-NEXT: selp.b16 %rs5, 0x7FC0, %rs4, %p3;
-; SM60-NOFTZ-NEXT: setp.eq.f32 %p4, %r9, %r11;
-; SM60-NOFTZ-NEXT: selp.b16 %rs6, %rs3, %rs5, %p4;
-; SM60-NOFTZ-NEXT: cvt.u32.u16 %r12, %rs6;
-; SM60-NOFTZ-NEXT: shl.b32 %r13, %r12, %r1;
-; SM60-NOFTZ-NEXT: and.b32 %r14, %r17, %r2;
-; SM60-NOFTZ-NEXT: or.b32 %r15, %r14, %r13;
-; SM60-NOFTZ-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r17, %r15;
-; SM60-NOFTZ-NEXT: setp.ne.b32 %p5, %r3, %r17;
-; SM60-NOFTZ-NEXT: mov.b32 %r17, %r3;
-; SM60-NOFTZ-NEXT: @%p5 bra $L__BB83_1;
-; SM60-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end
-; SM60-NOFTZ-NEXT: shr.u32 %r16, %r3, %r1;
-; SM60-NOFTZ-NEXT: membar.cta;
-; SM60-NOFTZ-NEXT: st.param.b16 [func_retval0], %r16;
-; SM60-NOFTZ-NEXT: ret;
+; SM60-NOFTZ-DISALLOW-LABEL: fmaximum_acq_rel_bfloat_global_cta(
+; SM60-NOFTZ-DISALLOW: {
+; SM60-NOFTZ-DISALLOW-NEXT: .reg .pred %p<6>;
+; SM60-NOFTZ-DISALLOW-NEXT: .reg .b16 %rs<7>;
+; SM60-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<18>;
+; SM60-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
+; SM60-NOFTZ-DISALLOW-EMPTY:
+; SM60-NOFTZ-DISALLOW-NEXT: // %bb.0:
+; SM60-NOFTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fmaximum_acq_rel_bfloat_global_cta_param_1];
+; SM60-NOFTZ-DISALLOW-NEXT: ld.param.b64 %rd2, [fmaximum_acq_rel_bfloat_global_cta_param_0];
+; SM60-NOFTZ-DISALLOW-NEXT: membar.cta;
+; SM60-NOFTZ-DISALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-NOFTZ-DISALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM60-NOFTZ-DISALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-NOFTZ-DISALLOW-NEXT: mov.b32 %r6, 65535;
+; SM60-NOFTZ-DISALLOW-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-NOFTZ-DISALLOW-NEXT: not.b32 %r2, %r7;
+; SM60-NOFTZ-DISALLOW-NEXT: ld.volatile.global.b32 %r17, [%rd1];
+; SM60-NOFTZ-DISALLOW-NEXT: cvt.u32.u16 %r10, %rs1;
+; SM60-NOFTZ-DISALLOW-NEXT: shl.b32 %r11, %r10, 16;
+; SM60-NOFTZ-DISALLOW-NEXT: $L__BB83_1: // %atomicrmw.start
+; SM60-NOFTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NOFTZ-DISALLOW-NEXT: shr.u32 %r8, %r17, %r1;
+; SM60-NOFTZ-DISALLOW-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM60-NOFTZ-DISALLOW-NEXT: setp.eq.b16 %p1, %rs2, 0;
+; SM60-NOFTZ-DISALLOW-NEXT: selp.b16 %rs3, %rs2, %rs1, %p1;
+; SM60-NOFTZ-DISALLOW-NEXT: shl.b32 %r9, %r8, 16;
+; SM60-NOFTZ-DISALLOW-NEXT: setp.gt.f32 %p2, %r9, %r11;
+; SM60-NOFTZ-DISALLOW-NEXT: selp.b16 %rs4, %rs2, %rs1, %p2;
+; SM60-NOFTZ-DISALLOW-NEXT: setp.nan.f32 %p3, %r9, %r11;
+; SM60-NOFTZ-DISALLOW-NEXT: selp.b16 %rs5, 0x7FC0, %rs4, %p3;
+; SM60-NOFTZ-DISALLOW-NEXT: setp.eq.f32 %p4, %r9, %r11;
+; SM60-NOFTZ-DISALLOW-NEXT: selp.b16 %rs6, %rs3, %rs5, %p4;
+; SM60-NOFTZ-DISALLOW-NEXT: cvt.u32.u16 %r12, %rs6;
+; SM60-NOFTZ-DISALLOW-NEXT: shl.b32 %r13, %r12, %r1;
+; SM60-NOFTZ-DISALLOW-NEXT: and.b32 %r14, %r17, %r2;
+; SM60-NOFTZ-DISALLOW-NEXT: or.b32 %r15, %r14, %r13;
+; SM60-NOFTZ-DISALLOW-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r17, %r15;
+; SM60-NOFTZ-DISALLOW-NEXT: setp.ne.b32 %p5, %r3, %r17;
+; SM60-NOFTZ-DISALLOW-NEXT: mov.b32 %r17, %r3;
+; SM60-NOFTZ-DISALLOW-NEXT: @%p5 bra $L__BB83_1;
+; SM60-NOFTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NOFTZ-DISALLOW-NEXT: shr.u32 %r16, %r3, %r1;
+; SM60-NOFTZ-DISALLOW-NEXT: membar.cta;
+; SM60-NOFTZ-DISALLOW-NEXT: st.param.b16 [func_retval0], %r16;
+; SM60-NOFTZ-DISALLOW-NEXT: ret;
+;
+; SM60-NOFTZ-ALLOW-LABEL: fmaximum_acq_rel_bfloat_global_cta(
+; SM60-NOFTZ-ALLOW: {
+; SM60-NOFTZ-ALLOW-NEXT: .reg .pred %p<6>;
+; SM60-NOFTZ-ALLOW-NEXT: .reg .b16 %rs<7>;
+; SM60-NOFTZ-ALLOW-NEXT: .reg .b32 %r<18>;
+; SM60-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
+; SM60-NOFTZ-ALLOW-EMPTY:
+; SM60-NOFTZ-ALLOW-NEXT: // %bb.0:
+; SM60-NOFTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fmaximum_acq_rel_bfloat_global_cta_param_1];
+; SM60-NOFTZ-ALLOW-NEXT: ld.param.b64 %rd2, [fmaximum_acq_rel_bfloat_global_cta_param_0];
+; SM60-NOFTZ-ALLOW-NEXT: membar.cta;
+; SM60-NOFTZ-ALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-NOFTZ-ALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-NOFTZ-ALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM60-NOFTZ-ALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-NOFTZ-ALLOW-NEXT: mov.b32 %r6, 65535;
+; SM60-NOFTZ-ALLOW-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-NOFTZ-ALLOW-NEXT: not.b32 %r2, %r7;
+; SM60-NOFTZ-ALLOW-NEXT: ld.volatile.global.b32 %r17, [%rd1];
+; SM60-NOFTZ-ALLOW-NEXT: cvt.u32.u16 %r10, %rs1;
+; SM60-NOFTZ-ALLOW-NEXT: shl.b32 %r11, %r10, 16;
+; SM60-NOFTZ-ALLOW-NEXT: $L__BB83_1: // %atomicrmw.start
+; SM60-NOFTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NOFTZ-ALLOW-NEXT: shr.u32 %r8, %r17, %r1;
+; SM60-NOFTZ-ALLOW-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM60-NOFTZ-ALLOW-NEXT: setp.eq.b16 %p1, %rs2, 0;
+; SM60-NOFTZ-ALLOW-NEXT: selp.b16 %rs3, %rs2, %rs1, %p1;
+; SM60-NOFTZ-ALLOW-NEXT: shl.b32 %r9, %r8, 16;
+; SM60-NOFTZ-ALLOW-NEXT: setp.gt.f32 %p2, %r9, %r11;
+; SM60-NOFTZ-ALLOW-NEXT: selp.b16 %rs4, %rs2, %rs1, %p2;
+; SM60-NOFTZ-ALLOW-NEXT: setp.nan.f32 %p3, %r9, %r11;
+; SM60-NOFTZ-ALLOW-NEXT: selp.b16 %rs5, 0x7FC0, %rs4, %p3;
+; SM60-NOFTZ-ALLOW-NEXT: setp.eq.f32 %p4, %r9, %r11;
+; SM60-NOFTZ-ALLOW-NEXT: selp.b16 %rs6, %rs3, %rs5, %p4;
+; SM60-NOFTZ-ALLOW-NEXT: cvt.u32.u16 %r12, %rs6;
+; SM60-NOFTZ-ALLOW-NEXT: shl.b32 %r13, %r12, %r1;
+; SM60-NOFTZ-ALLOW-NEXT: and.b32 %r14, %r17, %r2;
+; SM60-NOFTZ-ALLOW-NEXT: or.b32 %r15, %r14, %r13;
+; SM60-NOFTZ-ALLOW-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r17, %r15;
+; SM60-NOFTZ-ALLOW-NEXT: setp.ne.b32 %p5, %r3, %r17;
+; SM60-NOFTZ-ALLOW-NEXT: mov.b32 %r17, %r3;
+; SM60-NOFTZ-ALLOW-NEXT: @%p5 bra $L__BB83_1;
+; SM60-NOFTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NOFTZ-ALLOW-NEXT: shr.u32 %r16, %r3, %r1;
+; SM60-NOFTZ-ALLOW-NEXT: membar.cta;
+; SM60-NOFTZ-ALLOW-NEXT: st.param.b16 [func_retval0], %r16;
+; SM60-NOFTZ-ALLOW-NEXT: ret;
;
-; SM60-FTZ-LABEL: fmaximum_acq_rel_bfloat_global_cta(
-; SM60-FTZ: {
-; SM60-FTZ-NEXT: .reg .pred %p<6>;
-; SM60-FTZ-NEXT: .reg .b16 %rs<7>;
-; SM60-FTZ-NEXT: .reg .b32 %r<18>;
-; SM60-FTZ-NEXT: .reg .b64 %rd<3>;
-; SM60-FTZ-EMPTY:
-; SM60-FTZ-NEXT: // %bb.0:
-; SM60-FTZ-NEXT: ld.param.b16 %rs1, [fmaximum_acq_rel_bfloat_global_cta_param_1];
-; SM60-FTZ-NEXT: ld.param.b64 %rd2, [fmaximum_acq_rel_bfloat_global_cta_param_0];
-; SM60-FTZ-NEXT: membar.cta;
-; SM60-FTZ-NEXT: and.b64 %rd1, %rd2, -4;
-; SM60-FTZ-NEXT: cvt.u32.u64 %r4, %rd2;
-; SM60-FTZ-NEXT: and.b32 %r5, %r4, 3;
-; SM60-FTZ-NEXT: shl.b32 %r1, %r5, 3;
-; SM60-FTZ-NEXT: mov.b32 %r6, 65535;
-; SM60-FTZ-NEXT: shl.b32 %r7, %r6, %r1;
-; SM60-FTZ-NEXT: not.b32 %r2, %r7;
-; SM60-FTZ-NEXT: ld.volatile.global.b32 %r17, [%rd1];
-; SM60-FTZ-NEXT: cvt.u32.u16 %r10, %rs1;
-; SM60-FTZ-NEXT: shl.b32 %r11, %r10, 16;
-; SM60-FTZ-NEXT: $L__BB83_1: // %atomicrmw.start
-; SM60-FTZ-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-FTZ-NEXT: shr.u32 %r8, %r17, %r1;
-; SM60-FTZ-NEXT: cvt.u16.u32 %rs2, %r8;
-; SM60-FTZ-NEXT: setp.eq.b16 %p1, %rs2, 0;
-; SM60-FTZ-NEXT: selp.b16 %rs3, %rs2, %rs1, %p1;
-; SM60-FTZ-NEXT: shl.b32 %r9, %r8, 16;
-; SM60-FTZ-NEXT: setp.gt.ftz.f32 %p2, %r9, %r11;
-; SM60-FTZ-NEXT: selp.b16 %rs4, %rs2, %rs1, %p2;
-; SM60-FTZ-NEXT: setp.nan.ftz.f32 %p3, %r9, %r11;
-; SM60-FTZ-NEXT: selp.b16 %rs5, 0x7FC0, %rs4, %p3;
-; SM60-FTZ-NEXT: setp.eq.ftz.f32 %p4, %r9, %r11;
-; SM60-FTZ-NEXT: selp.b16 %rs6, %rs3, %rs5, %p4;
-; SM60-FTZ-NEXT: cvt.u32.u16 %r12, %rs6;
-; SM60-FTZ-NEXT: shl.b32 %r13, %r12, %r1;
-; SM60-FTZ-NEXT: and.b32 %r14, %r17, %r2;
-; SM60-FTZ-NEXT: or.b32 %r15, %r14, %r13;
-; SM60-FTZ-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r17, %r15;
-; SM60-FTZ-NEXT: setp.ne.b32 %p5, %r3, %r17;
-; SM60-FTZ-NEXT: mov.b32 %r17, %r3;
-; SM60-FTZ-NEXT: @%p5 bra $L__BB83_1;
-; SM60-FTZ-NEXT: // %bb.2: // %atomicrmw.end
-; SM60-FTZ-NEXT: shr.u32 %r16, %r3, %r1;
-; SM60-FTZ-NEXT: membar.cta;
-; SM60-FTZ-NEXT: st.param.b16 [func_retval0], %r16;
-; SM60-FTZ-NEXT: ret;
+; SM60-FTZ-DISALLOW-LABEL: fmaximum_acq_rel_bfloat_global_cta(
+; SM60-FTZ-DISALLOW: {
+; SM60-FTZ-DISALLOW-NEXT: .reg .pred %p<6>;
+; SM60-FTZ-DISALLOW-NEXT: .reg .b16 %rs<7>;
+; SM60-FTZ-DISALLOW-NEXT: .reg .b32 %r<18>;
+; SM60-FTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
+; SM60-FTZ-DISALLOW-EMPTY:
+; SM60-FTZ-DISALLOW-NEXT: // %bb.0:
+; SM60-FTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fmaximum_acq_rel_bfloat_global_cta_param_1];
+; SM60-FTZ-DISALLOW-NEXT: ld.param.b64 %rd2, [fmaximum_acq_rel_bfloat_global_cta_param_0];
+; SM60-FTZ-DISALLOW-NEXT: membar.cta;
+; SM60-FTZ-DISALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-FTZ-DISALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-FTZ-DISALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM60-FTZ-DISALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-FTZ-DISALLOW-NEXT: mov.b32 %r6, 65535;
+; SM60-FTZ-DISALLOW-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-FTZ-DISALLOW-NEXT: not.b32 %r2, %r7;
+; SM60-FTZ-DISALLOW-NEXT: ld.volatile.global.b32 %r17, [%rd1];
+; SM60-FTZ-DISALLOW-NEXT: cvt.u32.u16 %r10, %rs1;
+; SM60-FTZ-DISALLOW-NEXT: shl.b32 %r11, %r10, 16;
+; SM60-FTZ-DISALLOW-NEXT: $L__BB83_1: // %atomicrmw.start
+; SM60-FTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-FTZ-DISALLOW-NEXT: shr.u32 %r8, %r17, %r1;
+; SM60-FTZ-DISALLOW-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM60-FTZ-DISALLOW-NEXT: setp.eq.b16 %p1, %rs2, 0;
+; SM60-FTZ-DISALLOW-NEXT: selp.b16 %rs3, %rs2, %rs1, %p1;
+; SM60-FTZ-DISALLOW-NEXT: shl.b32 %r9, %r8, 16;
+; SM60-FTZ-DISALLOW-NEXT: setp.gt.ftz.f32 %p2, %r9, %r11;
+; SM60-FTZ-DISALLOW-NEXT: selp.b16 %rs4, %rs2, %rs1, %p2;
+; SM60-FTZ-DISALLOW-NEXT: setp.nan.ftz.f32 %p3, %r9, %r11;
+; SM60-FTZ-DISALLOW-NEXT: selp.b16 %rs5, 0x7FC0, %rs4, %p3;
+; SM60-FTZ-DISALLOW-NEXT: setp.eq.ftz.f32 %p4, %r9, %r11;
+; SM60-FTZ-DISALLOW-NEXT: selp.b16 %rs6, %rs3, %rs5, %p4;
+; SM60-FTZ-DISALLOW-NEXT: cvt.u32.u16 %r12, %rs6;
+; SM60-FTZ-DISALLOW-NEXT: shl.b32 %r13, %r12, %r1;
+; SM60-FTZ-DISALLOW-NEXT: and.b32 %r14, %r17, %r2;
+; SM60-FTZ-DISALLOW-NEXT: or.b32 %r15, %r14, %r13;
+; SM60-FTZ-DISALLOW-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r17, %r15;
+; SM60-FTZ-DISALLOW-NEXT: setp.ne.b32 %p5, %r3, %r17;
+; SM60-FTZ-DISALLOW-NEXT: mov.b32 %r17, %r3;
+; SM60-FTZ-DISALLOW-NEXT: @%p5 bra $L__BB83_1;
+; SM60-FTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-FTZ-DISALLOW-NEXT: shr.u32 %r16, %r3, %r1;
+; SM60-FTZ-DISALLOW-NEXT: membar.cta;
+; SM60-FTZ-DISALLOW-NEXT: st.param.b16 [func_retval0], %r16;
+; SM60-FTZ-DISALLOW-NEXT: ret;
+;
+; SM60-FTZ-ALLOW-LABEL: fmaximum_acq_rel_bfloat_global_cta(
+; SM60-FTZ-ALLOW: {
+; SM60-FTZ-ALLOW-NEXT: .reg .pred %p<6>;
+; SM60-FTZ-ALLOW-NEXT: .reg .b16 %rs<7>;
+; SM60-FTZ-ALLOW-NEXT: .reg .b32 %r<18>;
+; SM60-FTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
+; SM60-FTZ-ALLOW-EMPTY:
+; SM60-FTZ-ALLOW-NEXT: // %bb.0:
+; SM60-FTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fmaximum_acq_rel_bfloat_global_cta_param_1];
+; SM60-FTZ-ALLOW-NEXT: ld.param.b64 %rd2, [fmaximum_acq_rel_bfloat_global_cta_param_0];
+; SM60-FTZ-ALLOW-NEXT: membar.cta;
+; SM60-FTZ-ALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM60-FTZ-ALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM60-FTZ-ALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM60-FTZ-ALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM60-FTZ-ALLOW-NEXT: mov.b32 %r6, 65535;
+; SM60-FTZ-ALLOW-NEXT: shl.b32 %r7, %r6, %r1;
+; SM60-FTZ-ALLOW-NEXT: not.b32 %r2, %r7;
+; SM60-FTZ-ALLOW-NEXT: ld.volatile.global.b32 %r17, [%rd1];
+; SM60-FTZ-ALLOW-NEXT: cvt.u32.u16 %r10, %rs1;
+; SM60-FTZ-ALLOW-NEXT: shl.b32 %r11, %r10, 16;
+; SM60-FTZ-ALLOW-NEXT: $L__BB83_1: // %atomicrmw.start
+; SM60-FTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-FTZ-ALLOW-NEXT: shr.u32 %r8, %r17, %r1;
+; SM60-FTZ-ALLOW-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM60-FTZ-ALLOW-NEXT: setp.eq.b16 %p1, %rs2, 0;
+; SM60-FTZ-ALLOW-NEXT: selp.b16 %rs3, %rs2, %rs1, %p1;
+; SM60-FTZ-ALLOW-NEXT: shl.b32 %r9, %r8, 16;
+; SM60-FTZ-ALLOW-NEXT: setp.gt.ftz.f32 %p2, %r9, %r11;
+; SM60-FTZ-ALLOW-NEXT: selp.b16 %rs4, %rs2, %rs1, %p2;
+; SM60-FTZ-ALLOW-NEXT: setp.nan.ftz.f32 %p3, %r9, %r11;
+; SM60-FTZ-ALLOW-NEXT: selp.b16 %rs5, 0x7FC0, %rs4, %p3;
+; SM60-FTZ-ALLOW-NEXT: setp.eq.ftz.f32 %p4, %r9, %r11;
+; SM60-FTZ-ALLOW-NEXT: selp.b16 %rs6, %rs3, %rs5, %p4;
+; SM60-FTZ-ALLOW-NEXT: cvt.u32.u16 %r12, %rs6;
+; SM60-FTZ-ALLOW-NEXT: shl.b32 %r13, %r12, %r1;
+; SM60-FTZ-ALLOW-NEXT: and.b32 %r14, %r17, %r2;
+; SM60-FTZ-ALLOW-NEXT: or.b32 %r15, %r14, %r13;
+; SM60-FTZ-ALLOW-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r17, %r15;
+; SM60-FTZ-ALLOW-NEXT: setp.ne.b32 %p5, %r3, %r17;
+; SM60-FTZ-ALLOW-NEXT: mov.b32 %r17, %r3;
+; SM60-FTZ-ALLOW-NEXT: @%p5 bra $L__BB83_1;
+; SM60-FTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-FTZ-ALLOW-NEXT: shr.u32 %r16, %r3, %r1;
+; SM60-FTZ-ALLOW-NEXT: membar.cta;
+; SM60-FTZ-ALLOW-NEXT: st.param.b16 [func_retval0], %r16;
+; SM60-FTZ-ALLOW-NEXT: ret;
%retval = atomicrmw fmaximum ptr addrspace(1) %addr, bfloat %val syncscope("block") acq_rel
ret bfloat %retval
}
@@ -3912,40 +5254,64 @@ define i64 @nand_acq_rel_i64_global_sys(ptr addrspace(1) %addr, i64 %val) {
}
define float @fadd_acq_rel_float_global_sys(ptr addrspace(1) %addr, float %val) {
-; SM60-NOFTZ-LABEL: fadd_acq_rel_float_global_sys(
-; SM60-NOFTZ: {
-; SM60-NOFTZ-NEXT: .reg .pred %p<2>;
-; SM60-NOFTZ-NEXT: .reg .b32 %r<5>;
-; SM60-NOFTZ-NEXT: .reg .b64 %rd<2>;
-; SM60-NOFTZ-EMPTY:
-; SM60-NOFTZ-NEXT: // %bb.0:
-; SM60-NOFTZ-NEXT: ld.param.b32 %r2, [fadd_acq_rel_float_global_sys_param_1];
-; SM60-NOFTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_sys_param_0];
-; SM60-NOFTZ-NEXT: membar.sys;
-; SM60-NOFTZ-NEXT: ld.volatile.global.b32 %r4, [%rd1];
-; SM60-NOFTZ-NEXT: $L__BB106_1: // %atomicrmw.start
-; SM60-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NOFTZ-NEXT: add.rn.f32 %r3, %r4, %r2;
-; SM60-NOFTZ-NEXT: atom.sys.global.cas.b32 %r1, [%rd1], %r4, %r3;
-; SM60-NOFTZ-NEXT: setp.ne.b32 %p1, %r1, %r4;
-; SM60-NOFTZ-NEXT: mov.b32 %r4, %r1;
-; SM60-NOFTZ-NEXT: @%p1 bra $L__BB106_1;
-; SM60-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end
-; SM60-NOFTZ-NEXT: membar.sys;
-; SM60-NOFTZ-NEXT: st.param.b32 [func_retval0], %r1;
-; SM60-NOFTZ-NEXT: ret;
+; SM60-NOFTZ-DISALLOW-LABEL: fadd_acq_rel_float_global_sys(
+; SM60-NOFTZ-DISALLOW: {
+; SM60-NOFTZ-DISALLOW-NEXT: .reg .pred %p<2>;
+; SM60-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<5>;
+; SM60-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
+; SM60-NOFTZ-DISALLOW-EMPTY:
+; SM60-NOFTZ-DISALLOW-NEXT: // %bb.0:
+; SM60-NOFTZ-DISALLOW-NEXT: ld.param.b32 %r2, [fadd_acq_rel_float_global_sys_param_1];
+; SM60-NOFTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_sys_param_0];
+; SM60-NOFTZ-DISALLOW-NEXT: membar.sys;
+; SM60-NOFTZ-DISALLOW-NEXT: ld.volatile.global.b32 %r4, [%rd1];
+; SM60-NOFTZ-DISALLOW-NEXT: $L__BB106_1: // %atomicrmw.start
+; SM60-NOFTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NOFTZ-DISALLOW-NEXT: add.rn.f32 %r3, %r4, %r2;
+; SM60-NOFTZ-DISALLOW-NEXT: atom.sys.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM60-NOFTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM60-NOFTZ-DISALLOW-NEXT: mov.b32 %r4, %r1;
+; SM60-NOFTZ-DISALLOW-NEXT: @%p1 bra $L__BB106_1;
+; SM60-NOFTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NOFTZ-DISALLOW-NEXT: membar.sys;
+; SM60-NOFTZ-DISALLOW-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NOFTZ-DISALLOW-NEXT: ret;
+;
+; SM60-NOFTZ-ALLOW-LABEL: fadd_acq_rel_float_global_sys(
+; SM60-NOFTZ-ALLOW: {
+; SM60-NOFTZ-ALLOW-NEXT: .reg .b32 %r<3>;
+; SM60-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
+; SM60-NOFTZ-ALLOW-EMPTY:
+; SM60-NOFTZ-ALLOW-NEXT: // %bb.0:
+; SM60-NOFTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_sys_param_0];
+; SM60-NOFTZ-ALLOW-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_global_sys_param_1];
+; SM60-NOFTZ-ALLOW-NEXT: atom.sys.global.add.f32 %r2, [%rd1], %r1;
+; SM60-NOFTZ-ALLOW-NEXT: st.param.b32 [func_retval0], %r2;
+; SM60-NOFTZ-ALLOW-NEXT: ret;
+;
+; SM60-FTZ-DISALLOW-LABEL: fadd_acq_rel_float_global_sys(
+; SM60-FTZ-DISALLOW: {
+; SM60-FTZ-DISALLOW-NEXT: .reg .b32 %r<3>;
+; SM60-FTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
+; SM60-FTZ-DISALLOW-EMPTY:
+; SM60-FTZ-DISALLOW-NEXT: // %bb.0:
+; SM60-FTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_sys_param_0];
+; SM60-FTZ-DISALLOW-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_global_sys_param_1];
+; SM60-FTZ-DISALLOW-NEXT: atom.sys.global.add.f32 %r2, [%rd1], %r1;
+; SM60-FTZ-DISALLOW-NEXT: st.param.b32 [func_retval0], %r2;
+; SM60-FTZ-DISALLOW-NEXT: ret;
;
-; SM60-FTZ-LABEL: fadd_acq_rel_float_global_sys(
-; SM60-FTZ: {
-; SM60-FTZ-NEXT: .reg .b32 %r<3>;
-; SM60-FTZ-NEXT: .reg .b64 %rd<2>;
-; SM60-FTZ-EMPTY:
-; SM60-FTZ-NEXT: // %bb.0:
-; SM60-FTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_sys_param_0];
-; SM60-FTZ-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_global_sys_param_1];
-; SM60-FTZ-NEXT: atom.sys.global.add.f32 %r2, [%rd1], %r1;
-; SM60-FTZ-NEXT: st.param.b32 [func_retval0], %r2;
-; SM60-FTZ-NEXT: ret;
+; SM60-FTZ-ALLOW-LABEL: fadd_acq_rel_float_global_sys(
+; SM60-FTZ-ALLOW: {
+; SM60-FTZ-ALLOW-NEXT: .reg .b32 %r<3>;
+; SM60-FTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
+; SM60-FTZ-ALLOW-EMPTY:
+; SM60-FTZ-ALLOW-NEXT: // %bb.0:
+; SM60-FTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_sys_param_0];
+; SM60-FTZ-ALLOW-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_global_sys_param_1];
+; SM60-FTZ-ALLOW-NEXT: atom.sys.global.add.f32 %r2, [%rd1], %r1;
+; SM60-FTZ-ALLOW-NEXT: st.param.b32 [func_retval0], %r2;
+; SM60-FTZ-ALLOW-NEXT: ret;
%retval = atomicrmw fadd ptr addrspace(1) %addr, float %val syncscope("") acq_rel
ret float %retval
}
@@ -4084,40 +5450,64 @@ define i64 @nand_acq_rel_i64_global_cluster(ptr addrspace(1) %addr, i64 %val) {
}
define float @fadd_acq_rel_float_global_cluster(ptr addrspace(1) %addr, float %val) {
-; SM60-NOFTZ-LABEL: fadd_acq_rel_float_global_cluster(
-; SM60-NOFTZ: {
-; SM60-NOFTZ-NEXT: .reg .pred %p<2>;
-; SM60-NOFTZ-NEXT: .reg .b32 %r<5>;
-; SM60-NOFTZ-NEXT: .reg .b64 %rd<2>;
-; SM60-NOFTZ-EMPTY:
-; SM60-NOFTZ-NEXT: // %bb.0:
-; SM60-NOFTZ-NEXT: ld.param.b32 %r2, [fadd_acq_rel_float_global_cluster_param_1];
-; SM60-NOFTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_cluster_param_0];
-; SM60-NOFTZ-NEXT: membar.cta;
-; SM60-NOFTZ-NEXT: ld.volatile.global.b32 %r4, [%rd1];
-; SM60-NOFTZ-NEXT: $L__BB114_1: // %atomicrmw.start
-; SM60-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NOFTZ-NEXT: add.rn.f32 %r3, %r4, %r2;
-; SM60-NOFTZ-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
-; SM60-NOFTZ-NEXT: setp.ne.b32 %p1, %r1, %r4;
-; SM60-NOFTZ-NEXT: mov.b32 %r4, %r1;
-; SM60-NOFTZ-NEXT: @%p1 bra $L__BB114_1;
-; SM60-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end
-; SM60-NOFTZ-NEXT: membar.cta;
-; SM60-NOFTZ-NEXT: st.param.b32 [func_retval0], %r1;
-; SM60-NOFTZ-NEXT: ret;
+; SM60-NOFTZ-DISALLOW-LABEL: fadd_acq_rel_float_global_cluster(
+; SM60-NOFTZ-DISALLOW: {
+; SM60-NOFTZ-DISALLOW-NEXT: .reg .pred %p<2>;
+; SM60-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<5>;
+; SM60-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
+; SM60-NOFTZ-DISALLOW-EMPTY:
+; SM60-NOFTZ-DISALLOW-NEXT: // %bb.0:
+; SM60-NOFTZ-DISALLOW-NEXT: ld.param.b32 %r2, [fadd_acq_rel_float_global_cluster_param_1];
+; SM60-NOFTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_cluster_param_0];
+; SM60-NOFTZ-DISALLOW-NEXT: membar.cta;
+; SM60-NOFTZ-DISALLOW-NEXT: ld.volatile.global.b32 %r4, [%rd1];
+; SM60-NOFTZ-DISALLOW-NEXT: $L__BB114_1: // %atomicrmw.start
+; SM60-NOFTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NOFTZ-DISALLOW-NEXT: add.rn.f32 %r3, %r4, %r2;
+; SM60-NOFTZ-DISALLOW-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM60-NOFTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM60-NOFTZ-DISALLOW-NEXT: mov.b32 %r4, %r1;
+; SM60-NOFTZ-DISALLOW-NEXT: @%p1 bra $L__BB114_1;
+; SM60-NOFTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NOFTZ-DISALLOW-NEXT: membar.cta;
+; SM60-NOFTZ-DISALLOW-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NOFTZ-DISALLOW-NEXT: ret;
;
-; SM60-FTZ-LABEL: fadd_acq_rel_float_global_cluster(
-; SM60-FTZ: {
-; SM60-FTZ-NEXT: .reg .b32 %r<3>;
-; SM60-FTZ-NEXT: .reg .b64 %rd<2>;
-; SM60-FTZ-EMPTY:
-; SM60-FTZ-NEXT: // %bb.0:
-; SM60-FTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_cluster_param_0];
-; SM60-FTZ-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_global_cluster_param_1];
-; SM60-FTZ-NEXT: atom.cta.global.add.f32 %r2, [%rd1], %r1;
-; SM60-FTZ-NEXT: st.param.b32 [func_retval0], %r2;
-; SM60-FTZ-NEXT: ret;
+; SM60-NOFTZ-ALLOW-LABEL: fadd_acq_rel_float_global_cluster(
+; SM60-NOFTZ-ALLOW: {
+; SM60-NOFTZ-ALLOW-NEXT: .reg .b32 %r<3>;
+; SM60-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
+; SM60-NOFTZ-ALLOW-EMPTY:
+; SM60-NOFTZ-ALLOW-NEXT: // %bb.0:
+; SM60-NOFTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_cluster_param_0];
+; SM60-NOFTZ-ALLOW-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_global_cluster_param_1];
+; SM60-NOFTZ-ALLOW-NEXT: atom.cta.global.add.f32 %r2, [%rd1], %r1;
+; SM60-NOFTZ-ALLOW-NEXT: st.param.b32 [func_retval0], %r2;
+; SM60-NOFTZ-ALLOW-NEXT: ret;
+;
+; SM60-FTZ-DISALLOW-LABEL: fadd_acq_rel_float_global_cluster(
+; SM60-FTZ-DISALLOW: {
+; SM60-FTZ-DISALLOW-NEXT: .reg .b32 %r<3>;
+; SM60-FTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
+; SM60-FTZ-DISALLOW-EMPTY:
+; SM60-FTZ-DISALLOW-NEXT: // %bb.0:
+; SM60-FTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_cluster_param_0];
+; SM60-FTZ-DISALLOW-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_global_cluster_param_1];
+; SM60-FTZ-DISALLOW-NEXT: atom.cta.global.add.f32 %r2, [%rd1], %r1;
+; SM60-FTZ-DISALLOW-NEXT: st.param.b32 [func_retval0], %r2;
+; SM60-FTZ-DISALLOW-NEXT: ret;
+;
+; SM60-FTZ-ALLOW-LABEL: fadd_acq_rel_float_global_cluster(
+; SM60-FTZ-ALLOW: {
+; SM60-FTZ-ALLOW-NEXT: .reg .b32 %r<3>;
+; SM60-FTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
+; SM60-FTZ-ALLOW-EMPTY:
+; SM60-FTZ-ALLOW-NEXT: // %bb.0:
+; SM60-FTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_cluster_param_0];
+; SM60-FTZ-ALLOW-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_global_cluster_param_1];
+; SM60-FTZ-ALLOW-NEXT: atom.cta.global.add.f32 %r2, [%rd1], %r1;
+; SM60-FTZ-ALLOW-NEXT: st.param.b32 [func_retval0], %r2;
+; SM60-FTZ-ALLOW-NEXT: ret;
%retval = atomicrmw fadd ptr addrspace(1) %addr, float %val syncscope("cluster") acq_rel
ret float %retval
}
@@ -4256,40 +5646,64 @@ define i64 @nand_acq_rel_i64_global_gpu(ptr addrspace(1) %addr, i64 %val) {
}
define float @fadd_acq_rel_float_global_gpu(ptr addrspace(1) %addr, float %val) {
-; SM60-NOFTZ-LABEL: fadd_acq_rel_float_global_gpu(
-; SM60-NOFTZ: {
-; SM60-NOFTZ-NEXT: .reg .pred %p<2>;
-; SM60-NOFTZ-NEXT: .reg .b32 %r<5>;
-; SM60-NOFTZ-NEXT: .reg .b64 %rd<2>;
-; SM60-NOFTZ-EMPTY:
-; SM60-NOFTZ-NEXT: // %bb.0:
-; SM60-NOFTZ-NEXT: ld.param.b32 %r2, [fadd_acq_rel_float_global_gpu_param_1];
-; SM60-NOFTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_gpu_param_0];
-; SM60-NOFTZ-NEXT: membar.gl;
-; SM60-NOFTZ-NEXT: ld.volatile.global.b32 %r4, [%rd1];
-; SM60-NOFTZ-NEXT: $L__BB122_1: // %atomicrmw.start
-; SM60-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NOFTZ-NEXT: add.rn.f32 %r3, %r4, %r2;
-; SM60-NOFTZ-NEXT: atom.gpu.global.cas.b32 %r1, [%rd1], %r4, %r3;
-; SM60-NOFTZ-NEXT: setp.ne.b32 %p1, %r1, %r4;
-; SM60-NOFTZ-NEXT: mov.b32 %r4, %r1;
-; SM60-NOFTZ-NEXT: @%p1 bra $L__BB122_1;
-; SM60-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end
-; SM60-NOFTZ-NEXT: membar.gl;
-; SM60-NOFTZ-NEXT: st.param.b32 [func_retval0], %r1;
-; SM60-NOFTZ-NEXT: ret;
+; SM60-NOFTZ-DISALLOW-LABEL: fadd_acq_rel_float_global_gpu(
+; SM60-NOFTZ-DISALLOW: {
+; SM60-NOFTZ-DISALLOW-NEXT: .reg .pred %p<2>;
+; SM60-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<5>;
+; SM60-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
+; SM60-NOFTZ-DISALLOW-EMPTY:
+; SM60-NOFTZ-DISALLOW-NEXT: // %bb.0:
+; SM60-NOFTZ-DISALLOW-NEXT: ld.param.b32 %r2, [fadd_acq_rel_float_global_gpu_param_1];
+; SM60-NOFTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_gpu_param_0];
+; SM60-NOFTZ-DISALLOW-NEXT: membar.gl;
+; SM60-NOFTZ-DISALLOW-NEXT: ld.volatile.global.b32 %r4, [%rd1];
+; SM60-NOFTZ-DISALLOW-NEXT: $L__BB122_1: // %atomicrmw.start
+; SM60-NOFTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NOFTZ-DISALLOW-NEXT: add.rn.f32 %r3, %r4, %r2;
+; SM60-NOFTZ-DISALLOW-NEXT: atom.gpu.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM60-NOFTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM60-NOFTZ-DISALLOW-NEXT: mov.b32 %r4, %r1;
+; SM60-NOFTZ-DISALLOW-NEXT: @%p1 bra $L__BB122_1;
+; SM60-NOFTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NOFTZ-DISALLOW-NEXT: membar.gl;
+; SM60-NOFTZ-DISALLOW-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NOFTZ-DISALLOW-NEXT: ret;
+;
+; SM60-NOFTZ-ALLOW-LABEL: fadd_acq_rel_float_global_gpu(
+; SM60-NOFTZ-ALLOW: {
+; SM60-NOFTZ-ALLOW-NEXT: .reg .b32 %r<3>;
+; SM60-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
+; SM60-NOFTZ-ALLOW-EMPTY:
+; SM60-NOFTZ-ALLOW-NEXT: // %bb.0:
+; SM60-NOFTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_gpu_param_0];
+; SM60-NOFTZ-ALLOW-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_global_gpu_param_1];
+; SM60-NOFTZ-ALLOW-NEXT: atom.gpu.global.add.f32 %r2, [%rd1], %r1;
+; SM60-NOFTZ-ALLOW-NEXT: st.param.b32 [func_retval0], %r2;
+; SM60-NOFTZ-ALLOW-NEXT: ret;
;
-; SM60-FTZ-LABEL: fadd_acq_rel_float_global_gpu(
-; SM60-FTZ: {
-; SM60-FTZ-NEXT: .reg .b32 %r<3>;
-; SM60-FTZ-NEXT: .reg .b64 %rd<2>;
-; SM60-FTZ-EMPTY:
-; SM60-FTZ-NEXT: // %bb.0:
-; SM60-FTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_gpu_param_0];
-; SM60-FTZ-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_global_gpu_param_1];
-; SM60-FTZ-NEXT: atom.gpu.global.add.f32 %r2, [%rd1], %r1;
-; SM60-FTZ-NEXT: st.param.b32 [func_retval0], %r2;
-; SM60-FTZ-NEXT: ret;
+; SM60-FTZ-DISALLOW-LABEL: fadd_acq_rel_float_global_gpu(
+; SM60-FTZ-DISALLOW: {
+; SM60-FTZ-DISALLOW-NEXT: .reg .b32 %r<3>;
+; SM60-FTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
+; SM60-FTZ-DISALLOW-EMPTY:
+; SM60-FTZ-DISALLOW-NEXT: // %bb.0:
+; SM60-FTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_gpu_param_0];
+; SM60-FTZ-DISALLOW-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_global_gpu_param_1];
+; SM60-FTZ-DISALLOW-NEXT: atom.gpu.global.add.f32 %r2, [%rd1], %r1;
+; SM60-FTZ-DISALLOW-NEXT: st.param.b32 [func_retval0], %r2;
+; SM60-FTZ-DISALLOW-NEXT: ret;
+;
+; SM60-FTZ-ALLOW-LABEL: fadd_acq_rel_float_global_gpu(
+; SM60-FTZ-ALLOW: {
+; SM60-FTZ-ALLOW-NEXT: .reg .b32 %r<3>;
+; SM60-FTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
+; SM60-FTZ-ALLOW-EMPTY:
+; SM60-FTZ-ALLOW-NEXT: // %bb.0:
+; SM60-FTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_gpu_param_0];
+; SM60-FTZ-ALLOW-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_global_gpu_param_1];
+; SM60-FTZ-ALLOW-NEXT: atom.gpu.global.add.f32 %r2, [%rd1], %r1;
+; SM60-FTZ-ALLOW-NEXT: st.param.b32 [func_retval0], %r2;
+; SM60-FTZ-ALLOW-NEXT: ret;
%retval = atomicrmw fadd ptr addrspace(1) %addr, float %val syncscope("device") acq_rel
ret float %retval
}
@@ -4428,51 +5842,75 @@ define i64 @nand_acq_rel_i64_generic_cta(ptr %addr, i64 %val) {
}
define float @fadd_acq_rel_float_generic_cta(ptr %addr, float %val) {
-; SM60-NOFTZ-LABEL: fadd_acq_rel_float_generic_cta(
-; SM60-NOFTZ: {
-; SM60-NOFTZ-NEXT: .reg .pred %p<2>;
-; SM60-NOFTZ-NEXT: .reg .b32 %r<5>;
-; SM60-NOFTZ-NEXT: .reg .b64 %rd<2>;
-; SM60-NOFTZ-EMPTY:
-; SM60-NOFTZ-NEXT: // %bb.0:
-; SM60-NOFTZ-NEXT: ld.param.b32 %r2, [fadd_acq_rel_float_generic_cta_param_1];
-; SM60-NOFTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_generic_cta_param_0];
-; SM60-NOFTZ-NEXT: membar.cta;
-; SM60-NOFTZ-NEXT: ld.volatile.b32 %r4, [%rd1];
-; SM60-NOFTZ-NEXT: $L__BB130_1: // %atomicrmw.start
-; SM60-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-NOFTZ-NEXT: add.rn.f32 %r3, %r4, %r2;
-; SM60-NOFTZ-NEXT: atom.cta.cas.b32 %r1, [%rd1], %r4, %r3;
-; SM60-NOFTZ-NEXT: setp.ne.b32 %p1, %r1, %r4;
-; SM60-NOFTZ-NEXT: mov.b32 %r4, %r1;
-; SM60-NOFTZ-NEXT: @%p1 bra $L__BB130_1;
-; SM60-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end
-; SM60-NOFTZ-NEXT: membar.cta;
-; SM60-NOFTZ-NEXT: st.param.b32 [func_retval0], %r1;
-; SM60-NOFTZ-NEXT: ret;
+; SM60-NOFTZ-DISALLOW-LABEL: fadd_acq_rel_float_generic_cta(
+; SM60-NOFTZ-DISALLOW: {
+; SM60-NOFTZ-DISALLOW-NEXT: .reg .pred %p<2>;
+; SM60-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<5>;
+; SM60-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
+; SM60-NOFTZ-DISALLOW-EMPTY:
+; SM60-NOFTZ-DISALLOW-NEXT: // %bb.0:
+; SM60-NOFTZ-DISALLOW-NEXT: ld.param.b32 %r2, [fadd_acq_rel_float_generic_cta_param_1];
+; SM60-NOFTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_generic_cta_param_0];
+; SM60-NOFTZ-DISALLOW-NEXT: membar.cta;
+; SM60-NOFTZ-DISALLOW-NEXT: ld.volatile.b32 %r4, [%rd1];
+; SM60-NOFTZ-DISALLOW-NEXT: $L__BB130_1: // %atomicrmw.start
+; SM60-NOFTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-NOFTZ-DISALLOW-NEXT: add.rn.f32 %r3, %r4, %r2;
+; SM60-NOFTZ-DISALLOW-NEXT: atom.cta.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM60-NOFTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM60-NOFTZ-DISALLOW-NEXT: mov.b32 %r4, %r1;
+; SM60-NOFTZ-DISALLOW-NEXT: @%p1 bra $L__BB130_1;
+; SM60-NOFTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-NOFTZ-DISALLOW-NEXT: membar.cta;
+; SM60-NOFTZ-DISALLOW-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-NOFTZ-DISALLOW-NEXT: ret;
+;
+; SM60-NOFTZ-ALLOW-LABEL: fadd_acq_rel_float_generic_cta(
+; SM60-NOFTZ-ALLOW: {
+; SM60-NOFTZ-ALLOW-NEXT: .reg .b32 %r<3>;
+; SM60-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
+; SM60-NOFTZ-ALLOW-EMPTY:
+; SM60-NOFTZ-ALLOW-NEXT: // %bb.0:
+; SM60-NOFTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_generic_cta_param_0];
+; SM60-NOFTZ-ALLOW-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_generic_cta_param_1];
+; SM60-NOFTZ-ALLOW-NEXT: atom.cta.add.f32 %r2, [%rd1], %r1;
+; SM60-NOFTZ-ALLOW-NEXT: st.param.b32 [func_retval0], %r2;
+; SM60-NOFTZ-ALLOW-NEXT: ret;
+;
+; SM60-FTZ-DISALLOW-LABEL: fadd_acq_rel_float_generic_cta(
+; SM60-FTZ-DISALLOW: {
+; SM60-FTZ-DISALLOW-NEXT: .reg .pred %p<2>;
+; SM60-FTZ-DISALLOW-NEXT: .reg .b32 %r<5>;
+; SM60-FTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
+; SM60-FTZ-DISALLOW-EMPTY:
+; SM60-FTZ-DISALLOW-NEXT: // %bb.0:
+; SM60-FTZ-DISALLOW-NEXT: ld.param.b32 %r2, [fadd_acq_rel_float_generic_cta_param_1];
+; SM60-FTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_generic_cta_param_0];
+; SM60-FTZ-DISALLOW-NEXT: membar.cta;
+; SM60-FTZ-DISALLOW-NEXT: ld.volatile.b32 %r4, [%rd1];
+; SM60-FTZ-DISALLOW-NEXT: $L__BB130_1: // %atomicrmw.start
+; SM60-FTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-FTZ-DISALLOW-NEXT: add.rn.ftz.f32 %r3, %r4, %r2;
+; SM60-FTZ-DISALLOW-NEXT: atom.cta.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM60-FTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM60-FTZ-DISALLOW-NEXT: mov.b32 %r4, %r1;
+; SM60-FTZ-DISALLOW-NEXT: @%p1 bra $L__BB130_1;
+; SM60-FTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-FTZ-DISALLOW-NEXT: membar.cta;
+; SM60-FTZ-DISALLOW-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-FTZ-DISALLOW-NEXT: ret;
;
-; SM60-FTZ-LABEL: fadd_acq_rel_float_generic_cta(
-; SM60-FTZ: {
-; SM60-FTZ-NEXT: .reg .pred %p<2>;
-; SM60-FTZ-NEXT: .reg .b32 %r<5>;
-; SM60-FTZ-NEXT: .reg .b64 %rd<2>;
-; SM60-FTZ-EMPTY:
-; SM60-FTZ-NEXT: // %bb.0:
-; SM60-FTZ-NEXT: ld.param.b32 %r2, [fadd_acq_rel_float_generic_cta_param_1];
-; SM60-FTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_generic_cta_param_0];
-; SM60-FTZ-NEXT: membar.cta;
-; SM60-FTZ-NEXT: ld.volatile.b32 %r4, [%rd1];
-; SM60-FTZ-NEXT: $L__BB130_1: // %atomicrmw.start
-; SM60-FTZ-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-FTZ-NEXT: add.rn.ftz.f32 %r3, %r4, %r2;
-; SM60-FTZ-NEXT: atom.cta.cas.b32 %r1, [%rd1], %r4, %r3;
-; SM60-FTZ-NEXT: setp.ne.b32 %p1, %r1, %r4;
-; SM60-FTZ-NEXT: mov.b32 %r4, %r1;
-; SM60-FTZ-NEXT: @%p1 bra $L__BB130_1;
-; SM60-FTZ-NEXT: // %bb.2: // %atomicrmw.end
-; SM60-FTZ-NEXT: membar.cta;
-; SM60-FTZ-NEXT: st.param.b32 [func_retval0], %r1;
-; SM60-FTZ-NEXT: ret;
+; SM60-FTZ-ALLOW-LABEL: fadd_acq_rel_float_generic_cta(
+; SM60-FTZ-ALLOW: {
+; SM60-FTZ-ALLOW-NEXT: .reg .b32 %r<3>;
+; SM60-FTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
+; SM60-FTZ-ALLOW-EMPTY:
+; SM60-FTZ-ALLOW-NEXT: // %bb.0:
+; SM60-FTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_generic_cta_param_0];
+; SM60-FTZ-ALLOW-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_generic_cta_param_1];
+; SM60-FTZ-ALLOW-NEXT: atom.cta.add.f32 %r2, [%rd1], %r1;
+; SM60-FTZ-ALLOW-NEXT: st.param.b32 [func_retval0], %r2;
+; SM60-FTZ-ALLOW-NEXT: ret;
%retval = atomicrmw fadd ptr %addr, float %val syncscope("block") acq_rel
ret float %retval
}
@@ -4611,40 +6049,64 @@ define i64 @nand_acq_rel_i64_shared_cta(ptr addrspace(3) %addr, i64 %val) {
}
define float @fadd_acq_rel_float_shared_cta(ptr addrspace(3) %addr, float %val) {
-; SM60-NOFTZ-LABEL: fadd_acq_rel_float_shared_cta(
-; SM60-NOFTZ: {
-; SM60-NOFTZ-NEXT: .reg .b32 %r<3>;
-; SM60-NOFTZ-NEXT: .reg .b64 %rd<2>;
-; SM60-NOFTZ-EMPTY:
-; SM60-NOFTZ-NEXT: // %bb.0:
-; SM60-NOFTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_shared_cta_param_0];
-; SM60-NOFTZ-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_shared_cta_param_1];
-; SM60-NOFTZ-NEXT: atom.cta.shared.add.f32 %r2, [%rd1], %r1;
-; SM60-NOFTZ-NEXT: st.param.b32 [func_retval0], %r2;
-; SM60-NOFTZ-NEXT: ret;
+; SM60-NOFTZ-DISALLOW-LABEL: fadd_acq_rel_float_shared_cta(
+; SM60-NOFTZ-DISALLOW: {
+; SM60-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<3>;
+; SM60-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
+; SM60-NOFTZ-DISALLOW-EMPTY:
+; SM60-NOFTZ-DISALLOW-NEXT: // %bb.0:
+; SM60-NOFTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_shared_cta_param_0];
+; SM60-NOFTZ-DISALLOW-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_shared_cta_param_1];
+; SM60-NOFTZ-DISALLOW-NEXT: atom.cta.shared.add.f32 %r2, [%rd1], %r1;
+; SM60-NOFTZ-DISALLOW-NEXT: st.param.b32 [func_retval0], %r2;
+; SM60-NOFTZ-DISALLOW-NEXT: ret;
+;
+; SM60-NOFTZ-ALLOW-LABEL: fadd_acq_rel_float_shared_cta(
+; SM60-NOFTZ-ALLOW: {
+; SM60-NOFTZ-ALLOW-NEXT: .reg .b32 %r<3>;
+; SM60-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
+; SM60-NOFTZ-ALLOW-EMPTY:
+; SM60-NOFTZ-ALLOW-NEXT: // %bb.0:
+; SM60-NOFTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_shared_cta_param_0];
+; SM60-NOFTZ-ALLOW-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_shared_cta_param_1];
+; SM60-NOFTZ-ALLOW-NEXT: atom.cta.shared.add.f32 %r2, [%rd1], %r1;
+; SM60-NOFTZ-ALLOW-NEXT: st.param.b32 [func_retval0], %r2;
+; SM60-NOFTZ-ALLOW-NEXT: ret;
+;
+; SM60-FTZ-DISALLOW-LABEL: fadd_acq_rel_float_shared_cta(
+; SM60-FTZ-DISALLOW: {
+; SM60-FTZ-DISALLOW-NEXT: .reg .pred %p<2>;
+; SM60-FTZ-DISALLOW-NEXT: .reg .b32 %r<5>;
+; SM60-FTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
+; SM60-FTZ-DISALLOW-EMPTY:
+; SM60-FTZ-DISALLOW-NEXT: // %bb.0:
+; SM60-FTZ-DISALLOW-NEXT: ld.param.b32 %r2, [fadd_acq_rel_float_shared_cta_param_1];
+; SM60-FTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_shared_cta_param_0];
+; SM60-FTZ-DISALLOW-NEXT: membar.cta;
+; SM60-FTZ-DISALLOW-NEXT: ld.volatile.shared.b32 %r4, [%rd1];
+; SM60-FTZ-DISALLOW-NEXT: $L__BB138_1: // %atomicrmw.start
+; SM60-FTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM60-FTZ-DISALLOW-NEXT: add.rn.ftz.f32 %r3, %r4, %r2;
+; SM60-FTZ-DISALLOW-NEXT: atom.cta.shared.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM60-FTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM60-FTZ-DISALLOW-NEXT: mov.b32 %r4, %r1;
+; SM60-FTZ-DISALLOW-NEXT: @%p1 bra $L__BB138_1;
+; SM60-FTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM60-FTZ-DISALLOW-NEXT: membar.cta;
+; SM60-FTZ-DISALLOW-NEXT: st.param.b32 [func_retval0], %r1;
+; SM60-FTZ-DISALLOW-NEXT: ret;
;
-; SM60-FTZ-LABEL: fadd_acq_rel_float_shared_cta(
-; SM60-FTZ: {
-; SM60-FTZ-NEXT: .reg .pred %p<2>;
-; SM60-FTZ-NEXT: .reg .b32 %r<5>;
-; SM60-FTZ-NEXT: .reg .b64 %rd<2>;
-; SM60-FTZ-EMPTY:
-; SM60-FTZ-NEXT: // %bb.0:
-; SM60-FTZ-NEXT: ld.param.b32 %r2, [fadd_acq_rel_float_shared_cta_param_1];
-; SM60-FTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_shared_cta_param_0];
-; SM60-FTZ-NEXT: membar.cta;
-; SM60-FTZ-NEXT: ld.volatile.shared.b32 %r4, [%rd1];
-; SM60-FTZ-NEXT: $L__BB138_1: // %atomicrmw.start
-; SM60-FTZ-NEXT: // =>This Inner Loop Header: Depth=1
-; SM60-FTZ-NEXT: add.rn.ftz.f32 %r3, %r4, %r2;
-; SM60-FTZ-NEXT: atom.cta.shared.cas.b32 %r1, [%rd1], %r4, %r3;
-; SM60-FTZ-NEXT: setp.ne.b32 %p1, %r1, %r4;
-; SM60-FTZ-NEXT: mov.b32 %r4, %r1;
-; SM60-FTZ-NEXT: @%p1 bra $L__BB138_1;
-; SM60-FTZ-NEXT: // %bb.2: // %atomicrmw.end
-; SM60-FTZ-NEXT: membar.cta;
-; SM60-FTZ-NEXT: st.param.b32 [func_retval0], %r1;
-; SM60-FTZ-NEXT: ret;
+; SM60-FTZ-ALLOW-LABEL: fadd_acq_rel_float_shared_cta(
+; SM60-FTZ-ALLOW: {
+; SM60-FTZ-ALLOW-NEXT: .reg .b32 %r<3>;
+; SM60-FTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
+; SM60-FTZ-ALLOW-EMPTY:
+; SM60-FTZ-ALLOW-NEXT: // %bb.0:
+; SM60-FTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_shared_cta_param_0];
+; SM60-FTZ-ALLOW-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_shared_cta_param_1];
+; SM60-FTZ-ALLOW-NEXT: atom.cta.shared.add.f32 %r2, [%rd1], %r1;
+; SM60-FTZ-ALLOW-NEXT: st.param.b32 [func_retval0], %r2;
+; SM60-FTZ-ALLOW-NEXT: ret;
%retval = atomicrmw fadd ptr addrspace(3) %addr, float %val syncscope("block") acq_rel
ret float %retval
}
diff --git a/llvm/test/CodeGen/NVPTX/atomicrmw-sm70.ll b/llvm/test/CodeGen/NVPTX/atomicrmw-sm70.ll
index b0342d66c0afd..19291c6d0c055 100644
--- a/llvm/test/CodeGen/NVPTX/atomicrmw-sm70.ll
+++ b/llvm/test/CodeGen/NVPTX/atomicrmw-sm70.ll
@@ -1,7 +1,15 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
-; RUN: llc < %s -march=nvptx64 -mcpu=sm_70 -mattr=+ptx63 | FileCheck %s --check-prefixes=SM70,SM70-NOFTZ
-; RUN: llc < %s -march=nvptx64 -mcpu=sm_70 -mattr=+ptx63 -denormal-fp-math-f32=preserve-sign | FileCheck %s --check-prefixes=SM70,SM70-FTZ
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_70 -mattr=+ptx63 -nvptx-allow-ftz-atomics=false | FileCheck %s --check-prefixes=SM70,SM70-NOFTZ-DISALLOW
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_70 -mattr=+ptx63 -nvptx-allow-ftz-atomics=true | FileCheck %s --check-prefixes=SM70,SM70-NOFTZ-ALLOW
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_70 -mattr=+ptx63 | FileCheck %s --check-prefixes=SM70,SM70-NOFTZ-ALLOW
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_70 -mattr=+ptx63 -denormal-fp-math-f32=preserve-sign -nvptx-allow-ftz-atomics=false | FileCheck %s --check-prefixes=SM70,SM70-FTZ-DISALLOW
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_70 -mattr=+ptx63 -denormal-fp-math-f32=preserve-sign -nvptx-allow-ftz-atomics=true | FileCheck %s --check-prefixes=SM70,SM70-FTZ-ALLOW
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_70 -mattr=+ptx63 -denormal-fp-math-f32=preserve-sign | FileCheck %s --check-prefixes=SM70,SM70-FTZ-ALLOW
+; RUN: %if ptxas-sm_70 && ptxas-isa-6.3 %{ llc < %s -march=nvptx64 -mcpu=sm_70 -mattr=+ptx63 -nvptx-allow-ftz-atomics=false | %ptxas-verify -arch=sm_70 %}
+; RUN: %if ptxas-sm_70 && ptxas-isa-6.3 %{ llc < %s -march=nvptx64 -mcpu=sm_70 -mattr=+ptx63 -nvptx-allow-ftz-atomics=true | %ptxas-verify -arch=sm_70 %}
; RUN: %if ptxas-sm_70 && ptxas-isa-6.3 %{ llc < %s -march=nvptx64 -mcpu=sm_70 -mattr=+ptx63 | %ptxas-verify -arch=sm_70 %}
+; RUN: %if ptxas-sm_70 && ptxas-isa-6.3 %{ llc < %s -march=nvptx64 -mcpu=sm_70 -mattr=+ptx63 -denormal-fp-math-f32=preserve-sign -nvptx-allow-ftz-atomics=false | %ptxas-verify -arch=sm_70 %}
+; RUN: %if ptxas-sm_70 && ptxas-isa-6.3 %{ llc < %s -march=nvptx64 -mcpu=sm_70 -mattr=+ptx63 -denormal-fp-math-f32=preserve-sign -nvptx-allow-ftz-atomics=true | %ptxas-verify -arch=sm_70 %}
; RUN: %if ptxas-sm_70 && ptxas-isa-6.3 %{ llc < %s -march=nvptx64 -mcpu=sm_70 -mattr=+ptx63 -denormal-fp-math-f32=preserve-sign | %ptxas-verify -arch=sm_70 %}
define i8 @xchg_acq_rel_i8_global_cta(ptr addrspace(1) %addr, i8 %val) {
@@ -1739,314 +1747,592 @@ define i64 @usub_sat_acq_rel_i64_global_cta(ptr addrspace(1) %addr, i64 %val) {
}
define float @fadd_acq_rel_float_global_cta(ptr addrspace(1) %addr, float %val) {
-; SM70-NOFTZ-LABEL: fadd_acq_rel_float_global_cta(
-; SM70-NOFTZ: {
-; SM70-NOFTZ-NEXT: .reg .pred %p<2>;
-; SM70-NOFTZ-NEXT: .reg .b32 %r<5>;
-; SM70-NOFTZ-NEXT: .reg .b64 %rd<2>;
-; SM70-NOFTZ-EMPTY:
-; SM70-NOFTZ-NEXT: // %bb.0:
-; SM70-NOFTZ-NEXT: ld.param.b32 %r2, [fadd_acq_rel_float_global_cta_param_1];
-; SM70-NOFTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_cta_param_0];
-; SM70-NOFTZ-NEXT: fence.acq_rel.cta;
-; SM70-NOFTZ-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
-; SM70-NOFTZ-NEXT: $L__BB60_1: // %atomicrmw.start
-; SM70-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NOFTZ-NEXT: add.rn.f32 %r3, %r4, %r2;
-; SM70-NOFTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
-; SM70-NOFTZ-NEXT: setp.ne.b32 %p1, %r1, %r4;
-; SM70-NOFTZ-NEXT: mov.b32 %r4, %r1;
-; SM70-NOFTZ-NEXT: @%p1 bra $L__BB60_1;
-; SM70-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end
-; SM70-NOFTZ-NEXT: fence.acq_rel.cta;
-; SM70-NOFTZ-NEXT: st.param.b32 [func_retval0], %r1;
-; SM70-NOFTZ-NEXT: ret;
+; SM70-NOFTZ-DISALLOW-LABEL: fadd_acq_rel_float_global_cta(
+; SM70-NOFTZ-DISALLOW: {
+; SM70-NOFTZ-DISALLOW-NEXT: .reg .pred %p<2>;
+; SM70-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<5>;
+; SM70-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
+; SM70-NOFTZ-DISALLOW-EMPTY:
+; SM70-NOFTZ-DISALLOW-NEXT: // %bb.0:
+; SM70-NOFTZ-DISALLOW-NEXT: ld.param.b32 %r2, [fadd_acq_rel_float_global_cta_param_1];
+; SM70-NOFTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_cta_param_0];
+; SM70-NOFTZ-DISALLOW-NEXT: fence.acq_rel.cta;
+; SM70-NOFTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM70-NOFTZ-DISALLOW-NEXT: $L__BB60_1: // %atomicrmw.start
+; SM70-NOFTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NOFTZ-DISALLOW-NEXT: add.rn.f32 %r3, %r4, %r2;
+; SM70-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM70-NOFTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM70-NOFTZ-DISALLOW-NEXT: mov.b32 %r4, %r1;
+; SM70-NOFTZ-DISALLOW-NEXT: @%p1 bra $L__BB60_1;
+; SM70-NOFTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NOFTZ-DISALLOW-NEXT: fence.acq_rel.cta;
+; SM70-NOFTZ-DISALLOW-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NOFTZ-DISALLOW-NEXT: ret;
;
-; SM70-FTZ-LABEL: fadd_acq_rel_float_global_cta(
-; SM70-FTZ: {
-; SM70-FTZ-NEXT: .reg .b32 %r<3>;
-; SM70-FTZ-NEXT: .reg .b64 %rd<2>;
-; SM70-FTZ-EMPTY:
-; SM70-FTZ-NEXT: // %bb.0:
-; SM70-FTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_cta_param_0];
-; SM70-FTZ-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_global_cta_param_1];
-; SM70-FTZ-NEXT: atom.acq_rel.cta.global.add.f32 %r2, [%rd1], %r1;
-; SM70-FTZ-NEXT: st.param.b32 [func_retval0], %r2;
-; SM70-FTZ-NEXT: ret;
+; SM70-NOFTZ-ALLOW-LABEL: fadd_acq_rel_float_global_cta(
+; SM70-NOFTZ-ALLOW: {
+; SM70-NOFTZ-ALLOW-NEXT: .reg .b32 %r<3>;
+; SM70-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
+; SM70-NOFTZ-ALLOW-EMPTY:
+; SM70-NOFTZ-ALLOW-NEXT: // %bb.0:
+; SM70-NOFTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_cta_param_0];
+; SM70-NOFTZ-ALLOW-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_global_cta_param_1];
+; SM70-NOFTZ-ALLOW-NEXT: atom.acq_rel.cta.global.add.f32 %r2, [%rd1], %r1;
+; SM70-NOFTZ-ALLOW-NEXT: st.param.b32 [func_retval0], %r2;
+; SM70-NOFTZ-ALLOW-NEXT: ret;
+;
+; SM70-FTZ-DISALLOW-LABEL: fadd_acq_rel_float_global_cta(
+; SM70-FTZ-DISALLOW: {
+; SM70-FTZ-DISALLOW-NEXT: .reg .b32 %r<3>;
+; SM70-FTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
+; SM70-FTZ-DISALLOW-EMPTY:
+; SM70-FTZ-DISALLOW-NEXT: // %bb.0:
+; SM70-FTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_cta_param_0];
+; SM70-FTZ-DISALLOW-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_global_cta_param_1];
+; SM70-FTZ-DISALLOW-NEXT: atom.acq_rel.cta.global.add.f32 %r2, [%rd1], %r1;
+; SM70-FTZ-DISALLOW-NEXT: st.param.b32 [func_retval0], %r2;
+; SM70-FTZ-DISALLOW-NEXT: ret;
+;
+; SM70-FTZ-ALLOW-LABEL: fadd_acq_rel_float_global_cta(
+; SM70-FTZ-ALLOW: {
+; SM70-FTZ-ALLOW-NEXT: .reg .b32 %r<3>;
+; SM70-FTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
+; SM70-FTZ-ALLOW-EMPTY:
+; SM70-FTZ-ALLOW-NEXT: // %bb.0:
+; SM70-FTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_cta_param_0];
+; SM70-FTZ-ALLOW-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_global_cta_param_1];
+; SM70-FTZ-ALLOW-NEXT: atom.acq_rel.cta.global.add.f32 %r2, [%rd1], %r1;
+; SM70-FTZ-ALLOW-NEXT: st.param.b32 [func_retval0], %r2;
+; SM70-FTZ-ALLOW-NEXT: ret;
%retval = atomicrmw fadd ptr addrspace(1) %addr, float %val syncscope("block") acq_rel
ret float %retval
}
define float @fsub_acq_rel_float_global_cta(ptr addrspace(1) %addr, float %val) {
-; SM70-NOFTZ-LABEL: fsub_acq_rel_float_global_cta(
-; SM70-NOFTZ: {
-; SM70-NOFTZ-NEXT: .reg .pred %p<2>;
-; SM70-NOFTZ-NEXT: .reg .b32 %r<5>;
-; SM70-NOFTZ-NEXT: .reg .b64 %rd<2>;
-; SM70-NOFTZ-EMPTY:
-; SM70-NOFTZ-NEXT: // %bb.0:
-; SM70-NOFTZ-NEXT: ld.param.b32 %r2, [fsub_acq_rel_float_global_cta_param_1];
-; SM70-NOFTZ-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_float_global_cta_param_0];
-; SM70-NOFTZ-NEXT: fence.acq_rel.cta;
-; SM70-NOFTZ-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
-; SM70-NOFTZ-NEXT: $L__BB61_1: // %atomicrmw.start
-; SM70-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NOFTZ-NEXT: sub.rn.f32 %r3, %r4, %r2;
-; SM70-NOFTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
-; SM70-NOFTZ-NEXT: setp.ne.b32 %p1, %r1, %r4;
-; SM70-NOFTZ-NEXT: mov.b32 %r4, %r1;
-; SM70-NOFTZ-NEXT: @%p1 bra $L__BB61_1;
-; SM70-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end
-; SM70-NOFTZ-NEXT: fence.acq_rel.cta;
-; SM70-NOFTZ-NEXT: st.param.b32 [func_retval0], %r1;
-; SM70-NOFTZ-NEXT: ret;
+; SM70-NOFTZ-DISALLOW-LABEL: fsub_acq_rel_float_global_cta(
+; SM70-NOFTZ-DISALLOW: {
+; SM70-NOFTZ-DISALLOW-NEXT: .reg .pred %p<2>;
+; SM70-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<5>;
+; SM70-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
+; SM70-NOFTZ-DISALLOW-EMPTY:
+; SM70-NOFTZ-DISALLOW-NEXT: // %bb.0:
+; SM70-NOFTZ-DISALLOW-NEXT: ld.param.b32 %r2, [fsub_acq_rel_float_global_cta_param_1];
+; SM70-NOFTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_float_global_cta_param_0];
+; SM70-NOFTZ-DISALLOW-NEXT: fence.acq_rel.cta;
+; SM70-NOFTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM70-NOFTZ-DISALLOW-NEXT: $L__BB61_1: // %atomicrmw.start
+; SM70-NOFTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NOFTZ-DISALLOW-NEXT: sub.rn.f32 %r3, %r4, %r2;
+; SM70-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM70-NOFTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM70-NOFTZ-DISALLOW-NEXT: mov.b32 %r4, %r1;
+; SM70-NOFTZ-DISALLOW-NEXT: @%p1 bra $L__BB61_1;
+; SM70-NOFTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NOFTZ-DISALLOW-NEXT: fence.acq_rel.cta;
+; SM70-NOFTZ-DISALLOW-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NOFTZ-DISALLOW-NEXT: ret;
+;
+; SM70-NOFTZ-ALLOW-LABEL: fsub_acq_rel_float_global_cta(
+; SM70-NOFTZ-ALLOW: {
+; SM70-NOFTZ-ALLOW-NEXT: .reg .pred %p<2>;
+; SM70-NOFTZ-ALLOW-NEXT: .reg .b32 %r<5>;
+; SM70-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
+; SM70-NOFTZ-ALLOW-EMPTY:
+; SM70-NOFTZ-ALLOW-NEXT: // %bb.0:
+; SM70-NOFTZ-ALLOW-NEXT: ld.param.b32 %r2, [fsub_acq_rel_float_global_cta_param_1];
+; SM70-NOFTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_float_global_cta_param_0];
+; SM70-NOFTZ-ALLOW-NEXT: fence.acq_rel.cta;
+; SM70-NOFTZ-ALLOW-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM70-NOFTZ-ALLOW-NEXT: $L__BB61_1: // %atomicrmw.start
+; SM70-NOFTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NOFTZ-ALLOW-NEXT: sub.rn.f32 %r3, %r4, %r2;
+; SM70-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM70-NOFTZ-ALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM70-NOFTZ-ALLOW-NEXT: mov.b32 %r4, %r1;
+; SM70-NOFTZ-ALLOW-NEXT: @%p1 bra $L__BB61_1;
+; SM70-NOFTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NOFTZ-ALLOW-NEXT: fence.acq_rel.cta;
+; SM70-NOFTZ-ALLOW-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NOFTZ-ALLOW-NEXT: ret;
;
-; SM70-FTZ-LABEL: fsub_acq_rel_float_global_cta(
-; SM70-FTZ: {
-; SM70-FTZ-NEXT: .reg .pred %p<2>;
-; SM70-FTZ-NEXT: .reg .b32 %r<5>;
-; SM70-FTZ-NEXT: .reg .b64 %rd<2>;
-; SM70-FTZ-EMPTY:
-; SM70-FTZ-NEXT: // %bb.0:
-; SM70-FTZ-NEXT: ld.param.b32 %r2, [fsub_acq_rel_float_global_cta_param_1];
-; SM70-FTZ-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_float_global_cta_param_0];
-; SM70-FTZ-NEXT: fence.acq_rel.cta;
-; SM70-FTZ-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
-; SM70-FTZ-NEXT: $L__BB61_1: // %atomicrmw.start
-; SM70-FTZ-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-FTZ-NEXT: sub.rn.ftz.f32 %r3, %r4, %r2;
-; SM70-FTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
-; SM70-FTZ-NEXT: setp.ne.b32 %p1, %r1, %r4;
-; SM70-FTZ-NEXT: mov.b32 %r4, %r1;
-; SM70-FTZ-NEXT: @%p1 bra $L__BB61_1;
-; SM70-FTZ-NEXT: // %bb.2: // %atomicrmw.end
-; SM70-FTZ-NEXT: fence.acq_rel.cta;
-; SM70-FTZ-NEXT: st.param.b32 [func_retval0], %r1;
-; SM70-FTZ-NEXT: ret;
+; SM70-FTZ-DISALLOW-LABEL: fsub_acq_rel_float_global_cta(
+; SM70-FTZ-DISALLOW: {
+; SM70-FTZ-DISALLOW-NEXT: .reg .pred %p<2>;
+; SM70-FTZ-DISALLOW-NEXT: .reg .b32 %r<5>;
+; SM70-FTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
+; SM70-FTZ-DISALLOW-EMPTY:
+; SM70-FTZ-DISALLOW-NEXT: // %bb.0:
+; SM70-FTZ-DISALLOW-NEXT: ld.param.b32 %r2, [fsub_acq_rel_float_global_cta_param_1];
+; SM70-FTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_float_global_cta_param_0];
+; SM70-FTZ-DISALLOW-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM70-FTZ-DISALLOW-NEXT: $L__BB61_1: // %atomicrmw.start
+; SM70-FTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-FTZ-DISALLOW-NEXT: sub.rn.ftz.f32 %r3, %r4, %r2;
+; SM70-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM70-FTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM70-FTZ-DISALLOW-NEXT: mov.b32 %r4, %r1;
+; SM70-FTZ-DISALLOW-NEXT: @%p1 bra $L__BB61_1;
+; SM70-FTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-FTZ-DISALLOW-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-DISALLOW-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-FTZ-DISALLOW-NEXT: ret;
+;
+; SM70-FTZ-ALLOW-LABEL: fsub_acq_rel_float_global_cta(
+; SM70-FTZ-ALLOW: {
+; SM70-FTZ-ALLOW-NEXT: .reg .pred %p<2>;
+; SM70-FTZ-ALLOW-NEXT: .reg .b32 %r<5>;
+; SM70-FTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
+; SM70-FTZ-ALLOW-EMPTY:
+; SM70-FTZ-ALLOW-NEXT: // %bb.0:
+; SM70-FTZ-ALLOW-NEXT: ld.param.b32 %r2, [fsub_acq_rel_float_global_cta_param_1];
+; SM70-FTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_float_global_cta_param_0];
+; SM70-FTZ-ALLOW-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-ALLOW-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM70-FTZ-ALLOW-NEXT: $L__BB61_1: // %atomicrmw.start
+; SM70-FTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-FTZ-ALLOW-NEXT: sub.rn.ftz.f32 %r3, %r4, %r2;
+; SM70-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM70-FTZ-ALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM70-FTZ-ALLOW-NEXT: mov.b32 %r4, %r1;
+; SM70-FTZ-ALLOW-NEXT: @%p1 bra $L__BB61_1;
+; SM70-FTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-FTZ-ALLOW-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-ALLOW-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-FTZ-ALLOW-NEXT: ret;
%retval = atomicrmw fsub ptr addrspace(1) %addr, float %val syncscope("block") acq_rel
ret float %retval
}
define float @fmin_acq_rel_float_global_cta(ptr addrspace(1) %addr, float %val) {
-; SM70-NOFTZ-LABEL: fmin_acq_rel_float_global_cta(
-; SM70-NOFTZ: {
-; SM70-NOFTZ-NEXT: .reg .pred %p<2>;
-; SM70-NOFTZ-NEXT: .reg .b32 %r<5>;
-; SM70-NOFTZ-NEXT: .reg .b64 %rd<2>;
-; SM70-NOFTZ-EMPTY:
-; SM70-NOFTZ-NEXT: // %bb.0:
-; SM70-NOFTZ-NEXT: ld.param.b32 %r2, [fmin_acq_rel_float_global_cta_param_1];
-; SM70-NOFTZ-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_float_global_cta_param_0];
-; SM70-NOFTZ-NEXT: fence.acq_rel.cta;
-; SM70-NOFTZ-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
-; SM70-NOFTZ-NEXT: $L__BB62_1: // %atomicrmw.start
-; SM70-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NOFTZ-NEXT: min.f32 %r3, %r4, %r2;
-; SM70-NOFTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
-; SM70-NOFTZ-NEXT: setp.ne.b32 %p1, %r1, %r4;
-; SM70-NOFTZ-NEXT: mov.b32 %r4, %r1;
-; SM70-NOFTZ-NEXT: @%p1 bra $L__BB62_1;
-; SM70-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end
-; SM70-NOFTZ-NEXT: fence.acq_rel.cta;
-; SM70-NOFTZ-NEXT: st.param.b32 [func_retval0], %r1;
-; SM70-NOFTZ-NEXT: ret;
+; SM70-NOFTZ-DISALLOW-LABEL: fmin_acq_rel_float_global_cta(
+; SM70-NOFTZ-DISALLOW: {
+; SM70-NOFTZ-DISALLOW-NEXT: .reg .pred %p<2>;
+; SM70-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<5>;
+; SM70-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
+; SM70-NOFTZ-DISALLOW-EMPTY:
+; SM70-NOFTZ-DISALLOW-NEXT: // %bb.0:
+; SM70-NOFTZ-DISALLOW-NEXT: ld.param.b32 %r2, [fmin_acq_rel_float_global_cta_param_1];
+; SM70-NOFTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_float_global_cta_param_0];
+; SM70-NOFTZ-DISALLOW-NEXT: fence.acq_rel.cta;
+; SM70-NOFTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM70-NOFTZ-DISALLOW-NEXT: $L__BB62_1: // %atomicrmw.start
+; SM70-NOFTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NOFTZ-DISALLOW-NEXT: min.f32 %r3, %r4, %r2;
+; SM70-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM70-NOFTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM70-NOFTZ-DISALLOW-NEXT: mov.b32 %r4, %r1;
+; SM70-NOFTZ-DISALLOW-NEXT: @%p1 bra $L__BB62_1;
+; SM70-NOFTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NOFTZ-DISALLOW-NEXT: fence.acq_rel.cta;
+; SM70-NOFTZ-DISALLOW-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NOFTZ-DISALLOW-NEXT: ret;
+;
+; SM70-NOFTZ-ALLOW-LABEL: fmin_acq_rel_float_global_cta(
+; SM70-NOFTZ-ALLOW: {
+; SM70-NOFTZ-ALLOW-NEXT: .reg .pred %p<2>;
+; SM70-NOFTZ-ALLOW-NEXT: .reg .b32 %r<5>;
+; SM70-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
+; SM70-NOFTZ-ALLOW-EMPTY:
+; SM70-NOFTZ-ALLOW-NEXT: // %bb.0:
+; SM70-NOFTZ-ALLOW-NEXT: ld.param.b32 %r2, [fmin_acq_rel_float_global_cta_param_1];
+; SM70-NOFTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_float_global_cta_param_0];
+; SM70-NOFTZ-ALLOW-NEXT: fence.acq_rel.cta;
+; SM70-NOFTZ-ALLOW-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM70-NOFTZ-ALLOW-NEXT: $L__BB62_1: // %atomicrmw.start
+; SM70-NOFTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NOFTZ-ALLOW-NEXT: min.f32 %r3, %r4, %r2;
+; SM70-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM70-NOFTZ-ALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM70-NOFTZ-ALLOW-NEXT: mov.b32 %r4, %r1;
+; SM70-NOFTZ-ALLOW-NEXT: @%p1 bra $L__BB62_1;
+; SM70-NOFTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NOFTZ-ALLOW-NEXT: fence.acq_rel.cta;
+; SM70-NOFTZ-ALLOW-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NOFTZ-ALLOW-NEXT: ret;
+;
+; SM70-FTZ-DISALLOW-LABEL: fmin_acq_rel_float_global_cta(
+; SM70-FTZ-DISALLOW: {
+; SM70-FTZ-DISALLOW-NEXT: .reg .pred %p<2>;
+; SM70-FTZ-DISALLOW-NEXT: .reg .b32 %r<5>;
+; SM70-FTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
+; SM70-FTZ-DISALLOW-EMPTY:
+; SM70-FTZ-DISALLOW-NEXT: // %bb.0:
+; SM70-FTZ-DISALLOW-NEXT: ld.param.b32 %r2, [fmin_acq_rel_float_global_cta_param_1];
+; SM70-FTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_float_global_cta_param_0];
+; SM70-FTZ-DISALLOW-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM70-FTZ-DISALLOW-NEXT: $L__BB62_1: // %atomicrmw.start
+; SM70-FTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-FTZ-DISALLOW-NEXT: min.ftz.f32 %r3, %r4, %r2;
+; SM70-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM70-FTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM70-FTZ-DISALLOW-NEXT: mov.b32 %r4, %r1;
+; SM70-FTZ-DISALLOW-NEXT: @%p1 bra $L__BB62_1;
+; SM70-FTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-FTZ-DISALLOW-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-DISALLOW-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-FTZ-DISALLOW-NEXT: ret;
;
-; SM70-FTZ-LABEL: fmin_acq_rel_float_global_cta(
-; SM70-FTZ: {
-; SM70-FTZ-NEXT: .reg .pred %p<2>;
-; SM70-FTZ-NEXT: .reg .b32 %r<5>;
-; SM70-FTZ-NEXT: .reg .b64 %rd<2>;
-; SM70-FTZ-EMPTY:
-; SM70-FTZ-NEXT: // %bb.0:
-; SM70-FTZ-NEXT: ld.param.b32 %r2, [fmin_acq_rel_float_global_cta_param_1];
-; SM70-FTZ-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_float_global_cta_param_0];
-; SM70-FTZ-NEXT: fence.acq_rel.cta;
-; SM70-FTZ-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
-; SM70-FTZ-NEXT: $L__BB62_1: // %atomicrmw.start
-; SM70-FTZ-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-FTZ-NEXT: min.ftz.f32 %r3, %r4, %r2;
-; SM70-FTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
-; SM70-FTZ-NEXT: setp.ne.b32 %p1, %r1, %r4;
-; SM70-FTZ-NEXT: mov.b32 %r4, %r1;
-; SM70-FTZ-NEXT: @%p1 bra $L__BB62_1;
-; SM70-FTZ-NEXT: // %bb.2: // %atomicrmw.end
-; SM70-FTZ-NEXT: fence.acq_rel.cta;
-; SM70-FTZ-NEXT: st.param.b32 [func_retval0], %r1;
-; SM70-FTZ-NEXT: ret;
+; SM70-FTZ-ALLOW-LABEL: fmin_acq_rel_float_global_cta(
+; SM70-FTZ-ALLOW: {
+; SM70-FTZ-ALLOW-NEXT: .reg .pred %p<2>;
+; SM70-FTZ-ALLOW-NEXT: .reg .b32 %r<5>;
+; SM70-FTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
+; SM70-FTZ-ALLOW-EMPTY:
+; SM70-FTZ-ALLOW-NEXT: // %bb.0:
+; SM70-FTZ-ALLOW-NEXT: ld.param.b32 %r2, [fmin_acq_rel_float_global_cta_param_1];
+; SM70-FTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_float_global_cta_param_0];
+; SM70-FTZ-ALLOW-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-ALLOW-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM70-FTZ-ALLOW-NEXT: $L__BB62_1: // %atomicrmw.start
+; SM70-FTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-FTZ-ALLOW-NEXT: min.ftz.f32 %r3, %r4, %r2;
+; SM70-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM70-FTZ-ALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM70-FTZ-ALLOW-NEXT: mov.b32 %r4, %r1;
+; SM70-FTZ-ALLOW-NEXT: @%p1 bra $L__BB62_1;
+; SM70-FTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-FTZ-ALLOW-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-ALLOW-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-FTZ-ALLOW-NEXT: ret;
%retval = atomicrmw fmin ptr addrspace(1) %addr, float %val syncscope("block") acq_rel
ret float %retval
}
define float @fmax_acq_rel_float_global_cta(ptr addrspace(1) %addr, float %val) {
-; SM70-NOFTZ-LABEL: fmax_acq_rel_float_global_cta(
-; SM70-NOFTZ: {
-; SM70-NOFTZ-NEXT: .reg .pred %p<2>;
-; SM70-NOFTZ-NEXT: .reg .b32 %r<5>;
-; SM70-NOFTZ-NEXT: .reg .b64 %rd<2>;
-; SM70-NOFTZ-EMPTY:
-; SM70-NOFTZ-NEXT: // %bb.0:
-; SM70-NOFTZ-NEXT: ld.param.b32 %r2, [fmax_acq_rel_float_global_cta_param_1];
-; SM70-NOFTZ-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_float_global_cta_param_0];
-; SM70-NOFTZ-NEXT: fence.acq_rel.cta;
-; SM70-NOFTZ-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
-; SM70-NOFTZ-NEXT: $L__BB63_1: // %atomicrmw.start
-; SM70-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NOFTZ-NEXT: max.f32 %r3, %r4, %r2;
-; SM70-NOFTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
-; SM70-NOFTZ-NEXT: setp.ne.b32 %p1, %r1, %r4;
-; SM70-NOFTZ-NEXT: mov.b32 %r4, %r1;
-; SM70-NOFTZ-NEXT: @%p1 bra $L__BB63_1;
-; SM70-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end
-; SM70-NOFTZ-NEXT: fence.acq_rel.cta;
-; SM70-NOFTZ-NEXT: st.param.b32 [func_retval0], %r1;
-; SM70-NOFTZ-NEXT: ret;
+; SM70-NOFTZ-DISALLOW-LABEL: fmax_acq_rel_float_global_cta(
+; SM70-NOFTZ-DISALLOW: {
+; SM70-NOFTZ-DISALLOW-NEXT: .reg .pred %p<2>;
+; SM70-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<5>;
+; SM70-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
+; SM70-NOFTZ-DISALLOW-EMPTY:
+; SM70-NOFTZ-DISALLOW-NEXT: // %bb.0:
+; SM70-NOFTZ-DISALLOW-NEXT: ld.param.b32 %r2, [fmax_acq_rel_float_global_cta_param_1];
+; SM70-NOFTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_float_global_cta_param_0];
+; SM70-NOFTZ-DISALLOW-NEXT: fence.acq_rel.cta;
+; SM70-NOFTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM70-NOFTZ-DISALLOW-NEXT: $L__BB63_1: // %atomicrmw.start
+; SM70-NOFTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NOFTZ-DISALLOW-NEXT: max.f32 %r3, %r4, %r2;
+; SM70-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM70-NOFTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM70-NOFTZ-DISALLOW-NEXT: mov.b32 %r4, %r1;
+; SM70-NOFTZ-DISALLOW-NEXT: @%p1 bra $L__BB63_1;
+; SM70-NOFTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NOFTZ-DISALLOW-NEXT: fence.acq_rel.cta;
+; SM70-NOFTZ-DISALLOW-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NOFTZ-DISALLOW-NEXT: ret;
;
-; SM70-FTZ-LABEL: fmax_acq_rel_float_global_cta(
-; SM70-FTZ: {
-; SM70-FTZ-NEXT: .reg .pred %p<2>;
-; SM70-FTZ-NEXT: .reg .b32 %r<5>;
-; SM70-FTZ-NEXT: .reg .b64 %rd<2>;
-; SM70-FTZ-EMPTY:
-; SM70-FTZ-NEXT: // %bb.0:
-; SM70-FTZ-NEXT: ld.param.b32 %r2, [fmax_acq_rel_float_global_cta_param_1];
-; SM70-FTZ-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_float_global_cta_param_0];
-; SM70-FTZ-NEXT: fence.acq_rel.cta;
-; SM70-FTZ-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
-; SM70-FTZ-NEXT: $L__BB63_1: // %atomicrmw.start
-; SM70-FTZ-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-FTZ-NEXT: max.ftz.f32 %r3, %r4, %r2;
-; SM70-FTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
-; SM70-FTZ-NEXT: setp.ne.b32 %p1, %r1, %r4;
-; SM70-FTZ-NEXT: mov.b32 %r4, %r1;
-; SM70-FTZ-NEXT: @%p1 bra $L__BB63_1;
-; SM70-FTZ-NEXT: // %bb.2: // %atomicrmw.end
-; SM70-FTZ-NEXT: fence.acq_rel.cta;
-; SM70-FTZ-NEXT: st.param.b32 [func_retval0], %r1;
-; SM70-FTZ-NEXT: ret;
+; SM70-NOFTZ-ALLOW-LABEL: fmax_acq_rel_float_global_cta(
+; SM70-NOFTZ-ALLOW: {
+; SM70-NOFTZ-ALLOW-NEXT: .reg .pred %p<2>;
+; SM70-NOFTZ-ALLOW-NEXT: .reg .b32 %r<5>;
+; SM70-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
+; SM70-NOFTZ-ALLOW-EMPTY:
+; SM70-NOFTZ-ALLOW-NEXT: // %bb.0:
+; SM70-NOFTZ-ALLOW-NEXT: ld.param.b32 %r2, [fmax_acq_rel_float_global_cta_param_1];
+; SM70-NOFTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_float_global_cta_param_0];
+; SM70-NOFTZ-ALLOW-NEXT: fence.acq_rel.cta;
+; SM70-NOFTZ-ALLOW-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM70-NOFTZ-ALLOW-NEXT: $L__BB63_1: // %atomicrmw.start
+; SM70-NOFTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NOFTZ-ALLOW-NEXT: max.f32 %r3, %r4, %r2;
+; SM70-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM70-NOFTZ-ALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM70-NOFTZ-ALLOW-NEXT: mov.b32 %r4, %r1;
+; SM70-NOFTZ-ALLOW-NEXT: @%p1 bra $L__BB63_1;
+; SM70-NOFTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NOFTZ-ALLOW-NEXT: fence.acq_rel.cta;
+; SM70-NOFTZ-ALLOW-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NOFTZ-ALLOW-NEXT: ret;
+;
+; SM70-FTZ-DISALLOW-LABEL: fmax_acq_rel_float_global_cta(
+; SM70-FTZ-DISALLOW: {
+; SM70-FTZ-DISALLOW-NEXT: .reg .pred %p<2>;
+; SM70-FTZ-DISALLOW-NEXT: .reg .b32 %r<5>;
+; SM70-FTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
+; SM70-FTZ-DISALLOW-EMPTY:
+; SM70-FTZ-DISALLOW-NEXT: // %bb.0:
+; SM70-FTZ-DISALLOW-NEXT: ld.param.b32 %r2, [fmax_acq_rel_float_global_cta_param_1];
+; SM70-FTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_float_global_cta_param_0];
+; SM70-FTZ-DISALLOW-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM70-FTZ-DISALLOW-NEXT: $L__BB63_1: // %atomicrmw.start
+; SM70-FTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-FTZ-DISALLOW-NEXT: max.ftz.f32 %r3, %r4, %r2;
+; SM70-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM70-FTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM70-FTZ-DISALLOW-NEXT: mov.b32 %r4, %r1;
+; SM70-FTZ-DISALLOW-NEXT: @%p1 bra $L__BB63_1;
+; SM70-FTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-FTZ-DISALLOW-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-DISALLOW-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-FTZ-DISALLOW-NEXT: ret;
+;
+; SM70-FTZ-ALLOW-LABEL: fmax_acq_rel_float_global_cta(
+; SM70-FTZ-ALLOW: {
+; SM70-FTZ-ALLOW-NEXT: .reg .pred %p<2>;
+; SM70-FTZ-ALLOW-NEXT: .reg .b32 %r<5>;
+; SM70-FTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
+; SM70-FTZ-ALLOW-EMPTY:
+; SM70-FTZ-ALLOW-NEXT: // %bb.0:
+; SM70-FTZ-ALLOW-NEXT: ld.param.b32 %r2, [fmax_acq_rel_float_global_cta_param_1];
+; SM70-FTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_float_global_cta_param_0];
+; SM70-FTZ-ALLOW-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-ALLOW-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM70-FTZ-ALLOW-NEXT: $L__BB63_1: // %atomicrmw.start
+; SM70-FTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-FTZ-ALLOW-NEXT: max.ftz.f32 %r3, %r4, %r2;
+; SM70-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM70-FTZ-ALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM70-FTZ-ALLOW-NEXT: mov.b32 %r4, %r1;
+; SM70-FTZ-ALLOW-NEXT: @%p1 bra $L__BB63_1;
+; SM70-FTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-FTZ-ALLOW-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-ALLOW-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-FTZ-ALLOW-NEXT: ret;
%retval = atomicrmw fmax ptr addrspace(1) %addr, float %val syncscope("block") acq_rel
ret float %retval
}
define float @fminimum_acq_rel_float_global_cta(ptr addrspace(1) %addr, float %val) {
-; SM70-NOFTZ-LABEL: fminimum_acq_rel_float_global_cta(
-; SM70-NOFTZ: {
-; SM70-NOFTZ-NEXT: .reg .pred %p<5>;
-; SM70-NOFTZ-NEXT: .reg .b32 %r<8>;
-; SM70-NOFTZ-NEXT: .reg .b64 %rd<2>;
-; SM70-NOFTZ-EMPTY:
-; SM70-NOFTZ-NEXT: // %bb.0:
-; SM70-NOFTZ-NEXT: ld.param.b32 %r2, [fminimum_acq_rel_float_global_cta_param_1];
-; SM70-NOFTZ-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_float_global_cta_param_0];
-; SM70-NOFTZ-NEXT: fence.acq_rel.cta;
-; SM70-NOFTZ-NEXT: ld.relaxed.cta.global.b32 %r7, [%rd1];
-; SM70-NOFTZ-NEXT: $L__BB64_1: // %atomicrmw.start
-; SM70-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NOFTZ-NEXT: setp.eq.b32 %p1, %r7, -2147483648;
-; SM70-NOFTZ-NEXT: selp.f32 %r3, %r7, %r2, %p1;
-; SM70-NOFTZ-NEXT: setp.nan.f32 %p2, %r7, %r2;
-; SM70-NOFTZ-NEXT: min.f32 %r4, %r7, %r2;
-; SM70-NOFTZ-NEXT: selp.f32 %r5, 0f7FC00000, %r4, %p2;
-; SM70-NOFTZ-NEXT: setp.eq.f32 %p3, %r7, %r2;
-; SM70-NOFTZ-NEXT: selp.f32 %r6, %r3, %r5, %p3;
-; SM70-NOFTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r7, %r6;
-; SM70-NOFTZ-NEXT: setp.ne.b32 %p4, %r1, %r7;
-; SM70-NOFTZ-NEXT: mov.b32 %r7, %r1;
-; SM70-NOFTZ-NEXT: @%p4 bra $L__BB64_1;
-; SM70-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end
-; SM70-NOFTZ-NEXT: fence.acq_rel.cta;
-; SM70-NOFTZ-NEXT: st.param.b32 [func_retval0], %r1;
-; SM70-NOFTZ-NEXT: ret;
+; SM70-NOFTZ-DISALLOW-LABEL: fminimum_acq_rel_float_global_cta(
+; SM70-NOFTZ-DISALLOW: {
+; SM70-NOFTZ-DISALLOW-NEXT: .reg .pred %p<5>;
+; SM70-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<8>;
+; SM70-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
+; SM70-NOFTZ-DISALLOW-EMPTY:
+; SM70-NOFTZ-DISALLOW-NEXT: // %bb.0:
+; SM70-NOFTZ-DISALLOW-NEXT: ld.param.b32 %r2, [fminimum_acq_rel_float_global_cta_param_1];
+; SM70-NOFTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_float_global_cta_param_0];
+; SM70-NOFTZ-DISALLOW-NEXT: fence.acq_rel.cta;
+; SM70-NOFTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b32 %r7, [%rd1];
+; SM70-NOFTZ-DISALLOW-NEXT: $L__BB64_1: // %atomicrmw.start
+; SM70-NOFTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NOFTZ-DISALLOW-NEXT: setp.eq.b32 %p1, %r7, -2147483648;
+; SM70-NOFTZ-DISALLOW-NEXT: selp.f32 %r3, %r7, %r2, %p1;
+; SM70-NOFTZ-DISALLOW-NEXT: setp.nan.f32 %p2, %r7, %r2;
+; SM70-NOFTZ-DISALLOW-NEXT: min.f32 %r4, %r7, %r2;
+; SM70-NOFTZ-DISALLOW-NEXT: selp.f32 %r5, 0f7FC00000, %r4, %p2;
+; SM70-NOFTZ-DISALLOW-NEXT: setp.eq.f32 %p3, %r7, %r2;
+; SM70-NOFTZ-DISALLOW-NEXT: selp.f32 %r6, %r3, %r5, %p3;
+; SM70-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r7, %r6;
+; SM70-NOFTZ-DISALLOW-NEXT: setp.ne.b32 %p4, %r1, %r7;
+; SM70-NOFTZ-DISALLOW-NEXT: mov.b32 %r7, %r1;
+; SM70-NOFTZ-DISALLOW-NEXT: @%p4 bra $L__BB64_1;
+; SM70-NOFTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NOFTZ-DISALLOW-NEXT: fence.acq_rel.cta;
+; SM70-NOFTZ-DISALLOW-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NOFTZ-DISALLOW-NEXT: ret;
+;
+; SM70-NOFTZ-ALLOW-LABEL: fminimum_acq_rel_float_global_cta(
+; SM70-NOFTZ-ALLOW: {
+; SM70-NOFTZ-ALLOW-NEXT: .reg .pred %p<5>;
+; SM70-NOFTZ-ALLOW-NEXT: .reg .b32 %r<8>;
+; SM70-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
+; SM70-NOFTZ-ALLOW-EMPTY:
+; SM70-NOFTZ-ALLOW-NEXT: // %bb.0:
+; SM70-NOFTZ-ALLOW-NEXT: ld.param.b32 %r2, [fminimum_acq_rel_float_global_cta_param_1];
+; SM70-NOFTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_float_global_cta_param_0];
+; SM70-NOFTZ-ALLOW-NEXT: fence.acq_rel.cta;
+; SM70-NOFTZ-ALLOW-NEXT: ld.relaxed.cta.global.b32 %r7, [%rd1];
+; SM70-NOFTZ-ALLOW-NEXT: $L__BB64_1: // %atomicrmw.start
+; SM70-NOFTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NOFTZ-ALLOW-NEXT: setp.eq.b32 %p1, %r7, -2147483648;
+; SM70-NOFTZ-ALLOW-NEXT: selp.f32 %r3, %r7, %r2, %p1;
+; SM70-NOFTZ-ALLOW-NEXT: setp.nan.f32 %p2, %r7, %r2;
+; SM70-NOFTZ-ALLOW-NEXT: min.f32 %r4, %r7, %r2;
+; SM70-NOFTZ-ALLOW-NEXT: selp.f32 %r5, 0f7FC00000, %r4, %p2;
+; SM70-NOFTZ-ALLOW-NEXT: setp.eq.f32 %p3, %r7, %r2;
+; SM70-NOFTZ-ALLOW-NEXT: selp.f32 %r6, %r3, %r5, %p3;
+; SM70-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r7, %r6;
+; SM70-NOFTZ-ALLOW-NEXT: setp.ne.b32 %p4, %r1, %r7;
+; SM70-NOFTZ-ALLOW-NEXT: mov.b32 %r7, %r1;
+; SM70-NOFTZ-ALLOW-NEXT: @%p4 bra $L__BB64_1;
+; SM70-NOFTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NOFTZ-ALLOW-NEXT: fence.acq_rel.cta;
+; SM70-NOFTZ-ALLOW-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NOFTZ-ALLOW-NEXT: ret;
;
-; SM70-FTZ-LABEL: fminimum_acq_rel_float_global_cta(
-; SM70-FTZ: {
-; SM70-FTZ-NEXT: .reg .pred %p<5>;
-; SM70-FTZ-NEXT: .reg .b32 %r<8>;
-; SM70-FTZ-NEXT: .reg .b64 %rd<2>;
-; SM70-FTZ-EMPTY:
-; SM70-FTZ-NEXT: // %bb.0:
-; SM70-FTZ-NEXT: ld.param.b32 %r2, [fminimum_acq_rel_float_global_cta_param_1];
-; SM70-FTZ-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_float_global_cta_param_0];
-; SM70-FTZ-NEXT: fence.acq_rel.cta;
-; SM70-FTZ-NEXT: ld.relaxed.cta.global.b32 %r7, [%rd1];
-; SM70-FTZ-NEXT: $L__BB64_1: // %atomicrmw.start
-; SM70-FTZ-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-FTZ-NEXT: setp.eq.b32 %p1, %r7, -2147483648;
-; SM70-FTZ-NEXT: selp.f32 %r3, %r7, %r2, %p1;
-; SM70-FTZ-NEXT: setp.nan.ftz.f32 %p2, %r7, %r2;
-; SM70-FTZ-NEXT: min.ftz.f32 %r4, %r7, %r2;
-; SM70-FTZ-NEXT: selp.f32 %r5, 0f7FC00000, %r4, %p2;
-; SM70-FTZ-NEXT: setp.eq.ftz.f32 %p3, %r7, %r2;
-; SM70-FTZ-NEXT: selp.f32 %r6, %r3, %r5, %p3;
-; SM70-FTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r7, %r6;
-; SM70-FTZ-NEXT: setp.ne.b32 %p4, %r1, %r7;
-; SM70-FTZ-NEXT: mov.b32 %r7, %r1;
-; SM70-FTZ-NEXT: @%p4 bra $L__BB64_1;
-; SM70-FTZ-NEXT: // %bb.2: // %atomicrmw.end
-; SM70-FTZ-NEXT: fence.acq_rel.cta;
-; SM70-FTZ-NEXT: st.param.b32 [func_retval0], %r1;
-; SM70-FTZ-NEXT: ret;
+; SM70-FTZ-DISALLOW-LABEL: fminimum_acq_rel_float_global_cta(
+; SM70-FTZ-DISALLOW: {
+; SM70-FTZ-DISALLOW-NEXT: .reg .pred %p<5>;
+; SM70-FTZ-DISALLOW-NEXT: .reg .b32 %r<8>;
+; SM70-FTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
+; SM70-FTZ-DISALLOW-EMPTY:
+; SM70-FTZ-DISALLOW-NEXT: // %bb.0:
+; SM70-FTZ-DISALLOW-NEXT: ld.param.b32 %r2, [fminimum_acq_rel_float_global_cta_param_1];
+; SM70-FTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_float_global_cta_param_0];
+; SM70-FTZ-DISALLOW-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b32 %r7, [%rd1];
+; SM70-FTZ-DISALLOW-NEXT: $L__BB64_1: // %atomicrmw.start
+; SM70-FTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-FTZ-DISALLOW-NEXT: setp.eq.b32 %p1, %r7, -2147483648;
+; SM70-FTZ-DISALLOW-NEXT: selp.f32 %r3, %r7, %r2, %p1;
+; SM70-FTZ-DISALLOW-NEXT: setp.nan.ftz.f32 %p2, %r7, %r2;
+; SM70-FTZ-DISALLOW-NEXT: min.ftz.f32 %r4, %r7, %r2;
+; SM70-FTZ-DISALLOW-NEXT: selp.f32 %r5, 0f7FC00000, %r4, %p2;
+; SM70-FTZ-DISALLOW-NEXT: setp.eq.ftz.f32 %p3, %r7, %r2;
+; SM70-FTZ-DISALLOW-NEXT: selp.f32 %r6, %r3, %r5, %p3;
+; SM70-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r7, %r6;
+; SM70-FTZ-DISALLOW-NEXT: setp.ne.b32 %p4, %r1, %r7;
+; SM70-FTZ-DISALLOW-NEXT: mov.b32 %r7, %r1;
+; SM70-FTZ-DISALLOW-NEXT: @%p4 bra $L__BB64_1;
+; SM70-FTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-FTZ-DISALLOW-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-DISALLOW-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-FTZ-DISALLOW-NEXT: ret;
+;
+; SM70-FTZ-ALLOW-LABEL: fminimum_acq_rel_float_global_cta(
+; SM70-FTZ-ALLOW: {
+; SM70-FTZ-ALLOW-NEXT: .reg .pred %p<5>;
+; SM70-FTZ-ALLOW-NEXT: .reg .b32 %r<8>;
+; SM70-FTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
+; SM70-FTZ-ALLOW-EMPTY:
+; SM70-FTZ-ALLOW-NEXT: // %bb.0:
+; SM70-FTZ-ALLOW-NEXT: ld.param.b32 %r2, [fminimum_acq_rel_float_global_cta_param_1];
+; SM70-FTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_float_global_cta_param_0];
+; SM70-FTZ-ALLOW-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-ALLOW-NEXT: ld.relaxed.cta.global.b32 %r7, [%rd1];
+; SM70-FTZ-ALLOW-NEXT: $L__BB64_1: // %atomicrmw.start
+; SM70-FTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-FTZ-ALLOW-NEXT: setp.eq.b32 %p1, %r7, -2147483648;
+; SM70-FTZ-ALLOW-NEXT: selp.f32 %r3, %r7, %r2, %p1;
+; SM70-FTZ-ALLOW-NEXT: setp.nan.ftz.f32 %p2, %r7, %r2;
+; SM70-FTZ-ALLOW-NEXT: min.ftz.f32 %r4, %r7, %r2;
+; SM70-FTZ-ALLOW-NEXT: selp.f32 %r5, 0f7FC00000, %r4, %p2;
+; SM70-FTZ-ALLOW-NEXT: setp.eq.ftz.f32 %p3, %r7, %r2;
+; SM70-FTZ-ALLOW-NEXT: selp.f32 %r6, %r3, %r5, %p3;
+; SM70-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r7, %r6;
+; SM70-FTZ-ALLOW-NEXT: setp.ne.b32 %p4, %r1, %r7;
+; SM70-FTZ-ALLOW-NEXT: mov.b32 %r7, %r1;
+; SM70-FTZ-ALLOW-NEXT: @%p4 bra $L__BB64_1;
+; SM70-FTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-FTZ-ALLOW-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-ALLOW-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-FTZ-ALLOW-NEXT: ret;
%retval = atomicrmw fminimum ptr addrspace(1) %addr, float %val syncscope("block") acq_rel
ret float %retval
}
define float @fmaximum_acq_rel_float_global_cta(ptr addrspace(1) %addr, float %val) {
-; SM70-NOFTZ-LABEL: fmaximum_acq_rel_float_global_cta(
-; SM70-NOFTZ: {
-; SM70-NOFTZ-NEXT: .reg .pred %p<5>;
-; SM70-NOFTZ-NEXT: .reg .b32 %r<8>;
-; SM70-NOFTZ-NEXT: .reg .b64 %rd<2>;
-; SM70-NOFTZ-EMPTY:
-; SM70-NOFTZ-NEXT: // %bb.0:
-; SM70-NOFTZ-NEXT: ld.param.b32 %r2, [fmaximum_acq_rel_float_global_cta_param_1];
-; SM70-NOFTZ-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_float_global_cta_param_0];
-; SM70-NOFTZ-NEXT: fence.acq_rel.cta;
-; SM70-NOFTZ-NEXT: ld.relaxed.cta.global.b32 %r7, [%rd1];
-; SM70-NOFTZ-NEXT: $L__BB65_1: // %atomicrmw.start
-; SM70-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NOFTZ-NEXT: setp.eq.b32 %p1, %r7, 0;
-; SM70-NOFTZ-NEXT: selp.f32 %r3, %r7, %r2, %p1;
-; SM70-NOFTZ-NEXT: setp.nan.f32 %p2, %r7, %r2;
-; SM70-NOFTZ-NEXT: max.f32 %r4, %r7, %r2;
-; SM70-NOFTZ-NEXT: selp.f32 %r5, 0f7FC00000, %r4, %p2;
-; SM70-NOFTZ-NEXT: setp.eq.f32 %p3, %r7, %r2;
-; SM70-NOFTZ-NEXT: selp.f32 %r6, %r3, %r5, %p3;
-; SM70-NOFTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r7, %r6;
-; SM70-NOFTZ-NEXT: setp.ne.b32 %p4, %r1, %r7;
-; SM70-NOFTZ-NEXT: mov.b32 %r7, %r1;
-; SM70-NOFTZ-NEXT: @%p4 bra $L__BB65_1;
-; SM70-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end
-; SM70-NOFTZ-NEXT: fence.acq_rel.cta;
-; SM70-NOFTZ-NEXT: st.param.b32 [func_retval0], %r1;
-; SM70-NOFTZ-NEXT: ret;
+; SM70-NOFTZ-DISALLOW-LABEL: fmaximum_acq_rel_float_global_cta(
+; SM70-NOFTZ-DISALLOW: {
+; SM70-NOFTZ-DISALLOW-NEXT: .reg .pred %p<5>;
+; SM70-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<8>;
+; SM70-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
+; SM70-NOFTZ-DISALLOW-EMPTY:
+; SM70-NOFTZ-DISALLOW-NEXT: // %bb.0:
+; SM70-NOFTZ-DISALLOW-NEXT: ld.param.b32 %r2, [fmaximum_acq_rel_float_global_cta_param_1];
+; SM70-NOFTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_float_global_cta_param_0];
+; SM70-NOFTZ-DISALLOW-NEXT: fence.acq_rel.cta;
+; SM70-NOFTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b32 %r7, [%rd1];
+; SM70-NOFTZ-DISALLOW-NEXT: $L__BB65_1: // %atomicrmw.start
+; SM70-NOFTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NOFTZ-DISALLOW-NEXT: setp.eq.b32 %p1, %r7, 0;
+; SM70-NOFTZ-DISALLOW-NEXT: selp.f32 %r3, %r7, %r2, %p1;
+; SM70-NOFTZ-DISALLOW-NEXT: setp.nan.f32 %p2, %r7, %r2;
+; SM70-NOFTZ-DISALLOW-NEXT: max.f32 %r4, %r7, %r2;
+; SM70-NOFTZ-DISALLOW-NEXT: selp.f32 %r5, 0f7FC00000, %r4, %p2;
+; SM70-NOFTZ-DISALLOW-NEXT: setp.eq.f32 %p3, %r7, %r2;
+; SM70-NOFTZ-DISALLOW-NEXT: selp.f32 %r6, %r3, %r5, %p3;
+; SM70-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r7, %r6;
+; SM70-NOFTZ-DISALLOW-NEXT: setp.ne.b32 %p4, %r1, %r7;
+; SM70-NOFTZ-DISALLOW-NEXT: mov.b32 %r7, %r1;
+; SM70-NOFTZ-DISALLOW-NEXT: @%p4 bra $L__BB65_1;
+; SM70-NOFTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NOFTZ-DISALLOW-NEXT: fence.acq_rel.cta;
+; SM70-NOFTZ-DISALLOW-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NOFTZ-DISALLOW-NEXT: ret;
+;
+; SM70-NOFTZ-ALLOW-LABEL: fmaximum_acq_rel_float_global_cta(
+; SM70-NOFTZ-ALLOW: {
+; SM70-NOFTZ-ALLOW-NEXT: .reg .pred %p<5>;
+; SM70-NOFTZ-ALLOW-NEXT: .reg .b32 %r<8>;
+; SM70-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
+; SM70-NOFTZ-ALLOW-EMPTY:
+; SM70-NOFTZ-ALLOW-NEXT: // %bb.0:
+; SM70-NOFTZ-ALLOW-NEXT: ld.param.b32 %r2, [fmaximum_acq_rel_float_global_cta_param_1];
+; SM70-NOFTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_float_global_cta_param_0];
+; SM70-NOFTZ-ALLOW-NEXT: fence.acq_rel.cta;
+; SM70-NOFTZ-ALLOW-NEXT: ld.relaxed.cta.global.b32 %r7, [%rd1];
+; SM70-NOFTZ-ALLOW-NEXT: $L__BB65_1: // %atomicrmw.start
+; SM70-NOFTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NOFTZ-ALLOW-NEXT: setp.eq.b32 %p1, %r7, 0;
+; SM70-NOFTZ-ALLOW-NEXT: selp.f32 %r3, %r7, %r2, %p1;
+; SM70-NOFTZ-ALLOW-NEXT: setp.nan.f32 %p2, %r7, %r2;
+; SM70-NOFTZ-ALLOW-NEXT: max.f32 %r4, %r7, %r2;
+; SM70-NOFTZ-ALLOW-NEXT: selp.f32 %r5, 0f7FC00000, %r4, %p2;
+; SM70-NOFTZ-ALLOW-NEXT: setp.eq.f32 %p3, %r7, %r2;
+; SM70-NOFTZ-ALLOW-NEXT: selp.f32 %r6, %r3, %r5, %p3;
+; SM70-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r7, %r6;
+; SM70-NOFTZ-ALLOW-NEXT: setp.ne.b32 %p4, %r1, %r7;
+; SM70-NOFTZ-ALLOW-NEXT: mov.b32 %r7, %r1;
+; SM70-NOFTZ-ALLOW-NEXT: @%p4 bra $L__BB65_1;
+; SM70-NOFTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NOFTZ-ALLOW-NEXT: fence.acq_rel.cta;
+; SM70-NOFTZ-ALLOW-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NOFTZ-ALLOW-NEXT: ret;
+;
+; SM70-FTZ-DISALLOW-LABEL: fmaximum_acq_rel_float_global_cta(
+; SM70-FTZ-DISALLOW: {
+; SM70-FTZ-DISALLOW-NEXT: .reg .pred %p<5>;
+; SM70-FTZ-DISALLOW-NEXT: .reg .b32 %r<8>;
+; SM70-FTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
+; SM70-FTZ-DISALLOW-EMPTY:
+; SM70-FTZ-DISALLOW-NEXT: // %bb.0:
+; SM70-FTZ-DISALLOW-NEXT: ld.param.b32 %r2, [fmaximum_acq_rel_float_global_cta_param_1];
+; SM70-FTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_float_global_cta_param_0];
+; SM70-FTZ-DISALLOW-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b32 %r7, [%rd1];
+; SM70-FTZ-DISALLOW-NEXT: $L__BB65_1: // %atomicrmw.start
+; SM70-FTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-FTZ-DISALLOW-NEXT: setp.eq.b32 %p1, %r7, 0;
+; SM70-FTZ-DISALLOW-NEXT: selp.f32 %r3, %r7, %r2, %p1;
+; SM70-FTZ-DISALLOW-NEXT: setp.nan.ftz.f32 %p2, %r7, %r2;
+; SM70-FTZ-DISALLOW-NEXT: max.ftz.f32 %r4, %r7, %r2;
+; SM70-FTZ-DISALLOW-NEXT: selp.f32 %r5, 0f7FC00000, %r4, %p2;
+; SM70-FTZ-DISALLOW-NEXT: setp.eq.ftz.f32 %p3, %r7, %r2;
+; SM70-FTZ-DISALLOW-NEXT: selp.f32 %r6, %r3, %r5, %p3;
+; SM70-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r7, %r6;
+; SM70-FTZ-DISALLOW-NEXT: setp.ne.b32 %p4, %r1, %r7;
+; SM70-FTZ-DISALLOW-NEXT: mov.b32 %r7, %r1;
+; SM70-FTZ-DISALLOW-NEXT: @%p4 bra $L__BB65_1;
+; SM70-FTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-FTZ-DISALLOW-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-DISALLOW-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-FTZ-DISALLOW-NEXT: ret;
;
-; SM70-FTZ-LABEL: fmaximum_acq_rel_float_global_cta(
-; SM70-FTZ: {
-; SM70-FTZ-NEXT: .reg .pred %p<5>;
-; SM70-FTZ-NEXT: .reg .b32 %r<8>;
-; SM70-FTZ-NEXT: .reg .b64 %rd<2>;
-; SM70-FTZ-EMPTY:
-; SM70-FTZ-NEXT: // %bb.0:
-; SM70-FTZ-NEXT: ld.param.b32 %r2, [fmaximum_acq_rel_float_global_cta_param_1];
-; SM70-FTZ-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_float_global_cta_param_0];
-; SM70-FTZ-NEXT: fence.acq_rel.cta;
-; SM70-FTZ-NEXT: ld.relaxed.cta.global.b32 %r7, [%rd1];
-; SM70-FTZ-NEXT: $L__BB65_1: // %atomicrmw.start
-; SM70-FTZ-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-FTZ-NEXT: setp.eq.b32 %p1, %r7, 0;
-; SM70-FTZ-NEXT: selp.f32 %r3, %r7, %r2, %p1;
-; SM70-FTZ-NEXT: setp.nan.ftz.f32 %p2, %r7, %r2;
-; SM70-FTZ-NEXT: max.ftz.f32 %r4, %r7, %r2;
-; SM70-FTZ-NEXT: selp.f32 %r5, 0f7FC00000, %r4, %p2;
-; SM70-FTZ-NEXT: setp.eq.ftz.f32 %p3, %r7, %r2;
-; SM70-FTZ-NEXT: selp.f32 %r6, %r3, %r5, %p3;
-; SM70-FTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r7, %r6;
-; SM70-FTZ-NEXT: setp.ne.b32 %p4, %r1, %r7;
-; SM70-FTZ-NEXT: mov.b32 %r7, %r1;
-; SM70-FTZ-NEXT: @%p4 bra $L__BB65_1;
-; SM70-FTZ-NEXT: // %bb.2: // %atomicrmw.end
-; SM70-FTZ-NEXT: fence.acq_rel.cta;
-; SM70-FTZ-NEXT: st.param.b32 [func_retval0], %r1;
-; SM70-FTZ-NEXT: ret;
+; SM70-FTZ-ALLOW-LABEL: fmaximum_acq_rel_float_global_cta(
+; SM70-FTZ-ALLOW: {
+; SM70-FTZ-ALLOW-NEXT: .reg .pred %p<5>;
+; SM70-FTZ-ALLOW-NEXT: .reg .b32 %r<8>;
+; SM70-FTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
+; SM70-FTZ-ALLOW-EMPTY:
+; SM70-FTZ-ALLOW-NEXT: // %bb.0:
+; SM70-FTZ-ALLOW-NEXT: ld.param.b32 %r2, [fmaximum_acq_rel_float_global_cta_param_1];
+; SM70-FTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_float_global_cta_param_0];
+; SM70-FTZ-ALLOW-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-ALLOW-NEXT: ld.relaxed.cta.global.b32 %r7, [%rd1];
+; SM70-FTZ-ALLOW-NEXT: $L__BB65_1: // %atomicrmw.start
+; SM70-FTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-FTZ-ALLOW-NEXT: setp.eq.b32 %p1, %r7, 0;
+; SM70-FTZ-ALLOW-NEXT: selp.f32 %r3, %r7, %r2, %p1;
+; SM70-FTZ-ALLOW-NEXT: setp.nan.ftz.f32 %p2, %r7, %r2;
+; SM70-FTZ-ALLOW-NEXT: max.ftz.f32 %r4, %r7, %r2;
+; SM70-FTZ-ALLOW-NEXT: selp.f32 %r5, 0f7FC00000, %r4, %p2;
+; SM70-FTZ-ALLOW-NEXT: setp.eq.ftz.f32 %p3, %r7, %r2;
+; SM70-FTZ-ALLOW-NEXT: selp.f32 %r6, %r3, %r5, %p3;
+; SM70-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r7, %r6;
+; SM70-FTZ-ALLOW-NEXT: setp.ne.b32 %p4, %r1, %r7;
+; SM70-FTZ-ALLOW-NEXT: mov.b32 %r7, %r1;
+; SM70-FTZ-ALLOW-NEXT: @%p4 bra $L__BB65_1;
+; SM70-FTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-FTZ-ALLOW-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-ALLOW-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-FTZ-ALLOW-NEXT: ret;
%retval = atomicrmw fmaximum ptr addrspace(1) %addr, float %val syncscope("block") acq_rel
ret float %retval
}
@@ -2209,1079 +2495,2083 @@ define double @fmaximum_acq_rel_double_global_cta(ptr addrspace(1) %addr, double
}
define half @fadd_acq_rel_half_global_cta(ptr addrspace(1) %addr, half %val) {
-; SM70-NOFTZ-LABEL: fadd_acq_rel_half_global_cta(
-; SM70-NOFTZ: {
-; SM70-NOFTZ-NEXT: .reg .b16 %rs<3>;
-; SM70-NOFTZ-NEXT: .reg .b64 %rd<2>;
-; SM70-NOFTZ-EMPTY:
-; SM70-NOFTZ-NEXT: // %bb.0:
-; SM70-NOFTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_half_global_cta_param_0];
-; SM70-NOFTZ-NEXT: ld.param.b16 %rs1, [fadd_acq_rel_half_global_cta_param_1];
-; SM70-NOFTZ-NEXT: atom.acq_rel.cta.global.add.noftz.f16 %rs2, [%rd1], %rs1;
-; SM70-NOFTZ-NEXT: st.param.b16 [func_retval0], %rs2;
-; SM70-NOFTZ-NEXT: ret;
+; SM70-NOFTZ-DISALLOW-LABEL: fadd_acq_rel_half_global_cta(
+; SM70-NOFTZ-DISALLOW: {
+; SM70-NOFTZ-DISALLOW-NEXT: .reg .b16 %rs<3>;
+; SM70-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
+; SM70-NOFTZ-DISALLOW-EMPTY:
+; SM70-NOFTZ-DISALLOW-NEXT: // %bb.0:
+; SM70-NOFTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_half_global_cta_param_0];
+; SM70-NOFTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fadd_acq_rel_half_global_cta_param_1];
+; SM70-NOFTZ-DISALLOW-NEXT: atom.acq_rel.cta.global.add.noftz.f16 %rs2, [%rd1], %rs1;
+; SM70-NOFTZ-DISALLOW-NEXT: st.param.b16 [func_retval0], %rs2;
+; SM70-NOFTZ-DISALLOW-NEXT: ret;
;
-; SM70-FTZ-LABEL: fadd_acq_rel_half_global_cta(
-; SM70-FTZ: {
-; SM70-FTZ-NEXT: .reg .pred %p<2>;
-; SM70-FTZ-NEXT: .reg .b16 %rs<4>;
-; SM70-FTZ-NEXT: .reg .b32 %r<15>;
-; SM70-FTZ-NEXT: .reg .b64 %rd<3>;
-; SM70-FTZ-EMPTY:
-; SM70-FTZ-NEXT: // %bb.0:
-; SM70-FTZ-NEXT: ld.param.b16 %rs1, [fadd_acq_rel_half_global_cta_param_1];
-; SM70-FTZ-NEXT: ld.param.b64 %rd2, [fadd_acq_rel_half_global_cta_param_0];
-; SM70-FTZ-NEXT: fence.acq_rel.cta;
-; SM70-FTZ-NEXT: and.b64 %rd1, %rd2, -4;
-; SM70-FTZ-NEXT: cvt.u32.u64 %r4, %rd2;
-; SM70-FTZ-NEXT: and.b32 %r5, %r4, 3;
-; SM70-FTZ-NEXT: shl.b32 %r1, %r5, 3;
-; SM70-FTZ-NEXT: mov.b32 %r6, 65535;
-; SM70-FTZ-NEXT: shl.b32 %r7, %r6, %r1;
-; SM70-FTZ-NEXT: not.b32 %r2, %r7;
-; SM70-FTZ-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
-; SM70-FTZ-NEXT: $L__BB72_1: // %atomicrmw.start
-; SM70-FTZ-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-FTZ-NEXT: shr.u32 %r8, %r14, %r1;
-; SM70-FTZ-NEXT: cvt.u16.u32 %rs2, %r8;
-; SM70-FTZ-NEXT: add.rn.ftz.f16 %rs3, %rs2, %rs1;
-; SM70-FTZ-NEXT: cvt.u32.u16 %r9, %rs3;
-; SM70-FTZ-NEXT: shl.b32 %r10, %r9, %r1;
-; SM70-FTZ-NEXT: and.b32 %r11, %r14, %r2;
-; SM70-FTZ-NEXT: or.b32 %r12, %r11, %r10;
-; SM70-FTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
-; SM70-FTZ-NEXT: setp.ne.b32 %p1, %r3, %r14;
-; SM70-FTZ-NEXT: mov.b32 %r14, %r3;
-; SM70-FTZ-NEXT: @%p1 bra $L__BB72_1;
-; SM70-FTZ-NEXT: // %bb.2: // %atomicrmw.end
-; SM70-FTZ-NEXT: shr.u32 %r13, %r3, %r1;
-; SM70-FTZ-NEXT: fence.acq_rel.cta;
-; SM70-FTZ-NEXT: st.param.b16 [func_retval0], %r13;
-; SM70-FTZ-NEXT: ret;
+; SM70-NOFTZ-ALLOW-LABEL: fadd_acq_rel_half_global_cta(
+; SM70-NOFTZ-ALLOW: {
+; SM70-NOFTZ-ALLOW-NEXT: .reg .b16 %rs<3>;
+; SM70-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
+; SM70-NOFTZ-ALLOW-EMPTY:
+; SM70-NOFTZ-ALLOW-NEXT: // %bb.0:
+; SM70-NOFTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_half_global_cta_param_0];
+; SM70-NOFTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fadd_acq_rel_half_global_cta_param_1];
+; SM70-NOFTZ-ALLOW-NEXT: atom.acq_rel.cta.global.add.noftz.f16 %rs2, [%rd1], %rs1;
+; SM70-NOFTZ-ALLOW-NEXT: st.param.b16 [func_retval0], %rs2;
+; SM70-NOFTZ-ALLOW-NEXT: ret;
+;
+; SM70-FTZ-DISALLOW-LABEL: fadd_acq_rel_half_global_cta(
+; SM70-FTZ-DISALLOW: {
+; SM70-FTZ-DISALLOW-NEXT: .reg .pred %p<2>;
+; SM70-FTZ-DISALLOW-NEXT: .reg .b16 %rs<4>;
+; SM70-FTZ-DISALLOW-NEXT: .reg .b32 %r<15>;
+; SM70-FTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
+; SM70-FTZ-DISALLOW-EMPTY:
+; SM70-FTZ-DISALLOW-NEXT: // %bb.0:
+; SM70-FTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fadd_acq_rel_half_global_cta_param_1];
+; SM70-FTZ-DISALLOW-NEXT: ld.param.b64 %rd2, [fadd_acq_rel_half_global_cta_param_0];
+; SM70-FTZ-DISALLOW-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-DISALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-FTZ-DISALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-FTZ-DISALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM70-FTZ-DISALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-FTZ-DISALLOW-NEXT: mov.b32 %r6, 65535;
+; SM70-FTZ-DISALLOW-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-FTZ-DISALLOW-NEXT: not.b32 %r2, %r7;
+; SM70-FTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM70-FTZ-DISALLOW-NEXT: $L__BB72_1: // %atomicrmw.start
+; SM70-FTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-FTZ-DISALLOW-NEXT: shr.u32 %r8, %r14, %r1;
+; SM70-FTZ-DISALLOW-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM70-FTZ-DISALLOW-NEXT: add.rn.ftz.f16 %rs3, %rs2, %rs1;
+; SM70-FTZ-DISALLOW-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM70-FTZ-DISALLOW-NEXT: shl.b32 %r10, %r9, %r1;
+; SM70-FTZ-DISALLOW-NEXT: and.b32 %r11, %r14, %r2;
+; SM70-FTZ-DISALLOW-NEXT: or.b32 %r12, %r11, %r10;
+; SM70-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM70-FTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM70-FTZ-DISALLOW-NEXT: mov.b32 %r14, %r3;
+; SM70-FTZ-DISALLOW-NEXT: @%p1 bra $L__BB72_1;
+; SM70-FTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-FTZ-DISALLOW-NEXT: shr.u32 %r13, %r3, %r1;
+; SM70-FTZ-DISALLOW-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-DISALLOW-NEXT: st.param.b16 [func_retval0], %r13;
+; SM70-FTZ-DISALLOW-NEXT: ret;
+;
+; SM70-FTZ-ALLOW-LABEL: fadd_acq_rel_half_global_cta(
+; SM70-FTZ-ALLOW: {
+; SM70-FTZ-ALLOW-NEXT: .reg .b16 %rs<3>;
+; SM70-FTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
+; SM70-FTZ-ALLOW-EMPTY:
+; SM70-FTZ-ALLOW-NEXT: // %bb.0:
+; SM70-FTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_half_global_cta_param_0];
+; SM70-FTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fadd_acq_rel_half_global_cta_param_1];
+; SM70-FTZ-ALLOW-NEXT: atom.acq_rel.cta.global.add.noftz.f16 %rs2, [%rd1], %rs1;
+; SM70-FTZ-ALLOW-NEXT: st.param.b16 [func_retval0], %rs2;
+; SM70-FTZ-ALLOW-NEXT: ret;
%retval = atomicrmw fadd ptr addrspace(1) %addr, half %val syncscope("block") acq_rel
ret half %retval
}
define half @fsub_acq_rel_half_global_cta(ptr addrspace(1) %addr, half %val) {
-; SM70-NOFTZ-LABEL: fsub_acq_rel_half_global_cta(
-; SM70-NOFTZ: {
-; SM70-NOFTZ-NEXT: .reg .pred %p<2>;
-; SM70-NOFTZ-NEXT: .reg .b16 %rs<4>;
-; SM70-NOFTZ-NEXT: .reg .b32 %r<15>;
-; SM70-NOFTZ-NEXT: .reg .b64 %rd<3>;
-; SM70-NOFTZ-EMPTY:
-; SM70-NOFTZ-NEXT: // %bb.0:
-; SM70-NOFTZ-NEXT: ld.param.b16 %rs1, [fsub_acq_rel_half_global_cta_param_1];
-; SM70-NOFTZ-NEXT: ld.param.b64 %rd2, [fsub_acq_rel_half_global_cta_param_0];
-; SM70-NOFTZ-NEXT: fence.acq_rel.cta;
-; SM70-NOFTZ-NEXT: and.b64 %rd1, %rd2, -4;
-; SM70-NOFTZ-NEXT: cvt.u32.u64 %r4, %rd2;
-; SM70-NOFTZ-NEXT: and.b32 %r5, %r4, 3;
-; SM70-NOFTZ-NEXT: shl.b32 %r1, %r5, 3;
-; SM70-NOFTZ-NEXT: mov.b32 %r6, 65535;
-; SM70-NOFTZ-NEXT: shl.b32 %r7, %r6, %r1;
-; SM70-NOFTZ-NEXT: not.b32 %r2, %r7;
-; SM70-NOFTZ-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
-; SM70-NOFTZ-NEXT: $L__BB73_1: // %atomicrmw.start
-; SM70-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NOFTZ-NEXT: shr.u32 %r8, %r14, %r1;
-; SM70-NOFTZ-NEXT: cvt.u16.u32 %rs2, %r8;
-; SM70-NOFTZ-NEXT: sub.rn.f16 %rs3, %rs2, %rs1;
-; SM70-NOFTZ-NEXT: cvt.u32.u16 %r9, %rs3;
-; SM70-NOFTZ-NEXT: shl.b32 %r10, %r9, %r1;
-; SM70-NOFTZ-NEXT: and.b32 %r11, %r14, %r2;
-; SM70-NOFTZ-NEXT: or.b32 %r12, %r11, %r10;
-; SM70-NOFTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
-; SM70-NOFTZ-NEXT: setp.ne.b32 %p1, %r3, %r14;
-; SM70-NOFTZ-NEXT: mov.b32 %r14, %r3;
-; SM70-NOFTZ-NEXT: @%p1 bra $L__BB73_1;
-; SM70-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end
-; SM70-NOFTZ-NEXT: shr.u32 %r13, %r3, %r1;
-; SM70-NOFTZ-NEXT: fence.acq_rel.cta;
-; SM70-NOFTZ-NEXT: st.param.b16 [func_retval0], %r13;
-; SM70-NOFTZ-NEXT: ret;
+; SM70-NOFTZ-DISALLOW-LABEL: fsub_acq_rel_half_global_cta(
+; SM70-NOFTZ-DISALLOW: {
+; SM70-NOFTZ-DISALLOW-NEXT: .reg .pred %p<2>;
+; SM70-NOFTZ-DISALLOW-NEXT: .reg .b16 %rs<4>;
+; SM70-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<15>;
+; SM70-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
+; SM70-NOFTZ-DISALLOW-EMPTY:
+; SM70-NOFTZ-DISALLOW-NEXT: // %bb.0:
+; SM70-NOFTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fsub_acq_rel_half_global_cta_param_1];
+; SM70-NOFTZ-DISALLOW-NEXT: ld.param.b64 %rd2, [fsub_acq_rel_half_global_cta_param_0];
+; SM70-NOFTZ-DISALLOW-NEXT: fence.acq_rel.cta;
+; SM70-NOFTZ-DISALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-NOFTZ-DISALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM70-NOFTZ-DISALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-NOFTZ-DISALLOW-NEXT: mov.b32 %r6, 65535;
+; SM70-NOFTZ-DISALLOW-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-NOFTZ-DISALLOW-NEXT: not.b32 %r2, %r7;
+; SM70-NOFTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM70-NOFTZ-DISALLOW-NEXT: $L__BB73_1: // %atomicrmw.start
+; SM70-NOFTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NOFTZ-DISALLOW-NEXT: shr.u32 %r8, %r14, %r1;
+; SM70-NOFTZ-DISALLOW-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM70-NOFTZ-DISALLOW-NEXT: sub.rn.f16 %rs3, %rs2, %rs1;
+; SM70-NOFTZ-DISALLOW-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM70-NOFTZ-DISALLOW-NEXT: shl.b32 %r10, %r9, %r1;
+; SM70-NOFTZ-DISALLOW-NEXT: and.b32 %r11, %r14, %r2;
+; SM70-NOFTZ-DISALLOW-NEXT: or.b32 %r12, %r11, %r10;
+; SM70-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM70-NOFTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM70-NOFTZ-DISALLOW-NEXT: mov.b32 %r14, %r3;
+; SM70-NOFTZ-DISALLOW-NEXT: @%p1 bra $L__BB73_1;
+; SM70-NOFTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NOFTZ-DISALLOW-NEXT: shr.u32 %r13, %r3, %r1;
+; SM70-NOFTZ-DISALLOW-NEXT: fence.acq_rel.cta;
+; SM70-NOFTZ-DISALLOW-NEXT: st.param.b16 [func_retval0], %r13;
+; SM70-NOFTZ-DISALLOW-NEXT: ret;
+;
+; SM70-NOFTZ-ALLOW-LABEL: fsub_acq_rel_half_global_cta(
+; SM70-NOFTZ-ALLOW: {
+; SM70-NOFTZ-ALLOW-NEXT: .reg .pred %p<2>;
+; SM70-NOFTZ-ALLOW-NEXT: .reg .b16 %rs<4>;
+; SM70-NOFTZ-ALLOW-NEXT: .reg .b32 %r<15>;
+; SM70-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
+; SM70-NOFTZ-ALLOW-EMPTY:
+; SM70-NOFTZ-ALLOW-NEXT: // %bb.0:
+; SM70-NOFTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fsub_acq_rel_half_global_cta_param_1];
+; SM70-NOFTZ-ALLOW-NEXT: ld.param.b64 %rd2, [fsub_acq_rel_half_global_cta_param_0];
+; SM70-NOFTZ-ALLOW-NEXT: fence.acq_rel.cta;
+; SM70-NOFTZ-ALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NOFTZ-ALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-NOFTZ-ALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM70-NOFTZ-ALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-NOFTZ-ALLOW-NEXT: mov.b32 %r6, 65535;
+; SM70-NOFTZ-ALLOW-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-NOFTZ-ALLOW-NEXT: not.b32 %r2, %r7;
+; SM70-NOFTZ-ALLOW-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM70-NOFTZ-ALLOW-NEXT: $L__BB73_1: // %atomicrmw.start
+; SM70-NOFTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NOFTZ-ALLOW-NEXT: shr.u32 %r8, %r14, %r1;
+; SM70-NOFTZ-ALLOW-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM70-NOFTZ-ALLOW-NEXT: sub.rn.f16 %rs3, %rs2, %rs1;
+; SM70-NOFTZ-ALLOW-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM70-NOFTZ-ALLOW-NEXT: shl.b32 %r10, %r9, %r1;
+; SM70-NOFTZ-ALLOW-NEXT: and.b32 %r11, %r14, %r2;
+; SM70-NOFTZ-ALLOW-NEXT: or.b32 %r12, %r11, %r10;
+; SM70-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM70-NOFTZ-ALLOW-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM70-NOFTZ-ALLOW-NEXT: mov.b32 %r14, %r3;
+; SM70-NOFTZ-ALLOW-NEXT: @%p1 bra $L__BB73_1;
+; SM70-NOFTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NOFTZ-ALLOW-NEXT: shr.u32 %r13, %r3, %r1;
+; SM70-NOFTZ-ALLOW-NEXT: fence.acq_rel.cta;
+; SM70-NOFTZ-ALLOW-NEXT: st.param.b16 [func_retval0], %r13;
+; SM70-NOFTZ-ALLOW-NEXT: ret;
;
-; SM70-FTZ-LABEL: fsub_acq_rel_half_global_cta(
-; SM70-FTZ: {
-; SM70-FTZ-NEXT: .reg .pred %p<2>;
-; SM70-FTZ-NEXT: .reg .b16 %rs<4>;
-; SM70-FTZ-NEXT: .reg .b32 %r<15>;
-; SM70-FTZ-NEXT: .reg .b64 %rd<3>;
-; SM70-FTZ-EMPTY:
-; SM70-FTZ-NEXT: // %bb.0:
-; SM70-FTZ-NEXT: ld.param.b16 %rs1, [fsub_acq_rel_half_global_cta_param_1];
-; SM70-FTZ-NEXT: ld.param.b64 %rd2, [fsub_acq_rel_half_global_cta_param_0];
-; SM70-FTZ-NEXT: fence.acq_rel.cta;
-; SM70-FTZ-NEXT: and.b64 %rd1, %rd2, -4;
-; SM70-FTZ-NEXT: cvt.u32.u64 %r4, %rd2;
-; SM70-FTZ-NEXT: and.b32 %r5, %r4, 3;
-; SM70-FTZ-NEXT: shl.b32 %r1, %r5, 3;
-; SM70-FTZ-NEXT: mov.b32 %r6, 65535;
-; SM70-FTZ-NEXT: shl.b32 %r7, %r6, %r1;
-; SM70-FTZ-NEXT: not.b32 %r2, %r7;
-; SM70-FTZ-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
-; SM70-FTZ-NEXT: $L__BB73_1: // %atomicrmw.start
-; SM70-FTZ-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-FTZ-NEXT: shr.u32 %r8, %r14, %r1;
-; SM70-FTZ-NEXT: cvt.u16.u32 %rs2, %r8;
-; SM70-FTZ-NEXT: sub.rn.ftz.f16 %rs3, %rs2, %rs1;
-; SM70-FTZ-NEXT: cvt.u32.u16 %r9, %rs3;
-; SM70-FTZ-NEXT: shl.b32 %r10, %r9, %r1;
-; SM70-FTZ-NEXT: and.b32 %r11, %r14, %r2;
-; SM70-FTZ-NEXT: or.b32 %r12, %r11, %r10;
-; SM70-FTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
-; SM70-FTZ-NEXT: setp.ne.b32 %p1, %r3, %r14;
-; SM70-FTZ-NEXT: mov.b32 %r14, %r3;
-; SM70-FTZ-NEXT: @%p1 bra $L__BB73_1;
-; SM70-FTZ-NEXT: // %bb.2: // %atomicrmw.end
-; SM70-FTZ-NEXT: shr.u32 %r13, %r3, %r1;
-; SM70-FTZ-NEXT: fence.acq_rel.cta;
-; SM70-FTZ-NEXT: st.param.b16 [func_retval0], %r13;
-; SM70-FTZ-NEXT: ret;
+; SM70-FTZ-DISALLOW-LABEL: fsub_acq_rel_half_global_cta(
+; SM70-FTZ-DISALLOW: {
+; SM70-FTZ-DISALLOW-NEXT: .reg .pred %p<2>;
+; SM70-FTZ-DISALLOW-NEXT: .reg .b16 %rs<4>;
+; SM70-FTZ-DISALLOW-NEXT: .reg .b32 %r<15>;
+; SM70-FTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
+; SM70-FTZ-DISALLOW-EMPTY:
+; SM70-FTZ-DISALLOW-NEXT: // %bb.0:
+; SM70-FTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fsub_acq_rel_half_global_cta_param_1];
+; SM70-FTZ-DISALLOW-NEXT: ld.param.b64 %rd2, [fsub_acq_rel_half_global_cta_param_0];
+; SM70-FTZ-DISALLOW-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-DISALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-FTZ-DISALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-FTZ-DISALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM70-FTZ-DISALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-FTZ-DISALLOW-NEXT: mov.b32 %r6, 65535;
+; SM70-FTZ-DISALLOW-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-FTZ-DISALLOW-NEXT: not.b32 %r2, %r7;
+; SM70-FTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM70-FTZ-DISALLOW-NEXT: $L__BB73_1: // %atomicrmw.start
+; SM70-FTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-FTZ-DISALLOW-NEXT: shr.u32 %r8, %r14, %r1;
+; SM70-FTZ-DISALLOW-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM70-FTZ-DISALLOW-NEXT: sub.rn.ftz.f16 %rs3, %rs2, %rs1;
+; SM70-FTZ-DISALLOW-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM70-FTZ-DISALLOW-NEXT: shl.b32 %r10, %r9, %r1;
+; SM70-FTZ-DISALLOW-NEXT: and.b32 %r11, %r14, %r2;
+; SM70-FTZ-DISALLOW-NEXT: or.b32 %r12, %r11, %r10;
+; SM70-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM70-FTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM70-FTZ-DISALLOW-NEXT: mov.b32 %r14, %r3;
+; SM70-FTZ-DISALLOW-NEXT: @%p1 bra $L__BB73_1;
+; SM70-FTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-FTZ-DISALLOW-NEXT: shr.u32 %r13, %r3, %r1;
+; SM70-FTZ-DISALLOW-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-DISALLOW-NEXT: st.param.b16 [func_retval0], %r13;
+; SM70-FTZ-DISALLOW-NEXT: ret;
+;
+; SM70-FTZ-ALLOW-LABEL: fsub_acq_rel_half_global_cta(
+; SM70-FTZ-ALLOW: {
+; SM70-FTZ-ALLOW-NEXT: .reg .pred %p<2>;
+; SM70-FTZ-ALLOW-NEXT: .reg .b16 %rs<4>;
+; SM70-FTZ-ALLOW-NEXT: .reg .b32 %r<15>;
+; SM70-FTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
+; SM70-FTZ-ALLOW-EMPTY:
+; SM70-FTZ-ALLOW-NEXT: // %bb.0:
+; SM70-FTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fsub_acq_rel_half_global_cta_param_1];
+; SM70-FTZ-ALLOW-NEXT: ld.param.b64 %rd2, [fsub_acq_rel_half_global_cta_param_0];
+; SM70-FTZ-ALLOW-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-ALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-FTZ-ALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-FTZ-ALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM70-FTZ-ALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-FTZ-ALLOW-NEXT: mov.b32 %r6, 65535;
+; SM70-FTZ-ALLOW-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-FTZ-ALLOW-NEXT: not.b32 %r2, %r7;
+; SM70-FTZ-ALLOW-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM70-FTZ-ALLOW-NEXT: $L__BB73_1: // %atomicrmw.start
+; SM70-FTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-FTZ-ALLOW-NEXT: shr.u32 %r8, %r14, %r1;
+; SM70-FTZ-ALLOW-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM70-FTZ-ALLOW-NEXT: sub.rn.ftz.f16 %rs3, %rs2, %rs1;
+; SM70-FTZ-ALLOW-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM70-FTZ-ALLOW-NEXT: shl.b32 %r10, %r9, %r1;
+; SM70-FTZ-ALLOW-NEXT: and.b32 %r11, %r14, %r2;
+; SM70-FTZ-ALLOW-NEXT: or.b32 %r12, %r11, %r10;
+; SM70-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM70-FTZ-ALLOW-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM70-FTZ-ALLOW-NEXT: mov.b32 %r14, %r3;
+; SM70-FTZ-ALLOW-NEXT: @%p1 bra $L__BB73_1;
+; SM70-FTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-FTZ-ALLOW-NEXT: shr.u32 %r13, %r3, %r1;
+; SM70-FTZ-ALLOW-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-ALLOW-NEXT: st.param.b16 [func_retval0], %r13;
+; SM70-FTZ-ALLOW-NEXT: ret;
%retval = atomicrmw fsub ptr addrspace(1) %addr, half %val syncscope("block") acq_rel
ret half %retval
}
define half @fmin_acq_rel_half_global_cta(ptr addrspace(1) %addr, half %val) {
-; SM70-NOFTZ-LABEL: fmin_acq_rel_half_global_cta(
-; SM70-NOFTZ: {
-; SM70-NOFTZ-NEXT: .reg .pred %p<2>;
-; SM70-NOFTZ-NEXT: .reg .b16 %rs<4>;
-; SM70-NOFTZ-NEXT: .reg .b32 %r<18>;
-; SM70-NOFTZ-NEXT: .reg .b64 %rd<3>;
-; SM70-NOFTZ-EMPTY:
-; SM70-NOFTZ-NEXT: // %bb.0:
-; SM70-NOFTZ-NEXT: ld.param.b16 %rs1, [fmin_acq_rel_half_global_cta_param_1];
-; SM70-NOFTZ-NEXT: ld.param.b64 %rd2, [fmin_acq_rel_half_global_cta_param_0];
-; SM70-NOFTZ-NEXT: fence.acq_rel.cta;
-; SM70-NOFTZ-NEXT: and.b64 %rd1, %rd2, -4;
-; SM70-NOFTZ-NEXT: cvt.u32.u64 %r4, %rd2;
-; SM70-NOFTZ-NEXT: and.b32 %r5, %r4, 3;
-; SM70-NOFTZ-NEXT: shl.b32 %r1, %r5, 3;
-; SM70-NOFTZ-NEXT: mov.b32 %r6, 65535;
-; SM70-NOFTZ-NEXT: shl.b32 %r7, %r6, %r1;
-; SM70-NOFTZ-NEXT: not.b32 %r2, %r7;
-; SM70-NOFTZ-NEXT: ld.relaxed.cta.global.b32 %r17, [%rd1];
-; SM70-NOFTZ-NEXT: cvt.f32.f16 %r10, %rs1;
-; SM70-NOFTZ-NEXT: $L__BB74_1: // %atomicrmw.start
-; SM70-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NOFTZ-NEXT: shr.u32 %r8, %r17, %r1;
-; SM70-NOFTZ-NEXT: cvt.u16.u32 %rs2, %r8;
-; SM70-NOFTZ-NEXT: cvt.f32.f16 %r9, %rs2;
-; SM70-NOFTZ-NEXT: min.f32 %r11, %r9, %r10;
-; SM70-NOFTZ-NEXT: cvt.rn.f16.f32 %rs3, %r11;
-; SM70-NOFTZ-NEXT: cvt.u32.u16 %r12, %rs3;
-; SM70-NOFTZ-NEXT: shl.b32 %r13, %r12, %r1;
-; SM70-NOFTZ-NEXT: and.b32 %r14, %r17, %r2;
-; SM70-NOFTZ-NEXT: or.b32 %r15, %r14, %r13;
-; SM70-NOFTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r17, %r15;
-; SM70-NOFTZ-NEXT: setp.ne.b32 %p1, %r3, %r17;
-; SM70-NOFTZ-NEXT: mov.b32 %r17, %r3;
-; SM70-NOFTZ-NEXT: @%p1 bra $L__BB74_1;
-; SM70-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end
-; SM70-NOFTZ-NEXT: shr.u32 %r16, %r3, %r1;
-; SM70-NOFTZ-NEXT: fence.acq_rel.cta;
-; SM70-NOFTZ-NEXT: st.param.b16 [func_retval0], %r16;
-; SM70-NOFTZ-NEXT: ret;
+; SM70-NOFTZ-DISALLOW-LABEL: fmin_acq_rel_half_global_cta(
+; SM70-NOFTZ-DISALLOW: {
+; SM70-NOFTZ-DISALLOW-NEXT: .reg .pred %p<2>;
+; SM70-NOFTZ-DISALLOW-NEXT: .reg .b16 %rs<4>;
+; SM70-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<18>;
+; SM70-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
+; SM70-NOFTZ-DISALLOW-EMPTY:
+; SM70-NOFTZ-DISALLOW-NEXT: // %bb.0:
+; SM70-NOFTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fmin_acq_rel_half_global_cta_param_1];
+; SM70-NOFTZ-DISALLOW-NEXT: ld.param.b64 %rd2, [fmin_acq_rel_half_global_cta_param_0];
+; SM70-NOFTZ-DISALLOW-NEXT: fence.acq_rel.cta;
+; SM70-NOFTZ-DISALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-NOFTZ-DISALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM70-NOFTZ-DISALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-NOFTZ-DISALLOW-NEXT: mov.b32 %r6, 65535;
+; SM70-NOFTZ-DISALLOW-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-NOFTZ-DISALLOW-NEXT: not.b32 %r2, %r7;
+; SM70-NOFTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b32 %r17, [%rd1];
+; SM70-NOFTZ-DISALLOW-NEXT: cvt.f32.f16 %r10, %rs1;
+; SM70-NOFTZ-DISALLOW-NEXT: $L__BB74_1: // %atomicrmw.start
+; SM70-NOFTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NOFTZ-DISALLOW-NEXT: shr.u32 %r8, %r17, %r1;
+; SM70-NOFTZ-DISALLOW-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM70-NOFTZ-DISALLOW-NEXT: cvt.f32.f16 %r9, %rs2;
+; SM70-NOFTZ-DISALLOW-NEXT: min.f32 %r11, %r9, %r10;
+; SM70-NOFTZ-DISALLOW-NEXT: cvt.rn.f16.f32 %rs3, %r11;
+; SM70-NOFTZ-DISALLOW-NEXT: cvt.u32.u16 %r12, %rs3;
+; SM70-NOFTZ-DISALLOW-NEXT: shl.b32 %r13, %r12, %r1;
+; SM70-NOFTZ-DISALLOW-NEXT: and.b32 %r14, %r17, %r2;
+; SM70-NOFTZ-DISALLOW-NEXT: or.b32 %r15, %r14, %r13;
+; SM70-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r17, %r15;
+; SM70-NOFTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r3, %r17;
+; SM70-NOFTZ-DISALLOW-NEXT: mov.b32 %r17, %r3;
+; SM70-NOFTZ-DISALLOW-NEXT: @%p1 bra $L__BB74_1;
+; SM70-NOFTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NOFTZ-DISALLOW-NEXT: shr.u32 %r16, %r3, %r1;
+; SM70-NOFTZ-DISALLOW-NEXT: fence.acq_rel.cta;
+; SM70-NOFTZ-DISALLOW-NEXT: st.param.b16 [func_retval0], %r16;
+; SM70-NOFTZ-DISALLOW-NEXT: ret;
+;
+; SM70-NOFTZ-ALLOW-LABEL: fmin_acq_rel_half_global_cta(
+; SM70-NOFTZ-ALLOW: {
+; SM70-NOFTZ-ALLOW-NEXT: .reg .pred %p<2>;
+; SM70-NOFTZ-ALLOW-NEXT: .reg .b16 %rs<4>;
+; SM70-NOFTZ-ALLOW-NEXT: .reg .b32 %r<18>;
+; SM70-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
+; SM70-NOFTZ-ALLOW-EMPTY:
+; SM70-NOFTZ-ALLOW-NEXT: // %bb.0:
+; SM70-NOFTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fmin_acq_rel_half_global_cta_param_1];
+; SM70-NOFTZ-ALLOW-NEXT: ld.param.b64 %rd2, [fmin_acq_rel_half_global_cta_param_0];
+; SM70-NOFTZ-ALLOW-NEXT: fence.acq_rel.cta;
+; SM70-NOFTZ-ALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NOFTZ-ALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-NOFTZ-ALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM70-NOFTZ-ALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-NOFTZ-ALLOW-NEXT: mov.b32 %r6, 65535;
+; SM70-NOFTZ-ALLOW-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-NOFTZ-ALLOW-NEXT: not.b32 %r2, %r7;
+; SM70-NOFTZ-ALLOW-NEXT: ld.relaxed.cta.global.b32 %r17, [%rd1];
+; SM70-NOFTZ-ALLOW-NEXT: cvt.f32.f16 %r10, %rs1;
+; SM70-NOFTZ-ALLOW-NEXT: $L__BB74_1: // %atomicrmw.start
+; SM70-NOFTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NOFTZ-ALLOW-NEXT: shr.u32 %r8, %r17, %r1;
+; SM70-NOFTZ-ALLOW-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM70-NOFTZ-ALLOW-NEXT: cvt.f32.f16 %r9, %rs2;
+; SM70-NOFTZ-ALLOW-NEXT: min.f32 %r11, %r9, %r10;
+; SM70-NOFTZ-ALLOW-NEXT: cvt.rn.f16.f32 %rs3, %r11;
+; SM70-NOFTZ-ALLOW-NEXT: cvt.u32.u16 %r12, %rs3;
+; SM70-NOFTZ-ALLOW-NEXT: shl.b32 %r13, %r12, %r1;
+; SM70-NOFTZ-ALLOW-NEXT: and.b32 %r14, %r17, %r2;
+; SM70-NOFTZ-ALLOW-NEXT: or.b32 %r15, %r14, %r13;
+; SM70-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r17, %r15;
+; SM70-NOFTZ-ALLOW-NEXT: setp.ne.b32 %p1, %r3, %r17;
+; SM70-NOFTZ-ALLOW-NEXT: mov.b32 %r17, %r3;
+; SM70-NOFTZ-ALLOW-NEXT: @%p1 bra $L__BB74_1;
+; SM70-NOFTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NOFTZ-ALLOW-NEXT: shr.u32 %r16, %r3, %r1;
+; SM70-NOFTZ-ALLOW-NEXT: fence.acq_rel.cta;
+; SM70-NOFTZ-ALLOW-NEXT: st.param.b16 [func_retval0], %r16;
+; SM70-NOFTZ-ALLOW-NEXT: ret;
+;
+; SM70-FTZ-DISALLOW-LABEL: fmin_acq_rel_half_global_cta(
+; SM70-FTZ-DISALLOW: {
+; SM70-FTZ-DISALLOW-NEXT: .reg .pred %p<2>;
+; SM70-FTZ-DISALLOW-NEXT: .reg .b16 %rs<4>;
+; SM70-FTZ-DISALLOW-NEXT: .reg .b32 %r<18>;
+; SM70-FTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
+; SM70-FTZ-DISALLOW-EMPTY:
+; SM70-FTZ-DISALLOW-NEXT: // %bb.0:
+; SM70-FTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fmin_acq_rel_half_global_cta_param_1];
+; SM70-FTZ-DISALLOW-NEXT: ld.param.b64 %rd2, [fmin_acq_rel_half_global_cta_param_0];
+; SM70-FTZ-DISALLOW-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-DISALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-FTZ-DISALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-FTZ-DISALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM70-FTZ-DISALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-FTZ-DISALLOW-NEXT: mov.b32 %r6, 65535;
+; SM70-FTZ-DISALLOW-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-FTZ-DISALLOW-NEXT: not.b32 %r2, %r7;
+; SM70-FTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b32 %r17, [%rd1];
+; SM70-FTZ-DISALLOW-NEXT: cvt.ftz.f32.f16 %r10, %rs1;
+; SM70-FTZ-DISALLOW-NEXT: $L__BB74_1: // %atomicrmw.start
+; SM70-FTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-FTZ-DISALLOW-NEXT: shr.u32 %r8, %r17, %r1;
+; SM70-FTZ-DISALLOW-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM70-FTZ-DISALLOW-NEXT: cvt.ftz.f32.f16 %r9, %rs2;
+; SM70-FTZ-DISALLOW-NEXT: min.ftz.f32 %r11, %r9, %r10;
+; SM70-FTZ-DISALLOW-NEXT: cvt.rn.f16.f32 %rs3, %r11;
+; SM70-FTZ-DISALLOW-NEXT: cvt.u32.u16 %r12, %rs3;
+; SM70-FTZ-DISALLOW-NEXT: shl.b32 %r13, %r12, %r1;
+; SM70-FTZ-DISALLOW-NEXT: and.b32 %r14, %r17, %r2;
+; SM70-FTZ-DISALLOW-NEXT: or.b32 %r15, %r14, %r13;
+; SM70-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r17, %r15;
+; SM70-FTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r3, %r17;
+; SM70-FTZ-DISALLOW-NEXT: mov.b32 %r17, %r3;
+; SM70-FTZ-DISALLOW-NEXT: @%p1 bra $L__BB74_1;
+; SM70-FTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-FTZ-DISALLOW-NEXT: shr.u32 %r16, %r3, %r1;
+; SM70-FTZ-DISALLOW-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-DISALLOW-NEXT: st.param.b16 [func_retval0], %r16;
+; SM70-FTZ-DISALLOW-NEXT: ret;
;
-; SM70-FTZ-LABEL: fmin_acq_rel_half_global_cta(
-; SM70-FTZ: {
-; SM70-FTZ-NEXT: .reg .pred %p<2>;
-; SM70-FTZ-NEXT: .reg .b16 %rs<4>;
-; SM70-FTZ-NEXT: .reg .b32 %r<18>;
-; SM70-FTZ-NEXT: .reg .b64 %rd<3>;
-; SM70-FTZ-EMPTY:
-; SM70-FTZ-NEXT: // %bb.0:
-; SM70-FTZ-NEXT: ld.param.b16 %rs1, [fmin_acq_rel_half_global_cta_param_1];
-; SM70-FTZ-NEXT: ld.param.b64 %rd2, [fmin_acq_rel_half_global_cta_param_0];
-; SM70-FTZ-NEXT: fence.acq_rel.cta;
-; SM70-FTZ-NEXT: and.b64 %rd1, %rd2, -4;
-; SM70-FTZ-NEXT: cvt.u32.u64 %r4, %rd2;
-; SM70-FTZ-NEXT: and.b32 %r5, %r4, 3;
-; SM70-FTZ-NEXT: shl.b32 %r1, %r5, 3;
-; SM70-FTZ-NEXT: mov.b32 %r6, 65535;
-; SM70-FTZ-NEXT: shl.b32 %r7, %r6, %r1;
-; SM70-FTZ-NEXT: not.b32 %r2, %r7;
-; SM70-FTZ-NEXT: ld.relaxed.cta.global.b32 %r17, [%rd1];
-; SM70-FTZ-NEXT: cvt.ftz.f32.f16 %r10, %rs1;
-; SM70-FTZ-NEXT: $L__BB74_1: // %atomicrmw.start
-; SM70-FTZ-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-FTZ-NEXT: shr.u32 %r8, %r17, %r1;
-; SM70-FTZ-NEXT: cvt.u16.u32 %rs2, %r8;
-; SM70-FTZ-NEXT: cvt.ftz.f32.f16 %r9, %rs2;
-; SM70-FTZ-NEXT: min.ftz.f32 %r11, %r9, %r10;
-; SM70-FTZ-NEXT: cvt.rn.f16.f32 %rs3, %r11;
-; SM70-FTZ-NEXT: cvt.u32.u16 %r12, %rs3;
-; SM70-FTZ-NEXT: shl.b32 %r13, %r12, %r1;
-; SM70-FTZ-NEXT: and.b32 %r14, %r17, %r2;
-; SM70-FTZ-NEXT: or.b32 %r15, %r14, %r13;
-; SM70-FTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r17, %r15;
-; SM70-FTZ-NEXT: setp.ne.b32 %p1, %r3, %r17;
-; SM70-FTZ-NEXT: mov.b32 %r17, %r3;
-; SM70-FTZ-NEXT: @%p1 bra $L__BB74_1;
-; SM70-FTZ-NEXT: // %bb.2: // %atomicrmw.end
-; SM70-FTZ-NEXT: shr.u32 %r16, %r3, %r1;
-; SM70-FTZ-NEXT: fence.acq_rel.cta;
-; SM70-FTZ-NEXT: st.param.b16 [func_retval0], %r16;
-; SM70-FTZ-NEXT: ret;
+; SM70-FTZ-ALLOW-LABEL: fmin_acq_rel_half_global_cta(
+; SM70-FTZ-ALLOW: {
+; SM70-FTZ-ALLOW-NEXT: .reg .pred %p<2>;
+; SM70-FTZ-ALLOW-NEXT: .reg .b16 %rs<4>;
+; SM70-FTZ-ALLOW-NEXT: .reg .b32 %r<18>;
+; SM70-FTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
+; SM70-FTZ-ALLOW-EMPTY:
+; SM70-FTZ-ALLOW-NEXT: // %bb.0:
+; SM70-FTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fmin_acq_rel_half_global_cta_param_1];
+; SM70-FTZ-ALLOW-NEXT: ld.param.b64 %rd2, [fmin_acq_rel_half_global_cta_param_0];
+; SM70-FTZ-ALLOW-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-ALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-FTZ-ALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-FTZ-ALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM70-FTZ-ALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-FTZ-ALLOW-NEXT: mov.b32 %r6, 65535;
+; SM70-FTZ-ALLOW-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-FTZ-ALLOW-NEXT: not.b32 %r2, %r7;
+; SM70-FTZ-ALLOW-NEXT: ld.relaxed.cta.global.b32 %r17, [%rd1];
+; SM70-FTZ-ALLOW-NEXT: cvt.ftz.f32.f16 %r10, %rs1;
+; SM70-FTZ-ALLOW-NEXT: $L__BB74_1: // %atomicrmw.start
+; SM70-FTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-FTZ-ALLOW-NEXT: shr.u32 %r8, %r17, %r1;
+; SM70-FTZ-ALLOW-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM70-FTZ-ALLOW-NEXT: cvt.ftz.f32.f16 %r9, %rs2;
+; SM70-FTZ-ALLOW-NEXT: min.ftz.f32 %r11, %r9, %r10;
+; SM70-FTZ-ALLOW-NEXT: cvt.rn.f16.f32 %rs3, %r11;
+; SM70-FTZ-ALLOW-NEXT: cvt.u32.u16 %r12, %rs3;
+; SM70-FTZ-ALLOW-NEXT: shl.b32 %r13, %r12, %r1;
+; SM70-FTZ-ALLOW-NEXT: and.b32 %r14, %r17, %r2;
+; SM70-FTZ-ALLOW-NEXT: or.b32 %r15, %r14, %r13;
+; SM70-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r17, %r15;
+; SM70-FTZ-ALLOW-NEXT: setp.ne.b32 %p1, %r3, %r17;
+; SM70-FTZ-ALLOW-NEXT: mov.b32 %r17, %r3;
+; SM70-FTZ-ALLOW-NEXT: @%p1 bra $L__BB74_1;
+; SM70-FTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-FTZ-ALLOW-NEXT: shr.u32 %r16, %r3, %r1;
+; SM70-FTZ-ALLOW-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-ALLOW-NEXT: st.param.b16 [func_retval0], %r16;
+; SM70-FTZ-ALLOW-NEXT: ret;
%retval = atomicrmw fmin ptr addrspace(1) %addr, half %val syncscope("block") acq_rel
ret half %retval
}
define half @fmax_acq_rel_half_global_cta(ptr addrspace(1) %addr, half %val) {
-; SM70-NOFTZ-LABEL: fmax_acq_rel_half_global_cta(
-; SM70-NOFTZ: {
-; SM70-NOFTZ-NEXT: .reg .pred %p<2>;
-; SM70-NOFTZ-NEXT: .reg .b16 %rs<4>;
-; SM70-NOFTZ-NEXT: .reg .b32 %r<18>;
-; SM70-NOFTZ-NEXT: .reg .b64 %rd<3>;
-; SM70-NOFTZ-EMPTY:
-; SM70-NOFTZ-NEXT: // %bb.0:
-; SM70-NOFTZ-NEXT: ld.param.b16 %rs1, [fmax_acq_rel_half_global_cta_param_1];
-; SM70-NOFTZ-NEXT: ld.param.b64 %rd2, [fmax_acq_rel_half_global_cta_param_0];
-; SM70-NOFTZ-NEXT: fence.acq_rel.cta;
-; SM70-NOFTZ-NEXT: and.b64 %rd1, %rd2, -4;
-; SM70-NOFTZ-NEXT: cvt.u32.u64 %r4, %rd2;
-; SM70-NOFTZ-NEXT: and.b32 %r5, %r4, 3;
-; SM70-NOFTZ-NEXT: shl.b32 %r1, %r5, 3;
-; SM70-NOFTZ-NEXT: mov.b32 %r6, 65535;
-; SM70-NOFTZ-NEXT: shl.b32 %r7, %r6, %r1;
-; SM70-NOFTZ-NEXT: not.b32 %r2, %r7;
-; SM70-NOFTZ-NEXT: ld.relaxed.cta.global.b32 %r17, [%rd1];
-; SM70-NOFTZ-NEXT: cvt.f32.f16 %r10, %rs1;
-; SM70-NOFTZ-NEXT: $L__BB75_1: // %atomicrmw.start
-; SM70-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NOFTZ-NEXT: shr.u32 %r8, %r17, %r1;
-; SM70-NOFTZ-NEXT: cvt.u16.u32 %rs2, %r8;
-; SM70-NOFTZ-NEXT: cvt.f32.f16 %r9, %rs2;
-; SM70-NOFTZ-NEXT: max.f32 %r11, %r9, %r10;
-; SM70-NOFTZ-NEXT: cvt.rn.f16.f32 %rs3, %r11;
-; SM70-NOFTZ-NEXT: cvt.u32.u16 %r12, %rs3;
-; SM70-NOFTZ-NEXT: shl.b32 %r13, %r12, %r1;
-; SM70-NOFTZ-NEXT: and.b32 %r14, %r17, %r2;
-; SM70-NOFTZ-NEXT: or.b32 %r15, %r14, %r13;
-; SM70-NOFTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r17, %r15;
-; SM70-NOFTZ-NEXT: setp.ne.b32 %p1, %r3, %r17;
-; SM70-NOFTZ-NEXT: mov.b32 %r17, %r3;
-; SM70-NOFTZ-NEXT: @%p1 bra $L__BB75_1;
-; SM70-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end
-; SM70-NOFTZ-NEXT: shr.u32 %r16, %r3, %r1;
-; SM70-NOFTZ-NEXT: fence.acq_rel.cta;
-; SM70-NOFTZ-NEXT: st.param.b16 [func_retval0], %r16;
-; SM70-NOFTZ-NEXT: ret;
+; SM70-NOFTZ-DISALLOW-LABEL: fmax_acq_rel_half_global_cta(
+; SM70-NOFTZ-DISALLOW: {
+; SM70-NOFTZ-DISALLOW-NEXT: .reg .pred %p<2>;
+; SM70-NOFTZ-DISALLOW-NEXT: .reg .b16 %rs<4>;
+; SM70-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<18>;
+; SM70-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
+; SM70-NOFTZ-DISALLOW-EMPTY:
+; SM70-NOFTZ-DISALLOW-NEXT: // %bb.0:
+; SM70-NOFTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fmax_acq_rel_half_global_cta_param_1];
+; SM70-NOFTZ-DISALLOW-NEXT: ld.param.b64 %rd2, [fmax_acq_rel_half_global_cta_param_0];
+; SM70-NOFTZ-DISALLOW-NEXT: fence.acq_rel.cta;
+; SM70-NOFTZ-DISALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-NOFTZ-DISALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM70-NOFTZ-DISALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-NOFTZ-DISALLOW-NEXT: mov.b32 %r6, 65535;
+; SM70-NOFTZ-DISALLOW-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-NOFTZ-DISALLOW-NEXT: not.b32 %r2, %r7;
+; SM70-NOFTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b32 %r17, [%rd1];
+; SM70-NOFTZ-DISALLOW-NEXT: cvt.f32.f16 %r10, %rs1;
+; SM70-NOFTZ-DISALLOW-NEXT: $L__BB75_1: // %atomicrmw.start
+; SM70-NOFTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NOFTZ-DISALLOW-NEXT: shr.u32 %r8, %r17, %r1;
+; SM70-NOFTZ-DISALLOW-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM70-NOFTZ-DISALLOW-NEXT: cvt.f32.f16 %r9, %rs2;
+; SM70-NOFTZ-DISALLOW-NEXT: max.f32 %r11, %r9, %r10;
+; SM70-NOFTZ-DISALLOW-NEXT: cvt.rn.f16.f32 %rs3, %r11;
+; SM70-NOFTZ-DISALLOW-NEXT: cvt.u32.u16 %r12, %rs3;
+; SM70-NOFTZ-DISALLOW-NEXT: shl.b32 %r13, %r12, %r1;
+; SM70-NOFTZ-DISALLOW-NEXT: and.b32 %r14, %r17, %r2;
+; SM70-NOFTZ-DISALLOW-NEXT: or.b32 %r15, %r14, %r13;
+; SM70-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r17, %r15;
+; SM70-NOFTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r3, %r17;
+; SM70-NOFTZ-DISALLOW-NEXT: mov.b32 %r17, %r3;
+; SM70-NOFTZ-DISALLOW-NEXT: @%p1 bra $L__BB75_1;
+; SM70-NOFTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NOFTZ-DISALLOW-NEXT: shr.u32 %r16, %r3, %r1;
+; SM70-NOFTZ-DISALLOW-NEXT: fence.acq_rel.cta;
+; SM70-NOFTZ-DISALLOW-NEXT: st.param.b16 [func_retval0], %r16;
+; SM70-NOFTZ-DISALLOW-NEXT: ret;
;
-; SM70-FTZ-LABEL: fmax_acq_rel_half_global_cta(
-; SM70-FTZ: {
-; SM70-FTZ-NEXT: .reg .pred %p<2>;
-; SM70-FTZ-NEXT: .reg .b16 %rs<4>;
-; SM70-FTZ-NEXT: .reg .b32 %r<18>;
-; SM70-FTZ-NEXT: .reg .b64 %rd<3>;
-; SM70-FTZ-EMPTY:
-; SM70-FTZ-NEXT: // %bb.0:
-; SM70-FTZ-NEXT: ld.param.b16 %rs1, [fmax_acq_rel_half_global_cta_param_1];
-; SM70-FTZ-NEXT: ld.param.b64 %rd2, [fmax_acq_rel_half_global_cta_param_0];
-; SM70-FTZ-NEXT: fence.acq_rel.cta;
-; SM70-FTZ-NEXT: and.b64 %rd1, %rd2, -4;
-; SM70-FTZ-NEXT: cvt.u32.u64 %r4, %rd2;
-; SM70-FTZ-NEXT: and.b32 %r5, %r4, 3;
-; SM70-FTZ-NEXT: shl.b32 %r1, %r5, 3;
-; SM70-FTZ-NEXT: mov.b32 %r6, 65535;
-; SM70-FTZ-NEXT: shl.b32 %r7, %r6, %r1;
-; SM70-FTZ-NEXT: not.b32 %r2, %r7;
-; SM70-FTZ-NEXT: ld.relaxed.cta.global.b32 %r17, [%rd1];
-; SM70-FTZ-NEXT: cvt.ftz.f32.f16 %r10, %rs1;
-; SM70-FTZ-NEXT: $L__BB75_1: // %atomicrmw.start
-; SM70-FTZ-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-FTZ-NEXT: shr.u32 %r8, %r17, %r1;
-; SM70-FTZ-NEXT: cvt.u16.u32 %rs2, %r8;
-; SM70-FTZ-NEXT: cvt.ftz.f32.f16 %r9, %rs2;
-; SM70-FTZ-NEXT: max.ftz.f32 %r11, %r9, %r10;
-; SM70-FTZ-NEXT: cvt.rn.f16.f32 %rs3, %r11;
-; SM70-FTZ-NEXT: cvt.u32.u16 %r12, %rs3;
-; SM70-FTZ-NEXT: shl.b32 %r13, %r12, %r1;
-; SM70-FTZ-NEXT: and.b32 %r14, %r17, %r2;
-; SM70-FTZ-NEXT: or.b32 %r15, %r14, %r13;
-; SM70-FTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r17, %r15;
-; SM70-FTZ-NEXT: setp.ne.b32 %p1, %r3, %r17;
-; SM70-FTZ-NEXT: mov.b32 %r17, %r3;
-; SM70-FTZ-NEXT: @%p1 bra $L__BB75_1;
-; SM70-FTZ-NEXT: // %bb.2: // %atomicrmw.end
-; SM70-FTZ-NEXT: shr.u32 %r16, %r3, %r1;
-; SM70-FTZ-NEXT: fence.acq_rel.cta;
-; SM70-FTZ-NEXT: st.param.b16 [func_retval0], %r16;
-; SM70-FTZ-NEXT: ret;
+; SM70-NOFTZ-ALLOW-LABEL: fmax_acq_rel_half_global_cta(
+; SM70-NOFTZ-ALLOW: {
+; SM70-NOFTZ-ALLOW-NEXT: .reg .pred %p<2>;
+; SM70-NOFTZ-ALLOW-NEXT: .reg .b16 %rs<4>;
+; SM70-NOFTZ-ALLOW-NEXT: .reg .b32 %r<18>;
+; SM70-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
+; SM70-NOFTZ-ALLOW-EMPTY:
+; SM70-NOFTZ-ALLOW-NEXT: // %bb.0:
+; SM70-NOFTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fmax_acq_rel_half_global_cta_param_1];
+; SM70-NOFTZ-ALLOW-NEXT: ld.param.b64 %rd2, [fmax_acq_rel_half_global_cta_param_0];
+; SM70-NOFTZ-ALLOW-NEXT: fence.acq_rel.cta;
+; SM70-NOFTZ-ALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NOFTZ-ALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-NOFTZ-ALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM70-NOFTZ-ALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-NOFTZ-ALLOW-NEXT: mov.b32 %r6, 65535;
+; SM70-NOFTZ-ALLOW-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-NOFTZ-ALLOW-NEXT: not.b32 %r2, %r7;
+; SM70-NOFTZ-ALLOW-NEXT: ld.relaxed.cta.global.b32 %r17, [%rd1];
+; SM70-NOFTZ-ALLOW-NEXT: cvt.f32.f16 %r10, %rs1;
+; SM70-NOFTZ-ALLOW-NEXT: $L__BB75_1: // %atomicrmw.start
+; SM70-NOFTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NOFTZ-ALLOW-NEXT: shr.u32 %r8, %r17, %r1;
+; SM70-NOFTZ-ALLOW-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM70-NOFTZ-ALLOW-NEXT: cvt.f32.f16 %r9, %rs2;
+; SM70-NOFTZ-ALLOW-NEXT: max.f32 %r11, %r9, %r10;
+; SM70-NOFTZ-ALLOW-NEXT: cvt.rn.f16.f32 %rs3, %r11;
+; SM70-NOFTZ-ALLOW-NEXT: cvt.u32.u16 %r12, %rs3;
+; SM70-NOFTZ-ALLOW-NEXT: shl.b32 %r13, %r12, %r1;
+; SM70-NOFTZ-ALLOW-NEXT: and.b32 %r14, %r17, %r2;
+; SM70-NOFTZ-ALLOW-NEXT: or.b32 %r15, %r14, %r13;
+; SM70-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r17, %r15;
+; SM70-NOFTZ-ALLOW-NEXT: setp.ne.b32 %p1, %r3, %r17;
+; SM70-NOFTZ-ALLOW-NEXT: mov.b32 %r17, %r3;
+; SM70-NOFTZ-ALLOW-NEXT: @%p1 bra $L__BB75_1;
+; SM70-NOFTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NOFTZ-ALLOW-NEXT: shr.u32 %r16, %r3, %r1;
+; SM70-NOFTZ-ALLOW-NEXT: fence.acq_rel.cta;
+; SM70-NOFTZ-ALLOW-NEXT: st.param.b16 [func_retval0], %r16;
+; SM70-NOFTZ-ALLOW-NEXT: ret;
+;
+; SM70-FTZ-DISALLOW-LABEL: fmax_acq_rel_half_global_cta(
+; SM70-FTZ-DISALLOW: {
+; SM70-FTZ-DISALLOW-NEXT: .reg .pred %p<2>;
+; SM70-FTZ-DISALLOW-NEXT: .reg .b16 %rs<4>;
+; SM70-FTZ-DISALLOW-NEXT: .reg .b32 %r<18>;
+; SM70-FTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
+; SM70-FTZ-DISALLOW-EMPTY:
+; SM70-FTZ-DISALLOW-NEXT: // %bb.0:
+; SM70-FTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fmax_acq_rel_half_global_cta_param_1];
+; SM70-FTZ-DISALLOW-NEXT: ld.param.b64 %rd2, [fmax_acq_rel_half_global_cta_param_0];
+; SM70-FTZ-DISALLOW-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-DISALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-FTZ-DISALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-FTZ-DISALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM70-FTZ-DISALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-FTZ-DISALLOW-NEXT: mov.b32 %r6, 65535;
+; SM70-FTZ-DISALLOW-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-FTZ-DISALLOW-NEXT: not.b32 %r2, %r7;
+; SM70-FTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b32 %r17, [%rd1];
+; SM70-FTZ-DISALLOW-NEXT: cvt.ftz.f32.f16 %r10, %rs1;
+; SM70-FTZ-DISALLOW-NEXT: $L__BB75_1: // %atomicrmw.start
+; SM70-FTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-FTZ-DISALLOW-NEXT: shr.u32 %r8, %r17, %r1;
+; SM70-FTZ-DISALLOW-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM70-FTZ-DISALLOW-NEXT: cvt.ftz.f32.f16 %r9, %rs2;
+; SM70-FTZ-DISALLOW-NEXT: max.ftz.f32 %r11, %r9, %r10;
+; SM70-FTZ-DISALLOW-NEXT: cvt.rn.f16.f32 %rs3, %r11;
+; SM70-FTZ-DISALLOW-NEXT: cvt.u32.u16 %r12, %rs3;
+; SM70-FTZ-DISALLOW-NEXT: shl.b32 %r13, %r12, %r1;
+; SM70-FTZ-DISALLOW-NEXT: and.b32 %r14, %r17, %r2;
+; SM70-FTZ-DISALLOW-NEXT: or.b32 %r15, %r14, %r13;
+; SM70-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r17, %r15;
+; SM70-FTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r3, %r17;
+; SM70-FTZ-DISALLOW-NEXT: mov.b32 %r17, %r3;
+; SM70-FTZ-DISALLOW-NEXT: @%p1 bra $L__BB75_1;
+; SM70-FTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-FTZ-DISALLOW-NEXT: shr.u32 %r16, %r3, %r1;
+; SM70-FTZ-DISALLOW-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-DISALLOW-NEXT: st.param.b16 [func_retval0], %r16;
+; SM70-FTZ-DISALLOW-NEXT: ret;
+;
+; SM70-FTZ-ALLOW-LABEL: fmax_acq_rel_half_global_cta(
+; SM70-FTZ-ALLOW: {
+; SM70-FTZ-ALLOW-NEXT: .reg .pred %p<2>;
+; SM70-FTZ-ALLOW-NEXT: .reg .b16 %rs<4>;
+; SM70-FTZ-ALLOW-NEXT: .reg .b32 %r<18>;
+; SM70-FTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
+; SM70-FTZ-ALLOW-EMPTY:
+; SM70-FTZ-ALLOW-NEXT: // %bb.0:
+; SM70-FTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fmax_acq_rel_half_global_cta_param_1];
+; SM70-FTZ-ALLOW-NEXT: ld.param.b64 %rd2, [fmax_acq_rel_half_global_cta_param_0];
+; SM70-FTZ-ALLOW-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-ALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-FTZ-ALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-FTZ-ALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM70-FTZ-ALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-FTZ-ALLOW-NEXT: mov.b32 %r6, 65535;
+; SM70-FTZ-ALLOW-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-FTZ-ALLOW-NEXT: not.b32 %r2, %r7;
+; SM70-FTZ-ALLOW-NEXT: ld.relaxed.cta.global.b32 %r17, [%rd1];
+; SM70-FTZ-ALLOW-NEXT: cvt.ftz.f32.f16 %r10, %rs1;
+; SM70-FTZ-ALLOW-NEXT: $L__BB75_1: // %atomicrmw.start
+; SM70-FTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-FTZ-ALLOW-NEXT: shr.u32 %r8, %r17, %r1;
+; SM70-FTZ-ALLOW-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM70-FTZ-ALLOW-NEXT: cvt.ftz.f32.f16 %r9, %rs2;
+; SM70-FTZ-ALLOW-NEXT: max.ftz.f32 %r11, %r9, %r10;
+; SM70-FTZ-ALLOW-NEXT: cvt.rn.f16.f32 %rs3, %r11;
+; SM70-FTZ-ALLOW-NEXT: cvt.u32.u16 %r12, %rs3;
+; SM70-FTZ-ALLOW-NEXT: shl.b32 %r13, %r12, %r1;
+; SM70-FTZ-ALLOW-NEXT: and.b32 %r14, %r17, %r2;
+; SM70-FTZ-ALLOW-NEXT: or.b32 %r15, %r14, %r13;
+; SM70-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r17, %r15;
+; SM70-FTZ-ALLOW-NEXT: setp.ne.b32 %p1, %r3, %r17;
+; SM70-FTZ-ALLOW-NEXT: mov.b32 %r17, %r3;
+; SM70-FTZ-ALLOW-NEXT: @%p1 bra $L__BB75_1;
+; SM70-FTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-FTZ-ALLOW-NEXT: shr.u32 %r16, %r3, %r1;
+; SM70-FTZ-ALLOW-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-ALLOW-NEXT: st.param.b16 [func_retval0], %r16;
+; SM70-FTZ-ALLOW-NEXT: ret;
%retval = atomicrmw fmax ptr addrspace(1) %addr, half %val syncscope("block") acq_rel
ret half %retval
}
define half @fminimum_acq_rel_half_global_cta(ptr addrspace(1) %addr, half %val) {
-; SM70-NOFTZ-LABEL: fminimum_acq_rel_half_global_cta(
-; SM70-NOFTZ: {
-; SM70-NOFTZ-NEXT: .reg .pred %p<6>;
-; SM70-NOFTZ-NEXT: .reg .b16 %rs<7>;
-; SM70-NOFTZ-NEXT: .reg .b32 %r<15>;
-; SM70-NOFTZ-NEXT: .reg .b64 %rd<3>;
-; SM70-NOFTZ-EMPTY:
-; SM70-NOFTZ-NEXT: // %bb.0:
-; SM70-NOFTZ-NEXT: ld.param.b16 %rs1, [fminimum_acq_rel_half_global_cta_param_1];
-; SM70-NOFTZ-NEXT: ld.param.b64 %rd2, [fminimum_acq_rel_half_global_cta_param_0];
-; SM70-NOFTZ-NEXT: fence.acq_rel.cta;
-; SM70-NOFTZ-NEXT: and.b64 %rd1, %rd2, -4;
-; SM70-NOFTZ-NEXT: cvt.u32.u64 %r4, %rd2;
-; SM70-NOFTZ-NEXT: and.b32 %r5, %r4, 3;
-; SM70-NOFTZ-NEXT: shl.b32 %r1, %r5, 3;
-; SM70-NOFTZ-NEXT: mov.b32 %r6, 65535;
-; SM70-NOFTZ-NEXT: shl.b32 %r7, %r6, %r1;
-; SM70-NOFTZ-NEXT: not.b32 %r2, %r7;
-; SM70-NOFTZ-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
-; SM70-NOFTZ-NEXT: $L__BB76_1: // %atomicrmw.start
-; SM70-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NOFTZ-NEXT: shr.u32 %r8, %r14, %r1;
-; SM70-NOFTZ-NEXT: cvt.u16.u32 %rs2, %r8;
-; SM70-NOFTZ-NEXT: setp.eq.b16 %p1, %rs2, -32768;
-; SM70-NOFTZ-NEXT: selp.b16 %rs3, %rs2, %rs1, %p1;
-; SM70-NOFTZ-NEXT: setp.lt.f16 %p2, %rs2, %rs1;
-; SM70-NOFTZ-NEXT: selp.b16 %rs4, %rs2, %rs1, %p2;
-; SM70-NOFTZ-NEXT: setp.nan.f16 %p3, %rs2, %rs1;
-; SM70-NOFTZ-NEXT: selp.b16 %rs5, 0x7E00, %rs4, %p3;
-; SM70-NOFTZ-NEXT: setp.eq.f16 %p4, %rs2, %rs1;
-; SM70-NOFTZ-NEXT: selp.b16 %rs6, %rs3, %rs5, %p4;
-; SM70-NOFTZ-NEXT: cvt.u32.u16 %r9, %rs6;
-; SM70-NOFTZ-NEXT: shl.b32 %r10, %r9, %r1;
-; SM70-NOFTZ-NEXT: and.b32 %r11, %r14, %r2;
-; SM70-NOFTZ-NEXT: or.b32 %r12, %r11, %r10;
-; SM70-NOFTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
-; SM70-NOFTZ-NEXT: setp.ne.b32 %p5, %r3, %r14;
-; SM70-NOFTZ-NEXT: mov.b32 %r14, %r3;
-; SM70-NOFTZ-NEXT: @%p5 bra $L__BB76_1;
-; SM70-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end
-; SM70-NOFTZ-NEXT: shr.u32 %r13, %r3, %r1;
-; SM70-NOFTZ-NEXT: fence.acq_rel.cta;
-; SM70-NOFTZ-NEXT: st.param.b16 [func_retval0], %r13;
-; SM70-NOFTZ-NEXT: ret;
+; SM70-NOFTZ-DISALLOW-LABEL: fminimum_acq_rel_half_global_cta(
+; SM70-NOFTZ-DISALLOW: {
+; SM70-NOFTZ-DISALLOW-NEXT: .reg .pred %p<6>;
+; SM70-NOFTZ-DISALLOW-NEXT: .reg .b16 %rs<7>;
+; SM70-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<15>;
+; SM70-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
+; SM70-NOFTZ-DISALLOW-EMPTY:
+; SM70-NOFTZ-DISALLOW-NEXT: // %bb.0:
+; SM70-NOFTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fminimum_acq_rel_half_global_cta_param_1];
+; SM70-NOFTZ-DISALLOW-NEXT: ld.param.b64 %rd2, [fminimum_acq_rel_half_global_cta_param_0];
+; SM70-NOFTZ-DISALLOW-NEXT: fence.acq_rel.cta;
+; SM70-NOFTZ-DISALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-NOFTZ-DISALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM70-NOFTZ-DISALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-NOFTZ-DISALLOW-NEXT: mov.b32 %r6, 65535;
+; SM70-NOFTZ-DISALLOW-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-NOFTZ-DISALLOW-NEXT: not.b32 %r2, %r7;
+; SM70-NOFTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM70-NOFTZ-DISALLOW-NEXT: $L__BB76_1: // %atomicrmw.start
+; SM70-NOFTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NOFTZ-DISALLOW-NEXT: shr.u32 %r8, %r14, %r1;
+; SM70-NOFTZ-DISALLOW-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM70-NOFTZ-DISALLOW-NEXT: setp.eq.b16 %p1, %rs2, -32768;
+; SM70-NOFTZ-DISALLOW-NEXT: selp.b16 %rs3, %rs2, %rs1, %p1;
+; SM70-NOFTZ-DISALLOW-NEXT: setp.lt.f16 %p2, %rs2, %rs1;
+; SM70-NOFTZ-DISALLOW-NEXT: selp.b16 %rs4, %rs2, %rs1, %p2;
+; SM70-NOFTZ-DISALLOW-NEXT: setp.nan.f16 %p3, %rs2, %rs1;
+; SM70-NOFTZ-DISALLOW-NEXT: selp.b16 %rs5, 0x7E00, %rs4, %p3;
+; SM70-NOFTZ-DISALLOW-NEXT: setp.eq.f16 %p4, %rs2, %rs1;
+; SM70-NOFTZ-DISALLOW-NEXT: selp.b16 %rs6, %rs3, %rs5, %p4;
+; SM70-NOFTZ-DISALLOW-NEXT: cvt.u32.u16 %r9, %rs6;
+; SM70-NOFTZ-DISALLOW-NEXT: shl.b32 %r10, %r9, %r1;
+; SM70-NOFTZ-DISALLOW-NEXT: and.b32 %r11, %r14, %r2;
+; SM70-NOFTZ-DISALLOW-NEXT: or.b32 %r12, %r11, %r10;
+; SM70-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM70-NOFTZ-DISALLOW-NEXT: setp.ne.b32 %p5, %r3, %r14;
+; SM70-NOFTZ-DISALLOW-NEXT: mov.b32 %r14, %r3;
+; SM70-NOFTZ-DISALLOW-NEXT: @%p5 bra $L__BB76_1;
+; SM70-NOFTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NOFTZ-DISALLOW-NEXT: shr.u32 %r13, %r3, %r1;
+; SM70-NOFTZ-DISALLOW-NEXT: fence.acq_rel.cta;
+; SM70-NOFTZ-DISALLOW-NEXT: st.param.b16 [func_retval0], %r13;
+; SM70-NOFTZ-DISALLOW-NEXT: ret;
+;
+; SM70-NOFTZ-ALLOW-LABEL: fminimum_acq_rel_half_global_cta(
+; SM70-NOFTZ-ALLOW: {
+; SM70-NOFTZ-ALLOW-NEXT: .reg .pred %p<6>;
+; SM70-NOFTZ-ALLOW-NEXT: .reg .b16 %rs<7>;
+; SM70-NOFTZ-ALLOW-NEXT: .reg .b32 %r<15>;
+; SM70-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
+; SM70-NOFTZ-ALLOW-EMPTY:
+; SM70-NOFTZ-ALLOW-NEXT: // %bb.0:
+; SM70-NOFTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fminimum_acq_rel_half_global_cta_param_1];
+; SM70-NOFTZ-ALLOW-NEXT: ld.param.b64 %rd2, [fminimum_acq_rel_half_global_cta_param_0];
+; SM70-NOFTZ-ALLOW-NEXT: fence.acq_rel.cta;
+; SM70-NOFTZ-ALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NOFTZ-ALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-NOFTZ-ALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM70-NOFTZ-ALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-NOFTZ-ALLOW-NEXT: mov.b32 %r6, 65535;
+; SM70-NOFTZ-ALLOW-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-NOFTZ-ALLOW-NEXT: not.b32 %r2, %r7;
+; SM70-NOFTZ-ALLOW-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM70-NOFTZ-ALLOW-NEXT: $L__BB76_1: // %atomicrmw.start
+; SM70-NOFTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NOFTZ-ALLOW-NEXT: shr.u32 %r8, %r14, %r1;
+; SM70-NOFTZ-ALLOW-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM70-NOFTZ-ALLOW-NEXT: setp.eq.b16 %p1, %rs2, -32768;
+; SM70-NOFTZ-ALLOW-NEXT: selp.b16 %rs3, %rs2, %rs1, %p1;
+; SM70-NOFTZ-ALLOW-NEXT: setp.lt.f16 %p2, %rs2, %rs1;
+; SM70-NOFTZ-ALLOW-NEXT: selp.b16 %rs4, %rs2, %rs1, %p2;
+; SM70-NOFTZ-ALLOW-NEXT: setp.nan.f16 %p3, %rs2, %rs1;
+; SM70-NOFTZ-ALLOW-NEXT: selp.b16 %rs5, 0x7E00, %rs4, %p3;
+; SM70-NOFTZ-ALLOW-NEXT: setp.eq.f16 %p4, %rs2, %rs1;
+; SM70-NOFTZ-ALLOW-NEXT: selp.b16 %rs6, %rs3, %rs5, %p4;
+; SM70-NOFTZ-ALLOW-NEXT: cvt.u32.u16 %r9, %rs6;
+; SM70-NOFTZ-ALLOW-NEXT: shl.b32 %r10, %r9, %r1;
+; SM70-NOFTZ-ALLOW-NEXT: and.b32 %r11, %r14, %r2;
+; SM70-NOFTZ-ALLOW-NEXT: or.b32 %r12, %r11, %r10;
+; SM70-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM70-NOFTZ-ALLOW-NEXT: setp.ne.b32 %p5, %r3, %r14;
+; SM70-NOFTZ-ALLOW-NEXT: mov.b32 %r14, %r3;
+; SM70-NOFTZ-ALLOW-NEXT: @%p5 bra $L__BB76_1;
+; SM70-NOFTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NOFTZ-ALLOW-NEXT: shr.u32 %r13, %r3, %r1;
+; SM70-NOFTZ-ALLOW-NEXT: fence.acq_rel.cta;
+; SM70-NOFTZ-ALLOW-NEXT: st.param.b16 [func_retval0], %r13;
+; SM70-NOFTZ-ALLOW-NEXT: ret;
;
-; SM70-FTZ-LABEL: fminimum_acq_rel_half_global_cta(
-; SM70-FTZ: {
-; SM70-FTZ-NEXT: .reg .pred %p<6>;
-; SM70-FTZ-NEXT: .reg .b16 %rs<7>;
-; SM70-FTZ-NEXT: .reg .b32 %r<15>;
-; SM70-FTZ-NEXT: .reg .b64 %rd<3>;
-; SM70-FTZ-EMPTY:
-; SM70-FTZ-NEXT: // %bb.0:
-; SM70-FTZ-NEXT: ld.param.b16 %rs1, [fminimum_acq_rel_half_global_cta_param_1];
-; SM70-FTZ-NEXT: ld.param.b64 %rd2, [fminimum_acq_rel_half_global_cta_param_0];
-; SM70-FTZ-NEXT: fence.acq_rel.cta;
-; SM70-FTZ-NEXT: and.b64 %rd1, %rd2, -4;
-; SM70-FTZ-NEXT: cvt.u32.u64 %r4, %rd2;
-; SM70-FTZ-NEXT: and.b32 %r5, %r4, 3;
-; SM70-FTZ-NEXT: shl.b32 %r1, %r5, 3;
-; SM70-FTZ-NEXT: mov.b32 %r6, 65535;
-; SM70-FTZ-NEXT: shl.b32 %r7, %r6, %r1;
-; SM70-FTZ-NEXT: not.b32 %r2, %r7;
-; SM70-FTZ-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
-; SM70-FTZ-NEXT: $L__BB76_1: // %atomicrmw.start
-; SM70-FTZ-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-FTZ-NEXT: shr.u32 %r8, %r14, %r1;
-; SM70-FTZ-NEXT: cvt.u16.u32 %rs2, %r8;
-; SM70-FTZ-NEXT: setp.eq.b16 %p1, %rs2, -32768;
-; SM70-FTZ-NEXT: selp.b16 %rs3, %rs2, %rs1, %p1;
-; SM70-FTZ-NEXT: setp.lt.ftz.f16 %p2, %rs2, %rs1;
-; SM70-FTZ-NEXT: selp.b16 %rs4, %rs2, %rs1, %p2;
-; SM70-FTZ-NEXT: setp.nan.ftz.f16 %p3, %rs2, %rs1;
-; SM70-FTZ-NEXT: selp.b16 %rs5, 0x7E00, %rs4, %p3;
-; SM70-FTZ-NEXT: setp.eq.ftz.f16 %p4, %rs2, %rs1;
-; SM70-FTZ-NEXT: selp.b16 %rs6, %rs3, %rs5, %p4;
-; SM70-FTZ-NEXT: cvt.u32.u16 %r9, %rs6;
-; SM70-FTZ-NEXT: shl.b32 %r10, %r9, %r1;
-; SM70-FTZ-NEXT: and.b32 %r11, %r14, %r2;
-; SM70-FTZ-NEXT: or.b32 %r12, %r11, %r10;
-; SM70-FTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
-; SM70-FTZ-NEXT: setp.ne.b32 %p5, %r3, %r14;
-; SM70-FTZ-NEXT: mov.b32 %r14, %r3;
-; SM70-FTZ-NEXT: @%p5 bra $L__BB76_1;
-; SM70-FTZ-NEXT: // %bb.2: // %atomicrmw.end
-; SM70-FTZ-NEXT: shr.u32 %r13, %r3, %r1;
-; SM70-FTZ-NEXT: fence.acq_rel.cta;
-; SM70-FTZ-NEXT: st.param.b16 [func_retval0], %r13;
-; SM70-FTZ-NEXT: ret;
+; SM70-FTZ-DISALLOW-LABEL: fminimum_acq_rel_half_global_cta(
+; SM70-FTZ-DISALLOW: {
+; SM70-FTZ-DISALLOW-NEXT: .reg .pred %p<6>;
+; SM70-FTZ-DISALLOW-NEXT: .reg .b16 %rs<7>;
+; SM70-FTZ-DISALLOW-NEXT: .reg .b32 %r<15>;
+; SM70-FTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
+; SM70-FTZ-DISALLOW-EMPTY:
+; SM70-FTZ-DISALLOW-NEXT: // %bb.0:
+; SM70-FTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fminimum_acq_rel_half_global_cta_param_1];
+; SM70-FTZ-DISALLOW-NEXT: ld.param.b64 %rd2, [fminimum_acq_rel_half_global_cta_param_0];
+; SM70-FTZ-DISALLOW-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-DISALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-FTZ-DISALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-FTZ-DISALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM70-FTZ-DISALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-FTZ-DISALLOW-NEXT: mov.b32 %r6, 65535;
+; SM70-FTZ-DISALLOW-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-FTZ-DISALLOW-NEXT: not.b32 %r2, %r7;
+; SM70-FTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM70-FTZ-DISALLOW-NEXT: $L__BB76_1: // %atomicrmw.start
+; SM70-FTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-FTZ-DISALLOW-NEXT: shr.u32 %r8, %r14, %r1;
+; SM70-FTZ-DISALLOW-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM70-FTZ-DISALLOW-NEXT: setp.eq.b16 %p1, %rs2, -32768;
+; SM70-FTZ-DISALLOW-NEXT: selp.b16 %rs3, %rs2, %rs1, %p1;
+; SM70-FTZ-DISALLOW-NEXT: setp.lt.ftz.f16 %p2, %rs2, %rs1;
+; SM70-FTZ-DISALLOW-NEXT: selp.b16 %rs4, %rs2, %rs1, %p2;
+; SM70-FTZ-DISALLOW-NEXT: setp.nan.ftz.f16 %p3, %rs2, %rs1;
+; SM70-FTZ-DISALLOW-NEXT: selp.b16 %rs5, 0x7E00, %rs4, %p3;
+; SM70-FTZ-DISALLOW-NEXT: setp.eq.ftz.f16 %p4, %rs2, %rs1;
+; SM70-FTZ-DISALLOW-NEXT: selp.b16 %rs6, %rs3, %rs5, %p4;
+; SM70-FTZ-DISALLOW-NEXT: cvt.u32.u16 %r9, %rs6;
+; SM70-FTZ-DISALLOW-NEXT: shl.b32 %r10, %r9, %r1;
+; SM70-FTZ-DISALLOW-NEXT: and.b32 %r11, %r14, %r2;
+; SM70-FTZ-DISALLOW-NEXT: or.b32 %r12, %r11, %r10;
+; SM70-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM70-FTZ-DISALLOW-NEXT: setp.ne.b32 %p5, %r3, %r14;
+; SM70-FTZ-DISALLOW-NEXT: mov.b32 %r14, %r3;
+; SM70-FTZ-DISALLOW-NEXT: @%p5 bra $L__BB76_1;
+; SM70-FTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-FTZ-DISALLOW-NEXT: shr.u32 %r13, %r3, %r1;
+; SM70-FTZ-DISALLOW-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-DISALLOW-NEXT: st.param.b16 [func_retval0], %r13;
+; SM70-FTZ-DISALLOW-NEXT: ret;
+;
+; SM70-FTZ-ALLOW-LABEL: fminimum_acq_rel_half_global_cta(
+; SM70-FTZ-ALLOW: {
+; SM70-FTZ-ALLOW-NEXT: .reg .pred %p<6>;
+; SM70-FTZ-ALLOW-NEXT: .reg .b16 %rs<7>;
+; SM70-FTZ-ALLOW-NEXT: .reg .b32 %r<15>;
+; SM70-FTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
+; SM70-FTZ-ALLOW-EMPTY:
+; SM70-FTZ-ALLOW-NEXT: // %bb.0:
+; SM70-FTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fminimum_acq_rel_half_global_cta_param_1];
+; SM70-FTZ-ALLOW-NEXT: ld.param.b64 %rd2, [fminimum_acq_rel_half_global_cta_param_0];
+; SM70-FTZ-ALLOW-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-ALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-FTZ-ALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-FTZ-ALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM70-FTZ-ALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-FTZ-ALLOW-NEXT: mov.b32 %r6, 65535;
+; SM70-FTZ-ALLOW-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-FTZ-ALLOW-NEXT: not.b32 %r2, %r7;
+; SM70-FTZ-ALLOW-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM70-FTZ-ALLOW-NEXT: $L__BB76_1: // %atomicrmw.start
+; SM70-FTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-FTZ-ALLOW-NEXT: shr.u32 %r8, %r14, %r1;
+; SM70-FTZ-ALLOW-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM70-FTZ-ALLOW-NEXT: setp.eq.b16 %p1, %rs2, -32768;
+; SM70-FTZ-ALLOW-NEXT: selp.b16 %rs3, %rs2, %rs1, %p1;
+; SM70-FTZ-ALLOW-NEXT: setp.lt.ftz.f16 %p2, %rs2, %rs1;
+; SM70-FTZ-ALLOW-NEXT: selp.b16 %rs4, %rs2, %rs1, %p2;
+; SM70-FTZ-ALLOW-NEXT: setp.nan.ftz.f16 %p3, %rs2, %rs1;
+; SM70-FTZ-ALLOW-NEXT: selp.b16 %rs5, 0x7E00, %rs4, %p3;
+; SM70-FTZ-ALLOW-NEXT: setp.eq.ftz.f16 %p4, %rs2, %rs1;
+; SM70-FTZ-ALLOW-NEXT: selp.b16 %rs6, %rs3, %rs5, %p4;
+; SM70-FTZ-ALLOW-NEXT: cvt.u32.u16 %r9, %rs6;
+; SM70-FTZ-ALLOW-NEXT: shl.b32 %r10, %r9, %r1;
+; SM70-FTZ-ALLOW-NEXT: and.b32 %r11, %r14, %r2;
+; SM70-FTZ-ALLOW-NEXT: or.b32 %r12, %r11, %r10;
+; SM70-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM70-FTZ-ALLOW-NEXT: setp.ne.b32 %p5, %r3, %r14;
+; SM70-FTZ-ALLOW-NEXT: mov.b32 %r14, %r3;
+; SM70-FTZ-ALLOW-NEXT: @%p5 bra $L__BB76_1;
+; SM70-FTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-FTZ-ALLOW-NEXT: shr.u32 %r13, %r3, %r1;
+; SM70-FTZ-ALLOW-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-ALLOW-NEXT: st.param.b16 [func_retval0], %r13;
+; SM70-FTZ-ALLOW-NEXT: ret;
%retval = atomicrmw fminimum ptr addrspace(1) %addr, half %val syncscope("block") acq_rel
ret half %retval
}
define half @fmaximum_acq_rel_half_global_cta(ptr addrspace(1) %addr, half %val) {
-; SM70-NOFTZ-LABEL: fmaximum_acq_rel_half_global_cta(
-; SM70-NOFTZ: {
-; SM70-NOFTZ-NEXT: .reg .pred %p<6>;
-; SM70-NOFTZ-NEXT: .reg .b16 %rs<7>;
-; SM70-NOFTZ-NEXT: .reg .b32 %r<15>;
-; SM70-NOFTZ-NEXT: .reg .b64 %rd<3>;
-; SM70-NOFTZ-EMPTY:
-; SM70-NOFTZ-NEXT: // %bb.0:
-; SM70-NOFTZ-NEXT: ld.param.b16 %rs1, [fmaximum_acq_rel_half_global_cta_param_1];
-; SM70-NOFTZ-NEXT: ld.param.b64 %rd2, [fmaximum_acq_rel_half_global_cta_param_0];
-; SM70-NOFTZ-NEXT: fence.acq_rel.cta;
-; SM70-NOFTZ-NEXT: and.b64 %rd1, %rd2, -4;
-; SM70-NOFTZ-NEXT: cvt.u32.u64 %r4, %rd2;
-; SM70-NOFTZ-NEXT: and.b32 %r5, %r4, 3;
-; SM70-NOFTZ-NEXT: shl.b32 %r1, %r5, 3;
-; SM70-NOFTZ-NEXT: mov.b32 %r6, 65535;
-; SM70-NOFTZ-NEXT: shl.b32 %r7, %r6, %r1;
-; SM70-NOFTZ-NEXT: not.b32 %r2, %r7;
-; SM70-NOFTZ-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
-; SM70-NOFTZ-NEXT: $L__BB77_1: // %atomicrmw.start
-; SM70-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NOFTZ-NEXT: shr.u32 %r8, %r14, %r1;
-; SM70-NOFTZ-NEXT: cvt.u16.u32 %rs2, %r8;
-; SM70-NOFTZ-NEXT: setp.eq.b16 %p1, %rs2, 0;
-; SM70-NOFTZ-NEXT: selp.b16 %rs3, %rs2, %rs1, %p1;
-; SM70-NOFTZ-NEXT: setp.gt.f16 %p2, %rs2, %rs1;
-; SM70-NOFTZ-NEXT: selp.b16 %rs4, %rs2, %rs1, %p2;
-; SM70-NOFTZ-NEXT: setp.nan.f16 %p3, %rs2, %rs1;
-; SM70-NOFTZ-NEXT: selp.b16 %rs5, 0x7E00, %rs4, %p3;
-; SM70-NOFTZ-NEXT: setp.eq.f16 %p4, %rs2, %rs1;
-; SM70-NOFTZ-NEXT: selp.b16 %rs6, %rs3, %rs5, %p4;
-; SM70-NOFTZ-NEXT: cvt.u32.u16 %r9, %rs6;
-; SM70-NOFTZ-NEXT: shl.b32 %r10, %r9, %r1;
-; SM70-NOFTZ-NEXT: and.b32 %r11, %r14, %r2;
-; SM70-NOFTZ-NEXT: or.b32 %r12, %r11, %r10;
-; SM70-NOFTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
-; SM70-NOFTZ-NEXT: setp.ne.b32 %p5, %r3, %r14;
-; SM70-NOFTZ-NEXT: mov.b32 %r14, %r3;
-; SM70-NOFTZ-NEXT: @%p5 bra $L__BB77_1;
-; SM70-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end
-; SM70-NOFTZ-NEXT: shr.u32 %r13, %r3, %r1;
-; SM70-NOFTZ-NEXT: fence.acq_rel.cta;
-; SM70-NOFTZ-NEXT: st.param.b16 [func_retval0], %r13;
-; SM70-NOFTZ-NEXT: ret;
+; SM70-NOFTZ-DISALLOW-LABEL: fmaximum_acq_rel_half_global_cta(
+; SM70-NOFTZ-DISALLOW: {
+; SM70-NOFTZ-DISALLOW-NEXT: .reg .pred %p<6>;
+; SM70-NOFTZ-DISALLOW-NEXT: .reg .b16 %rs<7>;
+; SM70-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<15>;
+; SM70-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
+; SM70-NOFTZ-DISALLOW-EMPTY:
+; SM70-NOFTZ-DISALLOW-NEXT: // %bb.0:
+; SM70-NOFTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fmaximum_acq_rel_half_global_cta_param_1];
+; SM70-NOFTZ-DISALLOW-NEXT: ld.param.b64 %rd2, [fmaximum_acq_rel_half_global_cta_param_0];
+; SM70-NOFTZ-DISALLOW-NEXT: fence.acq_rel.cta;
+; SM70-NOFTZ-DISALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-NOFTZ-DISALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM70-NOFTZ-DISALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-NOFTZ-DISALLOW-NEXT: mov.b32 %r6, 65535;
+; SM70-NOFTZ-DISALLOW-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-NOFTZ-DISALLOW-NEXT: not.b32 %r2, %r7;
+; SM70-NOFTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM70-NOFTZ-DISALLOW-NEXT: $L__BB77_1: // %atomicrmw.start
+; SM70-NOFTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NOFTZ-DISALLOW-NEXT: shr.u32 %r8, %r14, %r1;
+; SM70-NOFTZ-DISALLOW-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM70-NOFTZ-DISALLOW-NEXT: setp.eq.b16 %p1, %rs2, 0;
+; SM70-NOFTZ-DISALLOW-NEXT: selp.b16 %rs3, %rs2, %rs1, %p1;
+; SM70-NOFTZ-DISALLOW-NEXT: setp.gt.f16 %p2, %rs2, %rs1;
+; SM70-NOFTZ-DISALLOW-NEXT: selp.b16 %rs4, %rs2, %rs1, %p2;
+; SM70-NOFTZ-DISALLOW-NEXT: setp.nan.f16 %p3, %rs2, %rs1;
+; SM70-NOFTZ-DISALLOW-NEXT: selp.b16 %rs5, 0x7E00, %rs4, %p3;
+; SM70-NOFTZ-DISALLOW-NEXT: setp.eq.f16 %p4, %rs2, %rs1;
+; SM70-NOFTZ-DISALLOW-NEXT: selp.b16 %rs6, %rs3, %rs5, %p4;
+; SM70-NOFTZ-DISALLOW-NEXT: cvt.u32.u16 %r9, %rs6;
+; SM70-NOFTZ-DISALLOW-NEXT: shl.b32 %r10, %r9, %r1;
+; SM70-NOFTZ-DISALLOW-NEXT: and.b32 %r11, %r14, %r2;
+; SM70-NOFTZ-DISALLOW-NEXT: or.b32 %r12, %r11, %r10;
+; SM70-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM70-NOFTZ-DISALLOW-NEXT: setp.ne.b32 %p5, %r3, %r14;
+; SM70-NOFTZ-DISALLOW-NEXT: mov.b32 %r14, %r3;
+; SM70-NOFTZ-DISALLOW-NEXT: @%p5 bra $L__BB77_1;
+; SM70-NOFTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NOFTZ-DISALLOW-NEXT: shr.u32 %r13, %r3, %r1;
+; SM70-NOFTZ-DISALLOW-NEXT: fence.acq_rel.cta;
+; SM70-NOFTZ-DISALLOW-NEXT: st.param.b16 [func_retval0], %r13;
+; SM70-NOFTZ-DISALLOW-NEXT: ret;
+;
+; SM70-NOFTZ-ALLOW-LABEL: fmaximum_acq_rel_half_global_cta(
+; SM70-NOFTZ-ALLOW: {
+; SM70-NOFTZ-ALLOW-NEXT: .reg .pred %p<6>;
+; SM70-NOFTZ-ALLOW-NEXT: .reg .b16 %rs<7>;
+; SM70-NOFTZ-ALLOW-NEXT: .reg .b32 %r<15>;
+; SM70-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
+; SM70-NOFTZ-ALLOW-EMPTY:
+; SM70-NOFTZ-ALLOW-NEXT: // %bb.0:
+; SM70-NOFTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fmaximum_acq_rel_half_global_cta_param_1];
+; SM70-NOFTZ-ALLOW-NEXT: ld.param.b64 %rd2, [fmaximum_acq_rel_half_global_cta_param_0];
+; SM70-NOFTZ-ALLOW-NEXT: fence.acq_rel.cta;
+; SM70-NOFTZ-ALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NOFTZ-ALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-NOFTZ-ALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM70-NOFTZ-ALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-NOFTZ-ALLOW-NEXT: mov.b32 %r6, 65535;
+; SM70-NOFTZ-ALLOW-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-NOFTZ-ALLOW-NEXT: not.b32 %r2, %r7;
+; SM70-NOFTZ-ALLOW-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM70-NOFTZ-ALLOW-NEXT: $L__BB77_1: // %atomicrmw.start
+; SM70-NOFTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NOFTZ-ALLOW-NEXT: shr.u32 %r8, %r14, %r1;
+; SM70-NOFTZ-ALLOW-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM70-NOFTZ-ALLOW-NEXT: setp.eq.b16 %p1, %rs2, 0;
+; SM70-NOFTZ-ALLOW-NEXT: selp.b16 %rs3, %rs2, %rs1, %p1;
+; SM70-NOFTZ-ALLOW-NEXT: setp.gt.f16 %p2, %rs2, %rs1;
+; SM70-NOFTZ-ALLOW-NEXT: selp.b16 %rs4, %rs2, %rs1, %p2;
+; SM70-NOFTZ-ALLOW-NEXT: setp.nan.f16 %p3, %rs2, %rs1;
+; SM70-NOFTZ-ALLOW-NEXT: selp.b16 %rs5, 0x7E00, %rs4, %p3;
+; SM70-NOFTZ-ALLOW-NEXT: setp.eq.f16 %p4, %rs2, %rs1;
+; SM70-NOFTZ-ALLOW-NEXT: selp.b16 %rs6, %rs3, %rs5, %p4;
+; SM70-NOFTZ-ALLOW-NEXT: cvt.u32.u16 %r9, %rs6;
+; SM70-NOFTZ-ALLOW-NEXT: shl.b32 %r10, %r9, %r1;
+; SM70-NOFTZ-ALLOW-NEXT: and.b32 %r11, %r14, %r2;
+; SM70-NOFTZ-ALLOW-NEXT: or.b32 %r12, %r11, %r10;
+; SM70-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM70-NOFTZ-ALLOW-NEXT: setp.ne.b32 %p5, %r3, %r14;
+; SM70-NOFTZ-ALLOW-NEXT: mov.b32 %r14, %r3;
+; SM70-NOFTZ-ALLOW-NEXT: @%p5 bra $L__BB77_1;
+; SM70-NOFTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NOFTZ-ALLOW-NEXT: shr.u32 %r13, %r3, %r1;
+; SM70-NOFTZ-ALLOW-NEXT: fence.acq_rel.cta;
+; SM70-NOFTZ-ALLOW-NEXT: st.param.b16 [func_retval0], %r13;
+; SM70-NOFTZ-ALLOW-NEXT: ret;
;
-; SM70-FTZ-LABEL: fmaximum_acq_rel_half_global_cta(
-; SM70-FTZ: {
-; SM70-FTZ-NEXT: .reg .pred %p<6>;
-; SM70-FTZ-NEXT: .reg .b16 %rs<7>;
-; SM70-FTZ-NEXT: .reg .b32 %r<15>;
-; SM70-FTZ-NEXT: .reg .b64 %rd<3>;
-; SM70-FTZ-EMPTY:
-; SM70-FTZ-NEXT: // %bb.0:
-; SM70-FTZ-NEXT: ld.param.b16 %rs1, [fmaximum_acq_rel_half_global_cta_param_1];
-; SM70-FTZ-NEXT: ld.param.b64 %rd2, [fmaximum_acq_rel_half_global_cta_param_0];
-; SM70-FTZ-NEXT: fence.acq_rel.cta;
-; SM70-FTZ-NEXT: and.b64 %rd1, %rd2, -4;
-; SM70-FTZ-NEXT: cvt.u32.u64 %r4, %rd2;
-; SM70-FTZ-NEXT: and.b32 %r5, %r4, 3;
-; SM70-FTZ-NEXT: shl.b32 %r1, %r5, 3;
-; SM70-FTZ-NEXT: mov.b32 %r6, 65535;
-; SM70-FTZ-NEXT: shl.b32 %r7, %r6, %r1;
-; SM70-FTZ-NEXT: not.b32 %r2, %r7;
-; SM70-FTZ-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
-; SM70-FTZ-NEXT: $L__BB77_1: // %atomicrmw.start
-; SM70-FTZ-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-FTZ-NEXT: shr.u32 %r8, %r14, %r1;
-; SM70-FTZ-NEXT: cvt.u16.u32 %rs2, %r8;
-; SM70-FTZ-NEXT: setp.eq.b16 %p1, %rs2, 0;
-; SM70-FTZ-NEXT: selp.b16 %rs3, %rs2, %rs1, %p1;
-; SM70-FTZ-NEXT: setp.gt.ftz.f16 %p2, %rs2, %rs1;
-; SM70-FTZ-NEXT: selp.b16 %rs4, %rs2, %rs1, %p2;
-; SM70-FTZ-NEXT: setp.nan.ftz.f16 %p3, %rs2, %rs1;
-; SM70-FTZ-NEXT: selp.b16 %rs5, 0x7E00, %rs4, %p3;
-; SM70-FTZ-NEXT: setp.eq.ftz.f16 %p4, %rs2, %rs1;
-; SM70-FTZ-NEXT: selp.b16 %rs6, %rs3, %rs5, %p4;
-; SM70-FTZ-NEXT: cvt.u32.u16 %r9, %rs6;
-; SM70-FTZ-NEXT: shl.b32 %r10, %r9, %r1;
-; SM70-FTZ-NEXT: and.b32 %r11, %r14, %r2;
-; SM70-FTZ-NEXT: or.b32 %r12, %r11, %r10;
-; SM70-FTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
-; SM70-FTZ-NEXT: setp.ne.b32 %p5, %r3, %r14;
-; SM70-FTZ-NEXT: mov.b32 %r14, %r3;
-; SM70-FTZ-NEXT: @%p5 bra $L__BB77_1;
-; SM70-FTZ-NEXT: // %bb.2: // %atomicrmw.end
-; SM70-FTZ-NEXT: shr.u32 %r13, %r3, %r1;
-; SM70-FTZ-NEXT: fence.acq_rel.cta;
-; SM70-FTZ-NEXT: st.param.b16 [func_retval0], %r13;
-; SM70-FTZ-NEXT: ret;
+; SM70-FTZ-DISALLOW-LABEL: fmaximum_acq_rel_half_global_cta(
+; SM70-FTZ-DISALLOW: {
+; SM70-FTZ-DISALLOW-NEXT: .reg .pred %p<6>;
+; SM70-FTZ-DISALLOW-NEXT: .reg .b16 %rs<7>;
+; SM70-FTZ-DISALLOW-NEXT: .reg .b32 %r<15>;
+; SM70-FTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
+; SM70-FTZ-DISALLOW-EMPTY:
+; SM70-FTZ-DISALLOW-NEXT: // %bb.0:
+; SM70-FTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fmaximum_acq_rel_half_global_cta_param_1];
+; SM70-FTZ-DISALLOW-NEXT: ld.param.b64 %rd2, [fmaximum_acq_rel_half_global_cta_param_0];
+; SM70-FTZ-DISALLOW-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-DISALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-FTZ-DISALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-FTZ-DISALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM70-FTZ-DISALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-FTZ-DISALLOW-NEXT: mov.b32 %r6, 65535;
+; SM70-FTZ-DISALLOW-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-FTZ-DISALLOW-NEXT: not.b32 %r2, %r7;
+; SM70-FTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM70-FTZ-DISALLOW-NEXT: $L__BB77_1: // %atomicrmw.start
+; SM70-FTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-FTZ-DISALLOW-NEXT: shr.u32 %r8, %r14, %r1;
+; SM70-FTZ-DISALLOW-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM70-FTZ-DISALLOW-NEXT: setp.eq.b16 %p1, %rs2, 0;
+; SM70-FTZ-DISALLOW-NEXT: selp.b16 %rs3, %rs2, %rs1, %p1;
+; SM70-FTZ-DISALLOW-NEXT: setp.gt.ftz.f16 %p2, %rs2, %rs1;
+; SM70-FTZ-DISALLOW-NEXT: selp.b16 %rs4, %rs2, %rs1, %p2;
+; SM70-FTZ-DISALLOW-NEXT: setp.nan.ftz.f16 %p3, %rs2, %rs1;
+; SM70-FTZ-DISALLOW-NEXT: selp.b16 %rs5, 0x7E00, %rs4, %p3;
+; SM70-FTZ-DISALLOW-NEXT: setp.eq.ftz.f16 %p4, %rs2, %rs1;
+; SM70-FTZ-DISALLOW-NEXT: selp.b16 %rs6, %rs3, %rs5, %p4;
+; SM70-FTZ-DISALLOW-NEXT: cvt.u32.u16 %r9, %rs6;
+; SM70-FTZ-DISALLOW-NEXT: shl.b32 %r10, %r9, %r1;
+; SM70-FTZ-DISALLOW-NEXT: and.b32 %r11, %r14, %r2;
+; SM70-FTZ-DISALLOW-NEXT: or.b32 %r12, %r11, %r10;
+; SM70-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM70-FTZ-DISALLOW-NEXT: setp.ne.b32 %p5, %r3, %r14;
+; SM70-FTZ-DISALLOW-NEXT: mov.b32 %r14, %r3;
+; SM70-FTZ-DISALLOW-NEXT: @%p5 bra $L__BB77_1;
+; SM70-FTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-FTZ-DISALLOW-NEXT: shr.u32 %r13, %r3, %r1;
+; SM70-FTZ-DISALLOW-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-DISALLOW-NEXT: st.param.b16 [func_retval0], %r13;
+; SM70-FTZ-DISALLOW-NEXT: ret;
+;
+; SM70-FTZ-ALLOW-LABEL: fmaximum_acq_rel_half_global_cta(
+; SM70-FTZ-ALLOW: {
+; SM70-FTZ-ALLOW-NEXT: .reg .pred %p<6>;
+; SM70-FTZ-ALLOW-NEXT: .reg .b16 %rs<7>;
+; SM70-FTZ-ALLOW-NEXT: .reg .b32 %r<15>;
+; SM70-FTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
+; SM70-FTZ-ALLOW-EMPTY:
+; SM70-FTZ-ALLOW-NEXT: // %bb.0:
+; SM70-FTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fmaximum_acq_rel_half_global_cta_param_1];
+; SM70-FTZ-ALLOW-NEXT: ld.param.b64 %rd2, [fmaximum_acq_rel_half_global_cta_param_0];
+; SM70-FTZ-ALLOW-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-ALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-FTZ-ALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-FTZ-ALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM70-FTZ-ALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-FTZ-ALLOW-NEXT: mov.b32 %r6, 65535;
+; SM70-FTZ-ALLOW-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-FTZ-ALLOW-NEXT: not.b32 %r2, %r7;
+; SM70-FTZ-ALLOW-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM70-FTZ-ALLOW-NEXT: $L__BB77_1: // %atomicrmw.start
+; SM70-FTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-FTZ-ALLOW-NEXT: shr.u32 %r8, %r14, %r1;
+; SM70-FTZ-ALLOW-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM70-FTZ-ALLOW-NEXT: setp.eq.b16 %p1, %rs2, 0;
+; SM70-FTZ-ALLOW-NEXT: selp.b16 %rs3, %rs2, %rs1, %p1;
+; SM70-FTZ-ALLOW-NEXT: setp.gt.ftz.f16 %p2, %rs2, %rs1;
+; SM70-FTZ-ALLOW-NEXT: selp.b16 %rs4, %rs2, %rs1, %p2;
+; SM70-FTZ-ALLOW-NEXT: setp.nan.ftz.f16 %p3, %rs2, %rs1;
+; SM70-FTZ-ALLOW-NEXT: selp.b16 %rs5, 0x7E00, %rs4, %p3;
+; SM70-FTZ-ALLOW-NEXT: setp.eq.ftz.f16 %p4, %rs2, %rs1;
+; SM70-FTZ-ALLOW-NEXT: selp.b16 %rs6, %rs3, %rs5, %p4;
+; SM70-FTZ-ALLOW-NEXT: cvt.u32.u16 %r9, %rs6;
+; SM70-FTZ-ALLOW-NEXT: shl.b32 %r10, %r9, %r1;
+; SM70-FTZ-ALLOW-NEXT: and.b32 %r11, %r14, %r2;
+; SM70-FTZ-ALLOW-NEXT: or.b32 %r12, %r11, %r10;
+; SM70-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM70-FTZ-ALLOW-NEXT: setp.ne.b32 %p5, %r3, %r14;
+; SM70-FTZ-ALLOW-NEXT: mov.b32 %r14, %r3;
+; SM70-FTZ-ALLOW-NEXT: @%p5 bra $L__BB77_1;
+; SM70-FTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-FTZ-ALLOW-NEXT: shr.u32 %r13, %r3, %r1;
+; SM70-FTZ-ALLOW-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-ALLOW-NEXT: st.param.b16 [func_retval0], %r13;
+; SM70-FTZ-ALLOW-NEXT: ret;
%retval = atomicrmw fmaximum ptr addrspace(1) %addr, half %val syncscope("block") acq_rel
ret half %retval
}
define bfloat @fadd_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat %val) {
-; SM70-NOFTZ-LABEL: fadd_acq_rel_bfloat_global_cta(
-; SM70-NOFTZ: {
-; SM70-NOFTZ-NEXT: .reg .pred %p<3>;
-; SM70-NOFTZ-NEXT: .reg .b16 %rs<2>;
-; SM70-NOFTZ-NEXT: .reg .b32 %r<24>;
-; SM70-NOFTZ-NEXT: .reg .b64 %rd<3>;
-; SM70-NOFTZ-EMPTY:
-; SM70-NOFTZ-NEXT: // %bb.0:
-; SM70-NOFTZ-NEXT: ld.param.b16 %rs1, [fadd_acq_rel_bfloat_global_cta_param_1];
-; SM70-NOFTZ-NEXT: ld.param.b64 %rd2, [fadd_acq_rel_bfloat_global_cta_param_0];
-; SM70-NOFTZ-NEXT: fence.acq_rel.cta;
-; SM70-NOFTZ-NEXT: and.b64 %rd1, %rd2, -4;
-; SM70-NOFTZ-NEXT: cvt.u32.u64 %r4, %rd2;
-; SM70-NOFTZ-NEXT: and.b32 %r5, %r4, 3;
-; SM70-NOFTZ-NEXT: shl.b32 %r1, %r5, 3;
-; SM70-NOFTZ-NEXT: mov.b32 %r6, 65535;
-; SM70-NOFTZ-NEXT: shl.b32 %r7, %r6, %r1;
-; SM70-NOFTZ-NEXT: not.b32 %r2, %r7;
-; SM70-NOFTZ-NEXT: ld.relaxed.cta.global.b32 %r23, [%rd1];
-; SM70-NOFTZ-NEXT: cvt.u32.u16 %r10, %rs1;
-; SM70-NOFTZ-NEXT: shl.b32 %r11, %r10, 16;
-; SM70-NOFTZ-NEXT: $L__BB78_1: // %atomicrmw.start
-; SM70-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NOFTZ-NEXT: shr.u32 %r8, %r23, %r1;
-; SM70-NOFTZ-NEXT: shl.b32 %r9, %r8, 16;
-; SM70-NOFTZ-NEXT: add.rn.f32 %r12, %r9, %r11;
-; SM70-NOFTZ-NEXT: bfe.u32 %r13, %r12, 16, 1;
-; SM70-NOFTZ-NEXT: add.s32 %r14, %r13, %r12;
-; SM70-NOFTZ-NEXT: add.s32 %r15, %r14, 32767;
-; SM70-NOFTZ-NEXT: setp.nan.f32 %p1, %r12, %r12;
-; SM70-NOFTZ-NEXT: or.b32 %r16, %r12, 4194304;
-; SM70-NOFTZ-NEXT: selp.b32 %r17, %r16, %r15, %p1;
-; SM70-NOFTZ-NEXT: shr.u32 %r18, %r17, 16;
-; SM70-NOFTZ-NEXT: shl.b32 %r19, %r18, %r1;
-; SM70-NOFTZ-NEXT: and.b32 %r20, %r23, %r2;
-; SM70-NOFTZ-NEXT: or.b32 %r21, %r20, %r19;
-; SM70-NOFTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r23, %r21;
-; SM70-NOFTZ-NEXT: setp.ne.b32 %p2, %r3, %r23;
-; SM70-NOFTZ-NEXT: mov.b32 %r23, %r3;
-; SM70-NOFTZ-NEXT: @%p2 bra $L__BB78_1;
-; SM70-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end
-; SM70-NOFTZ-NEXT: shr.u32 %r22, %r3, %r1;
-; SM70-NOFTZ-NEXT: fence.acq_rel.cta;
-; SM70-NOFTZ-NEXT: st.param.b16 [func_retval0], %r22;
-; SM70-NOFTZ-NEXT: ret;
+; SM70-NOFTZ-DISALLOW-LABEL: fadd_acq_rel_bfloat_global_cta(
+; SM70-NOFTZ-DISALLOW: {
+; SM70-NOFTZ-DISALLOW-NEXT: .reg .pred %p<3>;
+; SM70-NOFTZ-DISALLOW-NEXT: .reg .b16 %rs<2>;
+; SM70-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<24>;
+; SM70-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
+; SM70-NOFTZ-DISALLOW-EMPTY:
+; SM70-NOFTZ-DISALLOW-NEXT: // %bb.0:
+; SM70-NOFTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fadd_acq_rel_bfloat_global_cta_param_1];
+; SM70-NOFTZ-DISALLOW-NEXT: ld.param.b64 %rd2, [fadd_acq_rel_bfloat_global_cta_param_0];
+; SM70-NOFTZ-DISALLOW-NEXT: fence.acq_rel.cta;
+; SM70-NOFTZ-DISALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-NOFTZ-DISALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM70-NOFTZ-DISALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-NOFTZ-DISALLOW-NEXT: mov.b32 %r6, 65535;
+; SM70-NOFTZ-DISALLOW-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-NOFTZ-DISALLOW-NEXT: not.b32 %r2, %r7;
+; SM70-NOFTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b32 %r23, [%rd1];
+; SM70-NOFTZ-DISALLOW-NEXT: cvt.u32.u16 %r10, %rs1;
+; SM70-NOFTZ-DISALLOW-NEXT: shl.b32 %r11, %r10, 16;
+; SM70-NOFTZ-DISALLOW-NEXT: $L__BB78_1: // %atomicrmw.start
+; SM70-NOFTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NOFTZ-DISALLOW-NEXT: shr.u32 %r8, %r23, %r1;
+; SM70-NOFTZ-DISALLOW-NEXT: shl.b32 %r9, %r8, 16;
+; SM70-NOFTZ-DISALLOW-NEXT: add.rn.f32 %r12, %r9, %r11;
+; SM70-NOFTZ-DISALLOW-NEXT: bfe.u32 %r13, %r12, 16, 1;
+; SM70-NOFTZ-DISALLOW-NEXT: add.s32 %r14, %r13, %r12;
+; SM70-NOFTZ-DISALLOW-NEXT: add.s32 %r15, %r14, 32767;
+; SM70-NOFTZ-DISALLOW-NEXT: setp.nan.f32 %p1, %r12, %r12;
+; SM70-NOFTZ-DISALLOW-NEXT: or.b32 %r16, %r12, 4194304;
+; SM70-NOFTZ-DISALLOW-NEXT: selp.b32 %r17, %r16, %r15, %p1;
+; SM70-NOFTZ-DISALLOW-NEXT: shr.u32 %r18, %r17, 16;
+; SM70-NOFTZ-DISALLOW-NEXT: shl.b32 %r19, %r18, %r1;
+; SM70-NOFTZ-DISALLOW-NEXT: and.b32 %r20, %r23, %r2;
+; SM70-NOFTZ-DISALLOW-NEXT: or.b32 %r21, %r20, %r19;
+; SM70-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r23, %r21;
+; SM70-NOFTZ-DISALLOW-NEXT: setp.ne.b32 %p2, %r3, %r23;
+; SM70-NOFTZ-DISALLOW-NEXT: mov.b32 %r23, %r3;
+; SM70-NOFTZ-DISALLOW-NEXT: @%p2 bra $L__BB78_1;
+; SM70-NOFTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NOFTZ-DISALLOW-NEXT: shr.u32 %r22, %r3, %r1;
+; SM70-NOFTZ-DISALLOW-NEXT: fence.acq_rel.cta;
+; SM70-NOFTZ-DISALLOW-NEXT: st.param.b16 [func_retval0], %r22;
+; SM70-NOFTZ-DISALLOW-NEXT: ret;
+;
+; SM70-NOFTZ-ALLOW-LABEL: fadd_acq_rel_bfloat_global_cta(
+; SM70-NOFTZ-ALLOW: {
+; SM70-NOFTZ-ALLOW-NEXT: .reg .pred %p<3>;
+; SM70-NOFTZ-ALLOW-NEXT: .reg .b16 %rs<2>;
+; SM70-NOFTZ-ALLOW-NEXT: .reg .b32 %r<24>;
+; SM70-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
+; SM70-NOFTZ-ALLOW-EMPTY:
+; SM70-NOFTZ-ALLOW-NEXT: // %bb.0:
+; SM70-NOFTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fadd_acq_rel_bfloat_global_cta_param_1];
+; SM70-NOFTZ-ALLOW-NEXT: ld.param.b64 %rd2, [fadd_acq_rel_bfloat_global_cta_param_0];
+; SM70-NOFTZ-ALLOW-NEXT: fence.acq_rel.cta;
+; SM70-NOFTZ-ALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NOFTZ-ALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-NOFTZ-ALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM70-NOFTZ-ALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-NOFTZ-ALLOW-NEXT: mov.b32 %r6, 65535;
+; SM70-NOFTZ-ALLOW-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-NOFTZ-ALLOW-NEXT: not.b32 %r2, %r7;
+; SM70-NOFTZ-ALLOW-NEXT: ld.relaxed.cta.global.b32 %r23, [%rd1];
+; SM70-NOFTZ-ALLOW-NEXT: cvt.u32.u16 %r10, %rs1;
+; SM70-NOFTZ-ALLOW-NEXT: shl.b32 %r11, %r10, 16;
+; SM70-NOFTZ-ALLOW-NEXT: $L__BB78_1: // %atomicrmw.start
+; SM70-NOFTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NOFTZ-ALLOW-NEXT: shr.u32 %r8, %r23, %r1;
+; SM70-NOFTZ-ALLOW-NEXT: shl.b32 %r9, %r8, 16;
+; SM70-NOFTZ-ALLOW-NEXT: add.rn.f32 %r12, %r9, %r11;
+; SM70-NOFTZ-ALLOW-NEXT: bfe.u32 %r13, %r12, 16, 1;
+; SM70-NOFTZ-ALLOW-NEXT: add.s32 %r14, %r13, %r12;
+; SM70-NOFTZ-ALLOW-NEXT: add.s32 %r15, %r14, 32767;
+; SM70-NOFTZ-ALLOW-NEXT: setp.nan.f32 %p1, %r12, %r12;
+; SM70-NOFTZ-ALLOW-NEXT: or.b32 %r16, %r12, 4194304;
+; SM70-NOFTZ-ALLOW-NEXT: selp.b32 %r17, %r16, %r15, %p1;
+; SM70-NOFTZ-ALLOW-NEXT: shr.u32 %r18, %r17, 16;
+; SM70-NOFTZ-ALLOW-NEXT: shl.b32 %r19, %r18, %r1;
+; SM70-NOFTZ-ALLOW-NEXT: and.b32 %r20, %r23, %r2;
+; SM70-NOFTZ-ALLOW-NEXT: or.b32 %r21, %r20, %r19;
+; SM70-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r23, %r21;
+; SM70-NOFTZ-ALLOW-NEXT: setp.ne.b32 %p2, %r3, %r23;
+; SM70-NOFTZ-ALLOW-NEXT: mov.b32 %r23, %r3;
+; SM70-NOFTZ-ALLOW-NEXT: @%p2 bra $L__BB78_1;
+; SM70-NOFTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NOFTZ-ALLOW-NEXT: shr.u32 %r22, %r3, %r1;
+; SM70-NOFTZ-ALLOW-NEXT: fence.acq_rel.cta;
+; SM70-NOFTZ-ALLOW-NEXT: st.param.b16 [func_retval0], %r22;
+; SM70-NOFTZ-ALLOW-NEXT: ret;
+;
+; SM70-FTZ-DISALLOW-LABEL: fadd_acq_rel_bfloat_global_cta(
+; SM70-FTZ-DISALLOW: {
+; SM70-FTZ-DISALLOW-NEXT: .reg .pred %p<3>;
+; SM70-FTZ-DISALLOW-NEXT: .reg .b16 %rs<2>;
+; SM70-FTZ-DISALLOW-NEXT: .reg .b32 %r<24>;
+; SM70-FTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
+; SM70-FTZ-DISALLOW-EMPTY:
+; SM70-FTZ-DISALLOW-NEXT: // %bb.0:
+; SM70-FTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fadd_acq_rel_bfloat_global_cta_param_1];
+; SM70-FTZ-DISALLOW-NEXT: ld.param.b64 %rd2, [fadd_acq_rel_bfloat_global_cta_param_0];
+; SM70-FTZ-DISALLOW-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-DISALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-FTZ-DISALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-FTZ-DISALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM70-FTZ-DISALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-FTZ-DISALLOW-NEXT: mov.b32 %r6, 65535;
+; SM70-FTZ-DISALLOW-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-FTZ-DISALLOW-NEXT: not.b32 %r2, %r7;
+; SM70-FTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b32 %r23, [%rd1];
+; SM70-FTZ-DISALLOW-NEXT: cvt.u32.u16 %r10, %rs1;
+; SM70-FTZ-DISALLOW-NEXT: shl.b32 %r11, %r10, 16;
+; SM70-FTZ-DISALLOW-NEXT: $L__BB78_1: // %atomicrmw.start
+; SM70-FTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-FTZ-DISALLOW-NEXT: shr.u32 %r8, %r23, %r1;
+; SM70-FTZ-DISALLOW-NEXT: shl.b32 %r9, %r8, 16;
+; SM70-FTZ-DISALLOW-NEXT: add.rn.ftz.f32 %r12, %r9, %r11;
+; SM70-FTZ-DISALLOW-NEXT: bfe.u32 %r13, %r12, 16, 1;
+; SM70-FTZ-DISALLOW-NEXT: add.s32 %r14, %r13, %r12;
+; SM70-FTZ-DISALLOW-NEXT: add.s32 %r15, %r14, 32767;
+; SM70-FTZ-DISALLOW-NEXT: setp.nan.ftz.f32 %p1, %r12, %r12;
+; SM70-FTZ-DISALLOW-NEXT: or.b32 %r16, %r12, 4194304;
+; SM70-FTZ-DISALLOW-NEXT: selp.b32 %r17, %r16, %r15, %p1;
+; SM70-FTZ-DISALLOW-NEXT: shr.u32 %r18, %r17, 16;
+; SM70-FTZ-DISALLOW-NEXT: shl.b32 %r19, %r18, %r1;
+; SM70-FTZ-DISALLOW-NEXT: and.b32 %r20, %r23, %r2;
+; SM70-FTZ-DISALLOW-NEXT: or.b32 %r21, %r20, %r19;
+; SM70-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r23, %r21;
+; SM70-FTZ-DISALLOW-NEXT: setp.ne.b32 %p2, %r3, %r23;
+; SM70-FTZ-DISALLOW-NEXT: mov.b32 %r23, %r3;
+; SM70-FTZ-DISALLOW-NEXT: @%p2 bra $L__BB78_1;
+; SM70-FTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-FTZ-DISALLOW-NEXT: shr.u32 %r22, %r3, %r1;
+; SM70-FTZ-DISALLOW-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-DISALLOW-NEXT: st.param.b16 [func_retval0], %r22;
+; SM70-FTZ-DISALLOW-NEXT: ret;
;
-; SM70-FTZ-LABEL: fadd_acq_rel_bfloat_global_cta(
-; SM70-FTZ: {
-; SM70-FTZ-NEXT: .reg .pred %p<3>;
-; SM70-FTZ-NEXT: .reg .b16 %rs<2>;
-; SM70-FTZ-NEXT: .reg .b32 %r<24>;
-; SM70-FTZ-NEXT: .reg .b64 %rd<3>;
-; SM70-FTZ-EMPTY:
-; SM70-FTZ-NEXT: // %bb.0:
-; SM70-FTZ-NEXT: ld.param.b16 %rs1, [fadd_acq_rel_bfloat_global_cta_param_1];
-; SM70-FTZ-NEXT: ld.param.b64 %rd2, [fadd_acq_rel_bfloat_global_cta_param_0];
-; SM70-FTZ-NEXT: fence.acq_rel.cta;
-; SM70-FTZ-NEXT: and.b64 %rd1, %rd2, -4;
-; SM70-FTZ-NEXT: cvt.u32.u64 %r4, %rd2;
-; SM70-FTZ-NEXT: and.b32 %r5, %r4, 3;
-; SM70-FTZ-NEXT: shl.b32 %r1, %r5, 3;
-; SM70-FTZ-NEXT: mov.b32 %r6, 65535;
-; SM70-FTZ-NEXT: shl.b32 %r7, %r6, %r1;
-; SM70-FTZ-NEXT: not.b32 %r2, %r7;
-; SM70-FTZ-NEXT: ld.relaxed.cta.global.b32 %r23, [%rd1];
-; SM70-FTZ-NEXT: cvt.u32.u16 %r10, %rs1;
-; SM70-FTZ-NEXT: shl.b32 %r11, %r10, 16;
-; SM70-FTZ-NEXT: $L__BB78_1: // %atomicrmw.start
-; SM70-FTZ-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-FTZ-NEXT: shr.u32 %r8, %r23, %r1;
-; SM70-FTZ-NEXT: shl.b32 %r9, %r8, 16;
-; SM70-FTZ-NEXT: add.rn.ftz.f32 %r12, %r9, %r11;
-; SM70-FTZ-NEXT: bfe.u32 %r13, %r12, 16, 1;
-; SM70-FTZ-NEXT: add.s32 %r14, %r13, %r12;
-; SM70-FTZ-NEXT: add.s32 %r15, %r14, 32767;
-; SM70-FTZ-NEXT: setp.nan.ftz.f32 %p1, %r12, %r12;
-; SM70-FTZ-NEXT: or.b32 %r16, %r12, 4194304;
-; SM70-FTZ-NEXT: selp.b32 %r17, %r16, %r15, %p1;
-; SM70-FTZ-NEXT: shr.u32 %r18, %r17, 16;
-; SM70-FTZ-NEXT: shl.b32 %r19, %r18, %r1;
-; SM70-FTZ-NEXT: and.b32 %r20, %r23, %r2;
-; SM70-FTZ-NEXT: or.b32 %r21, %r20, %r19;
-; SM70-FTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r23, %r21;
-; SM70-FTZ-NEXT: setp.ne.b32 %p2, %r3, %r23;
-; SM70-FTZ-NEXT: mov.b32 %r23, %r3;
-; SM70-FTZ-NEXT: @%p2 bra $L__BB78_1;
-; SM70-FTZ-NEXT: // %bb.2: // %atomicrmw.end
-; SM70-FTZ-NEXT: shr.u32 %r22, %r3, %r1;
-; SM70-FTZ-NEXT: fence.acq_rel.cta;
-; SM70-FTZ-NEXT: st.param.b16 [func_retval0], %r22;
-; SM70-FTZ-NEXT: ret;
+; SM70-FTZ-ALLOW-LABEL: fadd_acq_rel_bfloat_global_cta(
+; SM70-FTZ-ALLOW: {
+; SM70-FTZ-ALLOW-NEXT: .reg .pred %p<3>;
+; SM70-FTZ-ALLOW-NEXT: .reg .b16 %rs<2>;
+; SM70-FTZ-ALLOW-NEXT: .reg .b32 %r<24>;
+; SM70-FTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
+; SM70-FTZ-ALLOW-EMPTY:
+; SM70-FTZ-ALLOW-NEXT: // %bb.0:
+; SM70-FTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fadd_acq_rel_bfloat_global_cta_param_1];
+; SM70-FTZ-ALLOW-NEXT: ld.param.b64 %rd2, [fadd_acq_rel_bfloat_global_cta_param_0];
+; SM70-FTZ-ALLOW-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-ALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-FTZ-ALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-FTZ-ALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM70-FTZ-ALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-FTZ-ALLOW-NEXT: mov.b32 %r6, 65535;
+; SM70-FTZ-ALLOW-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-FTZ-ALLOW-NEXT: not.b32 %r2, %r7;
+; SM70-FTZ-ALLOW-NEXT: ld.relaxed.cta.global.b32 %r23, [%rd1];
+; SM70-FTZ-ALLOW-NEXT: cvt.u32.u16 %r10, %rs1;
+; SM70-FTZ-ALLOW-NEXT: shl.b32 %r11, %r10, 16;
+; SM70-FTZ-ALLOW-NEXT: $L__BB78_1: // %atomicrmw.start
+; SM70-FTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-FTZ-ALLOW-NEXT: shr.u32 %r8, %r23, %r1;
+; SM70-FTZ-ALLOW-NEXT: shl.b32 %r9, %r8, 16;
+; SM70-FTZ-ALLOW-NEXT: add.rn.ftz.f32 %r12, %r9, %r11;
+; SM70-FTZ-ALLOW-NEXT: bfe.u32 %r13, %r12, 16, 1;
+; SM70-FTZ-ALLOW-NEXT: add.s32 %r14, %r13, %r12;
+; SM70-FTZ-ALLOW-NEXT: add.s32 %r15, %r14, 32767;
+; SM70-FTZ-ALLOW-NEXT: setp.nan.ftz.f32 %p1, %r12, %r12;
+; SM70-FTZ-ALLOW-NEXT: or.b32 %r16, %r12, 4194304;
+; SM70-FTZ-ALLOW-NEXT: selp.b32 %r17, %r16, %r15, %p1;
+; SM70-FTZ-ALLOW-NEXT: shr.u32 %r18, %r17, 16;
+; SM70-FTZ-ALLOW-NEXT: shl.b32 %r19, %r18, %r1;
+; SM70-FTZ-ALLOW-NEXT: and.b32 %r20, %r23, %r2;
+; SM70-FTZ-ALLOW-NEXT: or.b32 %r21, %r20, %r19;
+; SM70-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r23, %r21;
+; SM70-FTZ-ALLOW-NEXT: setp.ne.b32 %p2, %r3, %r23;
+; SM70-FTZ-ALLOW-NEXT: mov.b32 %r23, %r3;
+; SM70-FTZ-ALLOW-NEXT: @%p2 bra $L__BB78_1;
+; SM70-FTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-FTZ-ALLOW-NEXT: shr.u32 %r22, %r3, %r1;
+; SM70-FTZ-ALLOW-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-ALLOW-NEXT: st.param.b16 [func_retval0], %r22;
+; SM70-FTZ-ALLOW-NEXT: ret;
%retval = atomicrmw fadd ptr addrspace(1) %addr, bfloat %val syncscope("block") acq_rel
ret bfloat %retval
}
define bfloat @fsub_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat %val) {
-; SM70-NOFTZ-LABEL: fsub_acq_rel_bfloat_global_cta(
-; SM70-NOFTZ: {
-; SM70-NOFTZ-NEXT: .reg .pred %p<3>;
-; SM70-NOFTZ-NEXT: .reg .b16 %rs<2>;
-; SM70-NOFTZ-NEXT: .reg .b32 %r<24>;
-; SM70-NOFTZ-NEXT: .reg .b64 %rd<3>;
-; SM70-NOFTZ-EMPTY:
-; SM70-NOFTZ-NEXT: // %bb.0:
-; SM70-NOFTZ-NEXT: ld.param.b16 %rs1, [fsub_acq_rel_bfloat_global_cta_param_1];
-; SM70-NOFTZ-NEXT: ld.param.b64 %rd2, [fsub_acq_rel_bfloat_global_cta_param_0];
-; SM70-NOFTZ-NEXT: fence.acq_rel.cta;
-; SM70-NOFTZ-NEXT: and.b64 %rd1, %rd2, -4;
-; SM70-NOFTZ-NEXT: cvt.u32.u64 %r4, %rd2;
-; SM70-NOFTZ-NEXT: and.b32 %r5, %r4, 3;
-; SM70-NOFTZ-NEXT: shl.b32 %r1, %r5, 3;
-; SM70-NOFTZ-NEXT: mov.b32 %r6, 65535;
-; SM70-NOFTZ-NEXT: shl.b32 %r7, %r6, %r1;
-; SM70-NOFTZ-NEXT: not.b32 %r2, %r7;
-; SM70-NOFTZ-NEXT: ld.relaxed.cta.global.b32 %r23, [%rd1];
-; SM70-NOFTZ-NEXT: cvt.u32.u16 %r10, %rs1;
-; SM70-NOFTZ-NEXT: shl.b32 %r11, %r10, 16;
-; SM70-NOFTZ-NEXT: $L__BB79_1: // %atomicrmw.start
-; SM70-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NOFTZ-NEXT: shr.u32 %r8, %r23, %r1;
-; SM70-NOFTZ-NEXT: shl.b32 %r9, %r8, 16;
-; SM70-NOFTZ-NEXT: sub.rn.f32 %r12, %r9, %r11;
-; SM70-NOFTZ-NEXT: bfe.u32 %r13, %r12, 16, 1;
-; SM70-NOFTZ-NEXT: add.s32 %r14, %r13, %r12;
-; SM70-NOFTZ-NEXT: add.s32 %r15, %r14, 32767;
-; SM70-NOFTZ-NEXT: setp.nan.f32 %p1, %r12, %r12;
-; SM70-NOFTZ-NEXT: or.b32 %r16, %r12, 4194304;
-; SM70-NOFTZ-NEXT: selp.b32 %r17, %r16, %r15, %p1;
-; SM70-NOFTZ-NEXT: shr.u32 %r18, %r17, 16;
-; SM70-NOFTZ-NEXT: shl.b32 %r19, %r18, %r1;
-; SM70-NOFTZ-NEXT: and.b32 %r20, %r23, %r2;
-; SM70-NOFTZ-NEXT: or.b32 %r21, %r20, %r19;
-; SM70-NOFTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r23, %r21;
-; SM70-NOFTZ-NEXT: setp.ne.b32 %p2, %r3, %r23;
-; SM70-NOFTZ-NEXT: mov.b32 %r23, %r3;
-; SM70-NOFTZ-NEXT: @%p2 bra $L__BB79_1;
-; SM70-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end
-; SM70-NOFTZ-NEXT: shr.u32 %r22, %r3, %r1;
-; SM70-NOFTZ-NEXT: fence.acq_rel.cta;
-; SM70-NOFTZ-NEXT: st.param.b16 [func_retval0], %r22;
-; SM70-NOFTZ-NEXT: ret;
+; SM70-NOFTZ-DISALLOW-LABEL: fsub_acq_rel_bfloat_global_cta(
+; SM70-NOFTZ-DISALLOW: {
+; SM70-NOFTZ-DISALLOW-NEXT: .reg .pred %p<3>;
+; SM70-NOFTZ-DISALLOW-NEXT: .reg .b16 %rs<2>;
+; SM70-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<24>;
+; SM70-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
+; SM70-NOFTZ-DISALLOW-EMPTY:
+; SM70-NOFTZ-DISALLOW-NEXT: // %bb.0:
+; SM70-NOFTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fsub_acq_rel_bfloat_global_cta_param_1];
+; SM70-NOFTZ-DISALLOW-NEXT: ld.param.b64 %rd2, [fsub_acq_rel_bfloat_global_cta_param_0];
+; SM70-NOFTZ-DISALLOW-NEXT: fence.acq_rel.cta;
+; SM70-NOFTZ-DISALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-NOFTZ-DISALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM70-NOFTZ-DISALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-NOFTZ-DISALLOW-NEXT: mov.b32 %r6, 65535;
+; SM70-NOFTZ-DISALLOW-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-NOFTZ-DISALLOW-NEXT: not.b32 %r2, %r7;
+; SM70-NOFTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b32 %r23, [%rd1];
+; SM70-NOFTZ-DISALLOW-NEXT: cvt.u32.u16 %r10, %rs1;
+; SM70-NOFTZ-DISALLOW-NEXT: shl.b32 %r11, %r10, 16;
+; SM70-NOFTZ-DISALLOW-NEXT: $L__BB79_1: // %atomicrmw.start
+; SM70-NOFTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NOFTZ-DISALLOW-NEXT: shr.u32 %r8, %r23, %r1;
+; SM70-NOFTZ-DISALLOW-NEXT: shl.b32 %r9, %r8, 16;
+; SM70-NOFTZ-DISALLOW-NEXT: sub.rn.f32 %r12, %r9, %r11;
+; SM70-NOFTZ-DISALLOW-NEXT: bfe.u32 %r13, %r12, 16, 1;
+; SM70-NOFTZ-DISALLOW-NEXT: add.s32 %r14, %r13, %r12;
+; SM70-NOFTZ-DISALLOW-NEXT: add.s32 %r15, %r14, 32767;
+; SM70-NOFTZ-DISALLOW-NEXT: setp.nan.f32 %p1, %r12, %r12;
+; SM70-NOFTZ-DISALLOW-NEXT: or.b32 %r16, %r12, 4194304;
+; SM70-NOFTZ-DISALLOW-NEXT: selp.b32 %r17, %r16, %r15, %p1;
+; SM70-NOFTZ-DISALLOW-NEXT: shr.u32 %r18, %r17, 16;
+; SM70-NOFTZ-DISALLOW-NEXT: shl.b32 %r19, %r18, %r1;
+; SM70-NOFTZ-DISALLOW-NEXT: and.b32 %r20, %r23, %r2;
+; SM70-NOFTZ-DISALLOW-NEXT: or.b32 %r21, %r20, %r19;
+; SM70-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r23, %r21;
+; SM70-NOFTZ-DISALLOW-NEXT: setp.ne.b32 %p2, %r3, %r23;
+; SM70-NOFTZ-DISALLOW-NEXT: mov.b32 %r23, %r3;
+; SM70-NOFTZ-DISALLOW-NEXT: @%p2 bra $L__BB79_1;
+; SM70-NOFTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NOFTZ-DISALLOW-NEXT: shr.u32 %r22, %r3, %r1;
+; SM70-NOFTZ-DISALLOW-NEXT: fence.acq_rel.cta;
+; SM70-NOFTZ-DISALLOW-NEXT: st.param.b16 [func_retval0], %r22;
+; SM70-NOFTZ-DISALLOW-NEXT: ret;
;
-; SM70-FTZ-LABEL: fsub_acq_rel_bfloat_global_cta(
-; SM70-FTZ: {
-; SM70-FTZ-NEXT: .reg .pred %p<3>;
-; SM70-FTZ-NEXT: .reg .b16 %rs<2>;
-; SM70-FTZ-NEXT: .reg .b32 %r<24>;
-; SM70-FTZ-NEXT: .reg .b64 %rd<3>;
-; SM70-FTZ-EMPTY:
-; SM70-FTZ-NEXT: // %bb.0:
-; SM70-FTZ-NEXT: ld.param.b16 %rs1, [fsub_acq_rel_bfloat_global_cta_param_1];
-; SM70-FTZ-NEXT: ld.param.b64 %rd2, [fsub_acq_rel_bfloat_global_cta_param_0];
-; SM70-FTZ-NEXT: fence.acq_rel.cta;
-; SM70-FTZ-NEXT: and.b64 %rd1, %rd2, -4;
-; SM70-FTZ-NEXT: cvt.u32.u64 %r4, %rd2;
-; SM70-FTZ-NEXT: and.b32 %r5, %r4, 3;
-; SM70-FTZ-NEXT: shl.b32 %r1, %r5, 3;
-; SM70-FTZ-NEXT: mov.b32 %r6, 65535;
-; SM70-FTZ-NEXT: shl.b32 %r7, %r6, %r1;
-; SM70-FTZ-NEXT: not.b32 %r2, %r7;
-; SM70-FTZ-NEXT: ld.relaxed.cta.global.b32 %r23, [%rd1];
-; SM70-FTZ-NEXT: cvt.u32.u16 %r10, %rs1;
-; SM70-FTZ-NEXT: shl.b32 %r11, %r10, 16;
-; SM70-FTZ-NEXT: $L__BB79_1: // %atomicrmw.start
-; SM70-FTZ-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-FTZ-NEXT: shr.u32 %r8, %r23, %r1;
-; SM70-FTZ-NEXT: shl.b32 %r9, %r8, 16;
-; SM70-FTZ-NEXT: sub.rn.ftz.f32 %r12, %r9, %r11;
-; SM70-FTZ-NEXT: bfe.u32 %r13, %r12, 16, 1;
-; SM70-FTZ-NEXT: add.s32 %r14, %r13, %r12;
-; SM70-FTZ-NEXT: add.s32 %r15, %r14, 32767;
-; SM70-FTZ-NEXT: setp.nan.ftz.f32 %p1, %r12, %r12;
-; SM70-FTZ-NEXT: or.b32 %r16, %r12, 4194304;
-; SM70-FTZ-NEXT: selp.b32 %r17, %r16, %r15, %p1;
-; SM70-FTZ-NEXT: shr.u32 %r18, %r17, 16;
-; SM70-FTZ-NEXT: shl.b32 %r19, %r18, %r1;
-; SM70-FTZ-NEXT: and.b32 %r20, %r23, %r2;
-; SM70-FTZ-NEXT: or.b32 %r21, %r20, %r19;
-; SM70-FTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r23, %r21;
-; SM70-FTZ-NEXT: setp.ne.b32 %p2, %r3, %r23;
-; SM70-FTZ-NEXT: mov.b32 %r23, %r3;
-; SM70-FTZ-NEXT: @%p2 bra $L__BB79_1;
-; SM70-FTZ-NEXT: // %bb.2: // %atomicrmw.end
-; SM70-FTZ-NEXT: shr.u32 %r22, %r3, %r1;
-; SM70-FTZ-NEXT: fence.acq_rel.cta;
-; SM70-FTZ-NEXT: st.param.b16 [func_retval0], %r22;
-; SM70-FTZ-NEXT: ret;
+; SM70-NOFTZ-ALLOW-LABEL: fsub_acq_rel_bfloat_global_cta(
+; SM70-NOFTZ-ALLOW: {
+; SM70-NOFTZ-ALLOW-NEXT: .reg .pred %p<3>;
+; SM70-NOFTZ-ALLOW-NEXT: .reg .b16 %rs<2>;
+; SM70-NOFTZ-ALLOW-NEXT: .reg .b32 %r<24>;
+; SM70-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
+; SM70-NOFTZ-ALLOW-EMPTY:
+; SM70-NOFTZ-ALLOW-NEXT: // %bb.0:
+; SM70-NOFTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fsub_acq_rel_bfloat_global_cta_param_1];
+; SM70-NOFTZ-ALLOW-NEXT: ld.param.b64 %rd2, [fsub_acq_rel_bfloat_global_cta_param_0];
+; SM70-NOFTZ-ALLOW-NEXT: fence.acq_rel.cta;
+; SM70-NOFTZ-ALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NOFTZ-ALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-NOFTZ-ALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM70-NOFTZ-ALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-NOFTZ-ALLOW-NEXT: mov.b32 %r6, 65535;
+; SM70-NOFTZ-ALLOW-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-NOFTZ-ALLOW-NEXT: not.b32 %r2, %r7;
+; SM70-NOFTZ-ALLOW-NEXT: ld.relaxed.cta.global.b32 %r23, [%rd1];
+; SM70-NOFTZ-ALLOW-NEXT: cvt.u32.u16 %r10, %rs1;
+; SM70-NOFTZ-ALLOW-NEXT: shl.b32 %r11, %r10, 16;
+; SM70-NOFTZ-ALLOW-NEXT: $L__BB79_1: // %atomicrmw.start
+; SM70-NOFTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NOFTZ-ALLOW-NEXT: shr.u32 %r8, %r23, %r1;
+; SM70-NOFTZ-ALLOW-NEXT: shl.b32 %r9, %r8, 16;
+; SM70-NOFTZ-ALLOW-NEXT: sub.rn.f32 %r12, %r9, %r11;
+; SM70-NOFTZ-ALLOW-NEXT: bfe.u32 %r13, %r12, 16, 1;
+; SM70-NOFTZ-ALLOW-NEXT: add.s32 %r14, %r13, %r12;
+; SM70-NOFTZ-ALLOW-NEXT: add.s32 %r15, %r14, 32767;
+; SM70-NOFTZ-ALLOW-NEXT: setp.nan.f32 %p1, %r12, %r12;
+; SM70-NOFTZ-ALLOW-NEXT: or.b32 %r16, %r12, 4194304;
+; SM70-NOFTZ-ALLOW-NEXT: selp.b32 %r17, %r16, %r15, %p1;
+; SM70-NOFTZ-ALLOW-NEXT: shr.u32 %r18, %r17, 16;
+; SM70-NOFTZ-ALLOW-NEXT: shl.b32 %r19, %r18, %r1;
+; SM70-NOFTZ-ALLOW-NEXT: and.b32 %r20, %r23, %r2;
+; SM70-NOFTZ-ALLOW-NEXT: or.b32 %r21, %r20, %r19;
+; SM70-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r23, %r21;
+; SM70-NOFTZ-ALLOW-NEXT: setp.ne.b32 %p2, %r3, %r23;
+; SM70-NOFTZ-ALLOW-NEXT: mov.b32 %r23, %r3;
+; SM70-NOFTZ-ALLOW-NEXT: @%p2 bra $L__BB79_1;
+; SM70-NOFTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NOFTZ-ALLOW-NEXT: shr.u32 %r22, %r3, %r1;
+; SM70-NOFTZ-ALLOW-NEXT: fence.acq_rel.cta;
+; SM70-NOFTZ-ALLOW-NEXT: st.param.b16 [func_retval0], %r22;
+; SM70-NOFTZ-ALLOW-NEXT: ret;
+;
+; SM70-FTZ-DISALLOW-LABEL: fsub_acq_rel_bfloat_global_cta(
+; SM70-FTZ-DISALLOW: {
+; SM70-FTZ-DISALLOW-NEXT: .reg .pred %p<3>;
+; SM70-FTZ-DISALLOW-NEXT: .reg .b16 %rs<2>;
+; SM70-FTZ-DISALLOW-NEXT: .reg .b32 %r<24>;
+; SM70-FTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
+; SM70-FTZ-DISALLOW-EMPTY:
+; SM70-FTZ-DISALLOW-NEXT: // %bb.0:
+; SM70-FTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fsub_acq_rel_bfloat_global_cta_param_1];
+; SM70-FTZ-DISALLOW-NEXT: ld.param.b64 %rd2, [fsub_acq_rel_bfloat_global_cta_param_0];
+; SM70-FTZ-DISALLOW-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-DISALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-FTZ-DISALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-FTZ-DISALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM70-FTZ-DISALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-FTZ-DISALLOW-NEXT: mov.b32 %r6, 65535;
+; SM70-FTZ-DISALLOW-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-FTZ-DISALLOW-NEXT: not.b32 %r2, %r7;
+; SM70-FTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b32 %r23, [%rd1];
+; SM70-FTZ-DISALLOW-NEXT: cvt.u32.u16 %r10, %rs1;
+; SM70-FTZ-DISALLOW-NEXT: shl.b32 %r11, %r10, 16;
+; SM70-FTZ-DISALLOW-NEXT: $L__BB79_1: // %atomicrmw.start
+; SM70-FTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-FTZ-DISALLOW-NEXT: shr.u32 %r8, %r23, %r1;
+; SM70-FTZ-DISALLOW-NEXT: shl.b32 %r9, %r8, 16;
+; SM70-FTZ-DISALLOW-NEXT: sub.rn.ftz.f32 %r12, %r9, %r11;
+; SM70-FTZ-DISALLOW-NEXT: bfe.u32 %r13, %r12, 16, 1;
+; SM70-FTZ-DISALLOW-NEXT: add.s32 %r14, %r13, %r12;
+; SM70-FTZ-DISALLOW-NEXT: add.s32 %r15, %r14, 32767;
+; SM70-FTZ-DISALLOW-NEXT: setp.nan.ftz.f32 %p1, %r12, %r12;
+; SM70-FTZ-DISALLOW-NEXT: or.b32 %r16, %r12, 4194304;
+; SM70-FTZ-DISALLOW-NEXT: selp.b32 %r17, %r16, %r15, %p1;
+; SM70-FTZ-DISALLOW-NEXT: shr.u32 %r18, %r17, 16;
+; SM70-FTZ-DISALLOW-NEXT: shl.b32 %r19, %r18, %r1;
+; SM70-FTZ-DISALLOW-NEXT: and.b32 %r20, %r23, %r2;
+; SM70-FTZ-DISALLOW-NEXT: or.b32 %r21, %r20, %r19;
+; SM70-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r23, %r21;
+; SM70-FTZ-DISALLOW-NEXT: setp.ne.b32 %p2, %r3, %r23;
+; SM70-FTZ-DISALLOW-NEXT: mov.b32 %r23, %r3;
+; SM70-FTZ-DISALLOW-NEXT: @%p2 bra $L__BB79_1;
+; SM70-FTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-FTZ-DISALLOW-NEXT: shr.u32 %r22, %r3, %r1;
+; SM70-FTZ-DISALLOW-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-DISALLOW-NEXT: st.param.b16 [func_retval0], %r22;
+; SM70-FTZ-DISALLOW-NEXT: ret;
+;
+; SM70-FTZ-ALLOW-LABEL: fsub_acq_rel_bfloat_global_cta(
+; SM70-FTZ-ALLOW: {
+; SM70-FTZ-ALLOW-NEXT: .reg .pred %p<3>;
+; SM70-FTZ-ALLOW-NEXT: .reg .b16 %rs<2>;
+; SM70-FTZ-ALLOW-NEXT: .reg .b32 %r<24>;
+; SM70-FTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
+; SM70-FTZ-ALLOW-EMPTY:
+; SM70-FTZ-ALLOW-NEXT: // %bb.0:
+; SM70-FTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fsub_acq_rel_bfloat_global_cta_param_1];
+; SM70-FTZ-ALLOW-NEXT: ld.param.b64 %rd2, [fsub_acq_rel_bfloat_global_cta_param_0];
+; SM70-FTZ-ALLOW-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-ALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-FTZ-ALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-FTZ-ALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM70-FTZ-ALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-FTZ-ALLOW-NEXT: mov.b32 %r6, 65535;
+; SM70-FTZ-ALLOW-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-FTZ-ALLOW-NEXT: not.b32 %r2, %r7;
+; SM70-FTZ-ALLOW-NEXT: ld.relaxed.cta.global.b32 %r23, [%rd1];
+; SM70-FTZ-ALLOW-NEXT: cvt.u32.u16 %r10, %rs1;
+; SM70-FTZ-ALLOW-NEXT: shl.b32 %r11, %r10, 16;
+; SM70-FTZ-ALLOW-NEXT: $L__BB79_1: // %atomicrmw.start
+; SM70-FTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-FTZ-ALLOW-NEXT: shr.u32 %r8, %r23, %r1;
+; SM70-FTZ-ALLOW-NEXT: shl.b32 %r9, %r8, 16;
+; SM70-FTZ-ALLOW-NEXT: sub.rn.ftz.f32 %r12, %r9, %r11;
+; SM70-FTZ-ALLOW-NEXT: bfe.u32 %r13, %r12, 16, 1;
+; SM70-FTZ-ALLOW-NEXT: add.s32 %r14, %r13, %r12;
+; SM70-FTZ-ALLOW-NEXT: add.s32 %r15, %r14, 32767;
+; SM70-FTZ-ALLOW-NEXT: setp.nan.ftz.f32 %p1, %r12, %r12;
+; SM70-FTZ-ALLOW-NEXT: or.b32 %r16, %r12, 4194304;
+; SM70-FTZ-ALLOW-NEXT: selp.b32 %r17, %r16, %r15, %p1;
+; SM70-FTZ-ALLOW-NEXT: shr.u32 %r18, %r17, 16;
+; SM70-FTZ-ALLOW-NEXT: shl.b32 %r19, %r18, %r1;
+; SM70-FTZ-ALLOW-NEXT: and.b32 %r20, %r23, %r2;
+; SM70-FTZ-ALLOW-NEXT: or.b32 %r21, %r20, %r19;
+; SM70-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r23, %r21;
+; SM70-FTZ-ALLOW-NEXT: setp.ne.b32 %p2, %r3, %r23;
+; SM70-FTZ-ALLOW-NEXT: mov.b32 %r23, %r3;
+; SM70-FTZ-ALLOW-NEXT: @%p2 bra $L__BB79_1;
+; SM70-FTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-FTZ-ALLOW-NEXT: shr.u32 %r22, %r3, %r1;
+; SM70-FTZ-ALLOW-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-ALLOW-NEXT: st.param.b16 [func_retval0], %r22;
+; SM70-FTZ-ALLOW-NEXT: ret;
%retval = atomicrmw fsub ptr addrspace(1) %addr, bfloat %val syncscope("block") acq_rel
ret bfloat %retval
}
define bfloat @fmin_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat %val) {
-; SM70-NOFTZ-LABEL: fmin_acq_rel_bfloat_global_cta(
-; SM70-NOFTZ: {
-; SM70-NOFTZ-NEXT: .reg .pred %p<3>;
-; SM70-NOFTZ-NEXT: .reg .b16 %rs<2>;
-; SM70-NOFTZ-NEXT: .reg .b32 %r<24>;
-; SM70-NOFTZ-NEXT: .reg .b64 %rd<3>;
-; SM70-NOFTZ-EMPTY:
-; SM70-NOFTZ-NEXT: // %bb.0:
-; SM70-NOFTZ-NEXT: ld.param.b16 %rs1, [fmin_acq_rel_bfloat_global_cta_param_1];
-; SM70-NOFTZ-NEXT: ld.param.b64 %rd2, [fmin_acq_rel_bfloat_global_cta_param_0];
-; SM70-NOFTZ-NEXT: fence.acq_rel.cta;
-; SM70-NOFTZ-NEXT: and.b64 %rd1, %rd2, -4;
-; SM70-NOFTZ-NEXT: cvt.u32.u64 %r4, %rd2;
-; SM70-NOFTZ-NEXT: and.b32 %r5, %r4, 3;
-; SM70-NOFTZ-NEXT: shl.b32 %r1, %r5, 3;
-; SM70-NOFTZ-NEXT: mov.b32 %r6, 65535;
-; SM70-NOFTZ-NEXT: shl.b32 %r7, %r6, %r1;
-; SM70-NOFTZ-NEXT: not.b32 %r2, %r7;
-; SM70-NOFTZ-NEXT: ld.relaxed.cta.global.b32 %r23, [%rd1];
-; SM70-NOFTZ-NEXT: cvt.u32.u16 %r10, %rs1;
-; SM70-NOFTZ-NEXT: shl.b32 %r11, %r10, 16;
-; SM70-NOFTZ-NEXT: $L__BB80_1: // %atomicrmw.start
-; SM70-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NOFTZ-NEXT: shr.u32 %r8, %r23, %r1;
-; SM70-NOFTZ-NEXT: shl.b32 %r9, %r8, 16;
-; SM70-NOFTZ-NEXT: min.f32 %r12, %r9, %r11;
-; SM70-NOFTZ-NEXT: bfe.u32 %r13, %r12, 16, 1;
-; SM70-NOFTZ-NEXT: add.s32 %r14, %r13, %r12;
-; SM70-NOFTZ-NEXT: add.s32 %r15, %r14, 32767;
-; SM70-NOFTZ-NEXT: setp.nan.f32 %p1, %r12, %r12;
-; SM70-NOFTZ-NEXT: or.b32 %r16, %r12, 4194304;
-; SM70-NOFTZ-NEXT: selp.b32 %r17, %r16, %r15, %p1;
-; SM70-NOFTZ-NEXT: shr.u32 %r18, %r17, 16;
-; SM70-NOFTZ-NEXT: shl.b32 %r19, %r18, %r1;
-; SM70-NOFTZ-NEXT: and.b32 %r20, %r23, %r2;
-; SM70-NOFTZ-NEXT: or.b32 %r21, %r20, %r19;
-; SM70-NOFTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r23, %r21;
-; SM70-NOFTZ-NEXT: setp.ne.b32 %p2, %r3, %r23;
-; SM70-NOFTZ-NEXT: mov.b32 %r23, %r3;
-; SM70-NOFTZ-NEXT: @%p2 bra $L__BB80_1;
-; SM70-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end
-; SM70-NOFTZ-NEXT: shr.u32 %r22, %r3, %r1;
-; SM70-NOFTZ-NEXT: fence.acq_rel.cta;
-; SM70-NOFTZ-NEXT: st.param.b16 [func_retval0], %r22;
-; SM70-NOFTZ-NEXT: ret;
+; SM70-NOFTZ-DISALLOW-LABEL: fmin_acq_rel_bfloat_global_cta(
+; SM70-NOFTZ-DISALLOW: {
+; SM70-NOFTZ-DISALLOW-NEXT: .reg .pred %p<3>;
+; SM70-NOFTZ-DISALLOW-NEXT: .reg .b16 %rs<2>;
+; SM70-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<24>;
+; SM70-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
+; SM70-NOFTZ-DISALLOW-EMPTY:
+; SM70-NOFTZ-DISALLOW-NEXT: // %bb.0:
+; SM70-NOFTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fmin_acq_rel_bfloat_global_cta_param_1];
+; SM70-NOFTZ-DISALLOW-NEXT: ld.param.b64 %rd2, [fmin_acq_rel_bfloat_global_cta_param_0];
+; SM70-NOFTZ-DISALLOW-NEXT: fence.acq_rel.cta;
+; SM70-NOFTZ-DISALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-NOFTZ-DISALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM70-NOFTZ-DISALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-NOFTZ-DISALLOW-NEXT: mov.b32 %r6, 65535;
+; SM70-NOFTZ-DISALLOW-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-NOFTZ-DISALLOW-NEXT: not.b32 %r2, %r7;
+; SM70-NOFTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b32 %r23, [%rd1];
+; SM70-NOFTZ-DISALLOW-NEXT: cvt.u32.u16 %r10, %rs1;
+; SM70-NOFTZ-DISALLOW-NEXT: shl.b32 %r11, %r10, 16;
+; SM70-NOFTZ-DISALLOW-NEXT: $L__BB80_1: // %atomicrmw.start
+; SM70-NOFTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NOFTZ-DISALLOW-NEXT: shr.u32 %r8, %r23, %r1;
+; SM70-NOFTZ-DISALLOW-NEXT: shl.b32 %r9, %r8, 16;
+; SM70-NOFTZ-DISALLOW-NEXT: min.f32 %r12, %r9, %r11;
+; SM70-NOFTZ-DISALLOW-NEXT: bfe.u32 %r13, %r12, 16, 1;
+; SM70-NOFTZ-DISALLOW-NEXT: add.s32 %r14, %r13, %r12;
+; SM70-NOFTZ-DISALLOW-NEXT: add.s32 %r15, %r14, 32767;
+; SM70-NOFTZ-DISALLOW-NEXT: setp.nan.f32 %p1, %r12, %r12;
+; SM70-NOFTZ-DISALLOW-NEXT: or.b32 %r16, %r12, 4194304;
+; SM70-NOFTZ-DISALLOW-NEXT: selp.b32 %r17, %r16, %r15, %p1;
+; SM70-NOFTZ-DISALLOW-NEXT: shr.u32 %r18, %r17, 16;
+; SM70-NOFTZ-DISALLOW-NEXT: shl.b32 %r19, %r18, %r1;
+; SM70-NOFTZ-DISALLOW-NEXT: and.b32 %r20, %r23, %r2;
+; SM70-NOFTZ-DISALLOW-NEXT: or.b32 %r21, %r20, %r19;
+; SM70-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r23, %r21;
+; SM70-NOFTZ-DISALLOW-NEXT: setp.ne.b32 %p2, %r3, %r23;
+; SM70-NOFTZ-DISALLOW-NEXT: mov.b32 %r23, %r3;
+; SM70-NOFTZ-DISALLOW-NEXT: @%p2 bra $L__BB80_1;
+; SM70-NOFTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NOFTZ-DISALLOW-NEXT: shr.u32 %r22, %r3, %r1;
+; SM70-NOFTZ-DISALLOW-NEXT: fence.acq_rel.cta;
+; SM70-NOFTZ-DISALLOW-NEXT: st.param.b16 [func_retval0], %r22;
+; SM70-NOFTZ-DISALLOW-NEXT: ret;
+;
+; SM70-NOFTZ-ALLOW-LABEL: fmin_acq_rel_bfloat_global_cta(
+; SM70-NOFTZ-ALLOW: {
+; SM70-NOFTZ-ALLOW-NEXT: .reg .pred %p<3>;
+; SM70-NOFTZ-ALLOW-NEXT: .reg .b16 %rs<2>;
+; SM70-NOFTZ-ALLOW-NEXT: .reg .b32 %r<24>;
+; SM70-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
+; SM70-NOFTZ-ALLOW-EMPTY:
+; SM70-NOFTZ-ALLOW-NEXT: // %bb.0:
+; SM70-NOFTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fmin_acq_rel_bfloat_global_cta_param_1];
+; SM70-NOFTZ-ALLOW-NEXT: ld.param.b64 %rd2, [fmin_acq_rel_bfloat_global_cta_param_0];
+; SM70-NOFTZ-ALLOW-NEXT: fence.acq_rel.cta;
+; SM70-NOFTZ-ALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NOFTZ-ALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-NOFTZ-ALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM70-NOFTZ-ALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-NOFTZ-ALLOW-NEXT: mov.b32 %r6, 65535;
+; SM70-NOFTZ-ALLOW-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-NOFTZ-ALLOW-NEXT: not.b32 %r2, %r7;
+; SM70-NOFTZ-ALLOW-NEXT: ld.relaxed.cta.global.b32 %r23, [%rd1];
+; SM70-NOFTZ-ALLOW-NEXT: cvt.u32.u16 %r10, %rs1;
+; SM70-NOFTZ-ALLOW-NEXT: shl.b32 %r11, %r10, 16;
+; SM70-NOFTZ-ALLOW-NEXT: $L__BB80_1: // %atomicrmw.start
+; SM70-NOFTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NOFTZ-ALLOW-NEXT: shr.u32 %r8, %r23, %r1;
+; SM70-NOFTZ-ALLOW-NEXT: shl.b32 %r9, %r8, 16;
+; SM70-NOFTZ-ALLOW-NEXT: min.f32 %r12, %r9, %r11;
+; SM70-NOFTZ-ALLOW-NEXT: bfe.u32 %r13, %r12, 16, 1;
+; SM70-NOFTZ-ALLOW-NEXT: add.s32 %r14, %r13, %r12;
+; SM70-NOFTZ-ALLOW-NEXT: add.s32 %r15, %r14, 32767;
+; SM70-NOFTZ-ALLOW-NEXT: setp.nan.f32 %p1, %r12, %r12;
+; SM70-NOFTZ-ALLOW-NEXT: or.b32 %r16, %r12, 4194304;
+; SM70-NOFTZ-ALLOW-NEXT: selp.b32 %r17, %r16, %r15, %p1;
+; SM70-NOFTZ-ALLOW-NEXT: shr.u32 %r18, %r17, 16;
+; SM70-NOFTZ-ALLOW-NEXT: shl.b32 %r19, %r18, %r1;
+; SM70-NOFTZ-ALLOW-NEXT: and.b32 %r20, %r23, %r2;
+; SM70-NOFTZ-ALLOW-NEXT: or.b32 %r21, %r20, %r19;
+; SM70-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r23, %r21;
+; SM70-NOFTZ-ALLOW-NEXT: setp.ne.b32 %p2, %r3, %r23;
+; SM70-NOFTZ-ALLOW-NEXT: mov.b32 %r23, %r3;
+; SM70-NOFTZ-ALLOW-NEXT: @%p2 bra $L__BB80_1;
+; SM70-NOFTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NOFTZ-ALLOW-NEXT: shr.u32 %r22, %r3, %r1;
+; SM70-NOFTZ-ALLOW-NEXT: fence.acq_rel.cta;
+; SM70-NOFTZ-ALLOW-NEXT: st.param.b16 [func_retval0], %r22;
+; SM70-NOFTZ-ALLOW-NEXT: ret;
;
-; SM70-FTZ-LABEL: fmin_acq_rel_bfloat_global_cta(
-; SM70-FTZ: {
-; SM70-FTZ-NEXT: .reg .pred %p<3>;
-; SM70-FTZ-NEXT: .reg .b16 %rs<2>;
-; SM70-FTZ-NEXT: .reg .b32 %r<24>;
-; SM70-FTZ-NEXT: .reg .b64 %rd<3>;
-; SM70-FTZ-EMPTY:
-; SM70-FTZ-NEXT: // %bb.0:
-; SM70-FTZ-NEXT: ld.param.b16 %rs1, [fmin_acq_rel_bfloat_global_cta_param_1];
-; SM70-FTZ-NEXT: ld.param.b64 %rd2, [fmin_acq_rel_bfloat_global_cta_param_0];
-; SM70-FTZ-NEXT: fence.acq_rel.cta;
-; SM70-FTZ-NEXT: and.b64 %rd1, %rd2, -4;
-; SM70-FTZ-NEXT: cvt.u32.u64 %r4, %rd2;
-; SM70-FTZ-NEXT: and.b32 %r5, %r4, 3;
-; SM70-FTZ-NEXT: shl.b32 %r1, %r5, 3;
-; SM70-FTZ-NEXT: mov.b32 %r6, 65535;
-; SM70-FTZ-NEXT: shl.b32 %r7, %r6, %r1;
-; SM70-FTZ-NEXT: not.b32 %r2, %r7;
-; SM70-FTZ-NEXT: ld.relaxed.cta.global.b32 %r23, [%rd1];
-; SM70-FTZ-NEXT: cvt.u32.u16 %r10, %rs1;
-; SM70-FTZ-NEXT: shl.b32 %r11, %r10, 16;
-; SM70-FTZ-NEXT: $L__BB80_1: // %atomicrmw.start
-; SM70-FTZ-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-FTZ-NEXT: shr.u32 %r8, %r23, %r1;
-; SM70-FTZ-NEXT: shl.b32 %r9, %r8, 16;
-; SM70-FTZ-NEXT: min.ftz.f32 %r12, %r9, %r11;
-; SM70-FTZ-NEXT: bfe.u32 %r13, %r12, 16, 1;
-; SM70-FTZ-NEXT: add.s32 %r14, %r13, %r12;
-; SM70-FTZ-NEXT: add.s32 %r15, %r14, 32767;
-; SM70-FTZ-NEXT: setp.nan.ftz.f32 %p1, %r12, %r12;
-; SM70-FTZ-NEXT: or.b32 %r16, %r12, 4194304;
-; SM70-FTZ-NEXT: selp.b32 %r17, %r16, %r15, %p1;
-; SM70-FTZ-NEXT: shr.u32 %r18, %r17, 16;
-; SM70-FTZ-NEXT: shl.b32 %r19, %r18, %r1;
-; SM70-FTZ-NEXT: and.b32 %r20, %r23, %r2;
-; SM70-FTZ-NEXT: or.b32 %r21, %r20, %r19;
-; SM70-FTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r23, %r21;
-; SM70-FTZ-NEXT: setp.ne.b32 %p2, %r3, %r23;
-; SM70-FTZ-NEXT: mov.b32 %r23, %r3;
-; SM70-FTZ-NEXT: @%p2 bra $L__BB80_1;
-; SM70-FTZ-NEXT: // %bb.2: // %atomicrmw.end
-; SM70-FTZ-NEXT: shr.u32 %r22, %r3, %r1;
-; SM70-FTZ-NEXT: fence.acq_rel.cta;
-; SM70-FTZ-NEXT: st.param.b16 [func_retval0], %r22;
-; SM70-FTZ-NEXT: ret;
+; SM70-FTZ-DISALLOW-LABEL: fmin_acq_rel_bfloat_global_cta(
+; SM70-FTZ-DISALLOW: {
+; SM70-FTZ-DISALLOW-NEXT: .reg .pred %p<3>;
+; SM70-FTZ-DISALLOW-NEXT: .reg .b16 %rs<2>;
+; SM70-FTZ-DISALLOW-NEXT: .reg .b32 %r<24>;
+; SM70-FTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
+; SM70-FTZ-DISALLOW-EMPTY:
+; SM70-FTZ-DISALLOW-NEXT: // %bb.0:
+; SM70-FTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fmin_acq_rel_bfloat_global_cta_param_1];
+; SM70-FTZ-DISALLOW-NEXT: ld.param.b64 %rd2, [fmin_acq_rel_bfloat_global_cta_param_0];
+; SM70-FTZ-DISALLOW-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-DISALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-FTZ-DISALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-FTZ-DISALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM70-FTZ-DISALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-FTZ-DISALLOW-NEXT: mov.b32 %r6, 65535;
+; SM70-FTZ-DISALLOW-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-FTZ-DISALLOW-NEXT: not.b32 %r2, %r7;
+; SM70-FTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b32 %r23, [%rd1];
+; SM70-FTZ-DISALLOW-NEXT: cvt.u32.u16 %r10, %rs1;
+; SM70-FTZ-DISALLOW-NEXT: shl.b32 %r11, %r10, 16;
+; SM70-FTZ-DISALLOW-NEXT: $L__BB80_1: // %atomicrmw.start
+; SM70-FTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-FTZ-DISALLOW-NEXT: shr.u32 %r8, %r23, %r1;
+; SM70-FTZ-DISALLOW-NEXT: shl.b32 %r9, %r8, 16;
+; SM70-FTZ-DISALLOW-NEXT: min.ftz.f32 %r12, %r9, %r11;
+; SM70-FTZ-DISALLOW-NEXT: bfe.u32 %r13, %r12, 16, 1;
+; SM70-FTZ-DISALLOW-NEXT: add.s32 %r14, %r13, %r12;
+; SM70-FTZ-DISALLOW-NEXT: add.s32 %r15, %r14, 32767;
+; SM70-FTZ-DISALLOW-NEXT: setp.nan.ftz.f32 %p1, %r12, %r12;
+; SM70-FTZ-DISALLOW-NEXT: or.b32 %r16, %r12, 4194304;
+; SM70-FTZ-DISALLOW-NEXT: selp.b32 %r17, %r16, %r15, %p1;
+; SM70-FTZ-DISALLOW-NEXT: shr.u32 %r18, %r17, 16;
+; SM70-FTZ-DISALLOW-NEXT: shl.b32 %r19, %r18, %r1;
+; SM70-FTZ-DISALLOW-NEXT: and.b32 %r20, %r23, %r2;
+; SM70-FTZ-DISALLOW-NEXT: or.b32 %r21, %r20, %r19;
+; SM70-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r23, %r21;
+; SM70-FTZ-DISALLOW-NEXT: setp.ne.b32 %p2, %r3, %r23;
+; SM70-FTZ-DISALLOW-NEXT: mov.b32 %r23, %r3;
+; SM70-FTZ-DISALLOW-NEXT: @%p2 bra $L__BB80_1;
+; SM70-FTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-FTZ-DISALLOW-NEXT: shr.u32 %r22, %r3, %r1;
+; SM70-FTZ-DISALLOW-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-DISALLOW-NEXT: st.param.b16 [func_retval0], %r22;
+; SM70-FTZ-DISALLOW-NEXT: ret;
+;
+; SM70-FTZ-ALLOW-LABEL: fmin_acq_rel_bfloat_global_cta(
+; SM70-FTZ-ALLOW: {
+; SM70-FTZ-ALLOW-NEXT: .reg .pred %p<3>;
+; SM70-FTZ-ALLOW-NEXT: .reg .b16 %rs<2>;
+; SM70-FTZ-ALLOW-NEXT: .reg .b32 %r<24>;
+; SM70-FTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
+; SM70-FTZ-ALLOW-EMPTY:
+; SM70-FTZ-ALLOW-NEXT: // %bb.0:
+; SM70-FTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fmin_acq_rel_bfloat_global_cta_param_1];
+; SM70-FTZ-ALLOW-NEXT: ld.param.b64 %rd2, [fmin_acq_rel_bfloat_global_cta_param_0];
+; SM70-FTZ-ALLOW-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-ALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-FTZ-ALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-FTZ-ALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM70-FTZ-ALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-FTZ-ALLOW-NEXT: mov.b32 %r6, 65535;
+; SM70-FTZ-ALLOW-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-FTZ-ALLOW-NEXT: not.b32 %r2, %r7;
+; SM70-FTZ-ALLOW-NEXT: ld.relaxed.cta.global.b32 %r23, [%rd1];
+; SM70-FTZ-ALLOW-NEXT: cvt.u32.u16 %r10, %rs1;
+; SM70-FTZ-ALLOW-NEXT: shl.b32 %r11, %r10, 16;
+; SM70-FTZ-ALLOW-NEXT: $L__BB80_1: // %atomicrmw.start
+; SM70-FTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-FTZ-ALLOW-NEXT: shr.u32 %r8, %r23, %r1;
+; SM70-FTZ-ALLOW-NEXT: shl.b32 %r9, %r8, 16;
+; SM70-FTZ-ALLOW-NEXT: min.ftz.f32 %r12, %r9, %r11;
+; SM70-FTZ-ALLOW-NEXT: bfe.u32 %r13, %r12, 16, 1;
+; SM70-FTZ-ALLOW-NEXT: add.s32 %r14, %r13, %r12;
+; SM70-FTZ-ALLOW-NEXT: add.s32 %r15, %r14, 32767;
+; SM70-FTZ-ALLOW-NEXT: setp.nan.ftz.f32 %p1, %r12, %r12;
+; SM70-FTZ-ALLOW-NEXT: or.b32 %r16, %r12, 4194304;
+; SM70-FTZ-ALLOW-NEXT: selp.b32 %r17, %r16, %r15, %p1;
+; SM70-FTZ-ALLOW-NEXT: shr.u32 %r18, %r17, 16;
+; SM70-FTZ-ALLOW-NEXT: shl.b32 %r19, %r18, %r1;
+; SM70-FTZ-ALLOW-NEXT: and.b32 %r20, %r23, %r2;
+; SM70-FTZ-ALLOW-NEXT: or.b32 %r21, %r20, %r19;
+; SM70-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r23, %r21;
+; SM70-FTZ-ALLOW-NEXT: setp.ne.b32 %p2, %r3, %r23;
+; SM70-FTZ-ALLOW-NEXT: mov.b32 %r23, %r3;
+; SM70-FTZ-ALLOW-NEXT: @%p2 bra $L__BB80_1;
+; SM70-FTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-FTZ-ALLOW-NEXT: shr.u32 %r22, %r3, %r1;
+; SM70-FTZ-ALLOW-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-ALLOW-NEXT: st.param.b16 [func_retval0], %r22;
+; SM70-FTZ-ALLOW-NEXT: ret;
%retval = atomicrmw fmin ptr addrspace(1) %addr, bfloat %val syncscope("block") acq_rel
ret bfloat %retval
}
define bfloat @fmax_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat %val) {
-; SM70-NOFTZ-LABEL: fmax_acq_rel_bfloat_global_cta(
-; SM70-NOFTZ: {
-; SM70-NOFTZ-NEXT: .reg .pred %p<3>;
-; SM70-NOFTZ-NEXT: .reg .b16 %rs<2>;
-; SM70-NOFTZ-NEXT: .reg .b32 %r<24>;
-; SM70-NOFTZ-NEXT: .reg .b64 %rd<3>;
-; SM70-NOFTZ-EMPTY:
-; SM70-NOFTZ-NEXT: // %bb.0:
-; SM70-NOFTZ-NEXT: ld.param.b16 %rs1, [fmax_acq_rel_bfloat_global_cta_param_1];
-; SM70-NOFTZ-NEXT: ld.param.b64 %rd2, [fmax_acq_rel_bfloat_global_cta_param_0];
-; SM70-NOFTZ-NEXT: fence.acq_rel.cta;
-; SM70-NOFTZ-NEXT: and.b64 %rd1, %rd2, -4;
-; SM70-NOFTZ-NEXT: cvt.u32.u64 %r4, %rd2;
-; SM70-NOFTZ-NEXT: and.b32 %r5, %r4, 3;
-; SM70-NOFTZ-NEXT: shl.b32 %r1, %r5, 3;
-; SM70-NOFTZ-NEXT: mov.b32 %r6, 65535;
-; SM70-NOFTZ-NEXT: shl.b32 %r7, %r6, %r1;
-; SM70-NOFTZ-NEXT: not.b32 %r2, %r7;
-; SM70-NOFTZ-NEXT: ld.relaxed.cta.global.b32 %r23, [%rd1];
-; SM70-NOFTZ-NEXT: cvt.u32.u16 %r10, %rs1;
-; SM70-NOFTZ-NEXT: shl.b32 %r11, %r10, 16;
-; SM70-NOFTZ-NEXT: $L__BB81_1: // %atomicrmw.start
-; SM70-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NOFTZ-NEXT: shr.u32 %r8, %r23, %r1;
-; SM70-NOFTZ-NEXT: shl.b32 %r9, %r8, 16;
-; SM70-NOFTZ-NEXT: max.f32 %r12, %r9, %r11;
-; SM70-NOFTZ-NEXT: bfe.u32 %r13, %r12, 16, 1;
-; SM70-NOFTZ-NEXT: add.s32 %r14, %r13, %r12;
-; SM70-NOFTZ-NEXT: add.s32 %r15, %r14, 32767;
-; SM70-NOFTZ-NEXT: setp.nan.f32 %p1, %r12, %r12;
-; SM70-NOFTZ-NEXT: or.b32 %r16, %r12, 4194304;
-; SM70-NOFTZ-NEXT: selp.b32 %r17, %r16, %r15, %p1;
-; SM70-NOFTZ-NEXT: shr.u32 %r18, %r17, 16;
-; SM70-NOFTZ-NEXT: shl.b32 %r19, %r18, %r1;
-; SM70-NOFTZ-NEXT: and.b32 %r20, %r23, %r2;
-; SM70-NOFTZ-NEXT: or.b32 %r21, %r20, %r19;
-; SM70-NOFTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r23, %r21;
-; SM70-NOFTZ-NEXT: setp.ne.b32 %p2, %r3, %r23;
-; SM70-NOFTZ-NEXT: mov.b32 %r23, %r3;
-; SM70-NOFTZ-NEXT: @%p2 bra $L__BB81_1;
-; SM70-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end
-; SM70-NOFTZ-NEXT: shr.u32 %r22, %r3, %r1;
-; SM70-NOFTZ-NEXT: fence.acq_rel.cta;
-; SM70-NOFTZ-NEXT: st.param.b16 [func_retval0], %r22;
-; SM70-NOFTZ-NEXT: ret;
+; SM70-NOFTZ-DISALLOW-LABEL: fmax_acq_rel_bfloat_global_cta(
+; SM70-NOFTZ-DISALLOW: {
+; SM70-NOFTZ-DISALLOW-NEXT: .reg .pred %p<3>;
+; SM70-NOFTZ-DISALLOW-NEXT: .reg .b16 %rs<2>;
+; SM70-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<24>;
+; SM70-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
+; SM70-NOFTZ-DISALLOW-EMPTY:
+; SM70-NOFTZ-DISALLOW-NEXT: // %bb.0:
+; SM70-NOFTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fmax_acq_rel_bfloat_global_cta_param_1];
+; SM70-NOFTZ-DISALLOW-NEXT: ld.param.b64 %rd2, [fmax_acq_rel_bfloat_global_cta_param_0];
+; SM70-NOFTZ-DISALLOW-NEXT: fence.acq_rel.cta;
+; SM70-NOFTZ-DISALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-NOFTZ-DISALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM70-NOFTZ-DISALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-NOFTZ-DISALLOW-NEXT: mov.b32 %r6, 65535;
+; SM70-NOFTZ-DISALLOW-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-NOFTZ-DISALLOW-NEXT: not.b32 %r2, %r7;
+; SM70-NOFTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b32 %r23, [%rd1];
+; SM70-NOFTZ-DISALLOW-NEXT: cvt.u32.u16 %r10, %rs1;
+; SM70-NOFTZ-DISALLOW-NEXT: shl.b32 %r11, %r10, 16;
+; SM70-NOFTZ-DISALLOW-NEXT: $L__BB81_1: // %atomicrmw.start
+; SM70-NOFTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NOFTZ-DISALLOW-NEXT: shr.u32 %r8, %r23, %r1;
+; SM70-NOFTZ-DISALLOW-NEXT: shl.b32 %r9, %r8, 16;
+; SM70-NOFTZ-DISALLOW-NEXT: max.f32 %r12, %r9, %r11;
+; SM70-NOFTZ-DISALLOW-NEXT: bfe.u32 %r13, %r12, 16, 1;
+; SM70-NOFTZ-DISALLOW-NEXT: add.s32 %r14, %r13, %r12;
+; SM70-NOFTZ-DISALLOW-NEXT: add.s32 %r15, %r14, 32767;
+; SM70-NOFTZ-DISALLOW-NEXT: setp.nan.f32 %p1, %r12, %r12;
+; SM70-NOFTZ-DISALLOW-NEXT: or.b32 %r16, %r12, 4194304;
+; SM70-NOFTZ-DISALLOW-NEXT: selp.b32 %r17, %r16, %r15, %p1;
+; SM70-NOFTZ-DISALLOW-NEXT: shr.u32 %r18, %r17, 16;
+; SM70-NOFTZ-DISALLOW-NEXT: shl.b32 %r19, %r18, %r1;
+; SM70-NOFTZ-DISALLOW-NEXT: and.b32 %r20, %r23, %r2;
+; SM70-NOFTZ-DISALLOW-NEXT: or.b32 %r21, %r20, %r19;
+; SM70-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r23, %r21;
+; SM70-NOFTZ-DISALLOW-NEXT: setp.ne.b32 %p2, %r3, %r23;
+; SM70-NOFTZ-DISALLOW-NEXT: mov.b32 %r23, %r3;
+; SM70-NOFTZ-DISALLOW-NEXT: @%p2 bra $L__BB81_1;
+; SM70-NOFTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NOFTZ-DISALLOW-NEXT: shr.u32 %r22, %r3, %r1;
+; SM70-NOFTZ-DISALLOW-NEXT: fence.acq_rel.cta;
+; SM70-NOFTZ-DISALLOW-NEXT: st.param.b16 [func_retval0], %r22;
+; SM70-NOFTZ-DISALLOW-NEXT: ret;
+;
+; SM70-NOFTZ-ALLOW-LABEL: fmax_acq_rel_bfloat_global_cta(
+; SM70-NOFTZ-ALLOW: {
+; SM70-NOFTZ-ALLOW-NEXT: .reg .pred %p<3>;
+; SM70-NOFTZ-ALLOW-NEXT: .reg .b16 %rs<2>;
+; SM70-NOFTZ-ALLOW-NEXT: .reg .b32 %r<24>;
+; SM70-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
+; SM70-NOFTZ-ALLOW-EMPTY:
+; SM70-NOFTZ-ALLOW-NEXT: // %bb.0:
+; SM70-NOFTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fmax_acq_rel_bfloat_global_cta_param_1];
+; SM70-NOFTZ-ALLOW-NEXT: ld.param.b64 %rd2, [fmax_acq_rel_bfloat_global_cta_param_0];
+; SM70-NOFTZ-ALLOW-NEXT: fence.acq_rel.cta;
+; SM70-NOFTZ-ALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NOFTZ-ALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-NOFTZ-ALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM70-NOFTZ-ALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-NOFTZ-ALLOW-NEXT: mov.b32 %r6, 65535;
+; SM70-NOFTZ-ALLOW-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-NOFTZ-ALLOW-NEXT: not.b32 %r2, %r7;
+; SM70-NOFTZ-ALLOW-NEXT: ld.relaxed.cta.global.b32 %r23, [%rd1];
+; SM70-NOFTZ-ALLOW-NEXT: cvt.u32.u16 %r10, %rs1;
+; SM70-NOFTZ-ALLOW-NEXT: shl.b32 %r11, %r10, 16;
+; SM70-NOFTZ-ALLOW-NEXT: $L__BB81_1: // %atomicrmw.start
+; SM70-NOFTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NOFTZ-ALLOW-NEXT: shr.u32 %r8, %r23, %r1;
+; SM70-NOFTZ-ALLOW-NEXT: shl.b32 %r9, %r8, 16;
+; SM70-NOFTZ-ALLOW-NEXT: max.f32 %r12, %r9, %r11;
+; SM70-NOFTZ-ALLOW-NEXT: bfe.u32 %r13, %r12, 16, 1;
+; SM70-NOFTZ-ALLOW-NEXT: add.s32 %r14, %r13, %r12;
+; SM70-NOFTZ-ALLOW-NEXT: add.s32 %r15, %r14, 32767;
+; SM70-NOFTZ-ALLOW-NEXT: setp.nan.f32 %p1, %r12, %r12;
+; SM70-NOFTZ-ALLOW-NEXT: or.b32 %r16, %r12, 4194304;
+; SM70-NOFTZ-ALLOW-NEXT: selp.b32 %r17, %r16, %r15, %p1;
+; SM70-NOFTZ-ALLOW-NEXT: shr.u32 %r18, %r17, 16;
+; SM70-NOFTZ-ALLOW-NEXT: shl.b32 %r19, %r18, %r1;
+; SM70-NOFTZ-ALLOW-NEXT: and.b32 %r20, %r23, %r2;
+; SM70-NOFTZ-ALLOW-NEXT: or.b32 %r21, %r20, %r19;
+; SM70-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r23, %r21;
+; SM70-NOFTZ-ALLOW-NEXT: setp.ne.b32 %p2, %r3, %r23;
+; SM70-NOFTZ-ALLOW-NEXT: mov.b32 %r23, %r3;
+; SM70-NOFTZ-ALLOW-NEXT: @%p2 bra $L__BB81_1;
+; SM70-NOFTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NOFTZ-ALLOW-NEXT: shr.u32 %r22, %r3, %r1;
+; SM70-NOFTZ-ALLOW-NEXT: fence.acq_rel.cta;
+; SM70-NOFTZ-ALLOW-NEXT: st.param.b16 [func_retval0], %r22;
+; SM70-NOFTZ-ALLOW-NEXT: ret;
+;
+; SM70-FTZ-DISALLOW-LABEL: fmax_acq_rel_bfloat_global_cta(
+; SM70-FTZ-DISALLOW: {
+; SM70-FTZ-DISALLOW-NEXT: .reg .pred %p<3>;
+; SM70-FTZ-DISALLOW-NEXT: .reg .b16 %rs<2>;
+; SM70-FTZ-DISALLOW-NEXT: .reg .b32 %r<24>;
+; SM70-FTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
+; SM70-FTZ-DISALLOW-EMPTY:
+; SM70-FTZ-DISALLOW-NEXT: // %bb.0:
+; SM70-FTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fmax_acq_rel_bfloat_global_cta_param_1];
+; SM70-FTZ-DISALLOW-NEXT: ld.param.b64 %rd2, [fmax_acq_rel_bfloat_global_cta_param_0];
+; SM70-FTZ-DISALLOW-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-DISALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-FTZ-DISALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-FTZ-DISALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM70-FTZ-DISALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-FTZ-DISALLOW-NEXT: mov.b32 %r6, 65535;
+; SM70-FTZ-DISALLOW-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-FTZ-DISALLOW-NEXT: not.b32 %r2, %r7;
+; SM70-FTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b32 %r23, [%rd1];
+; SM70-FTZ-DISALLOW-NEXT: cvt.u32.u16 %r10, %rs1;
+; SM70-FTZ-DISALLOW-NEXT: shl.b32 %r11, %r10, 16;
+; SM70-FTZ-DISALLOW-NEXT: $L__BB81_1: // %atomicrmw.start
+; SM70-FTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-FTZ-DISALLOW-NEXT: shr.u32 %r8, %r23, %r1;
+; SM70-FTZ-DISALLOW-NEXT: shl.b32 %r9, %r8, 16;
+; SM70-FTZ-DISALLOW-NEXT: max.ftz.f32 %r12, %r9, %r11;
+; SM70-FTZ-DISALLOW-NEXT: bfe.u32 %r13, %r12, 16, 1;
+; SM70-FTZ-DISALLOW-NEXT: add.s32 %r14, %r13, %r12;
+; SM70-FTZ-DISALLOW-NEXT: add.s32 %r15, %r14, 32767;
+; SM70-FTZ-DISALLOW-NEXT: setp.nan.ftz.f32 %p1, %r12, %r12;
+; SM70-FTZ-DISALLOW-NEXT: or.b32 %r16, %r12, 4194304;
+; SM70-FTZ-DISALLOW-NEXT: selp.b32 %r17, %r16, %r15, %p1;
+; SM70-FTZ-DISALLOW-NEXT: shr.u32 %r18, %r17, 16;
+; SM70-FTZ-DISALLOW-NEXT: shl.b32 %r19, %r18, %r1;
+; SM70-FTZ-DISALLOW-NEXT: and.b32 %r20, %r23, %r2;
+; SM70-FTZ-DISALLOW-NEXT: or.b32 %r21, %r20, %r19;
+; SM70-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r23, %r21;
+; SM70-FTZ-DISALLOW-NEXT: setp.ne.b32 %p2, %r3, %r23;
+; SM70-FTZ-DISALLOW-NEXT: mov.b32 %r23, %r3;
+; SM70-FTZ-DISALLOW-NEXT: @%p2 bra $L__BB81_1;
+; SM70-FTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-FTZ-DISALLOW-NEXT: shr.u32 %r22, %r3, %r1;
+; SM70-FTZ-DISALLOW-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-DISALLOW-NEXT: st.param.b16 [func_retval0], %r22;
+; SM70-FTZ-DISALLOW-NEXT: ret;
;
-; SM70-FTZ-LABEL: fmax_acq_rel_bfloat_global_cta(
-; SM70-FTZ: {
-; SM70-FTZ-NEXT: .reg .pred %p<3>;
-; SM70-FTZ-NEXT: .reg .b16 %rs<2>;
-; SM70-FTZ-NEXT: .reg .b32 %r<24>;
-; SM70-FTZ-NEXT: .reg .b64 %rd<3>;
-; SM70-FTZ-EMPTY:
-; SM70-FTZ-NEXT: // %bb.0:
-; SM70-FTZ-NEXT: ld.param.b16 %rs1, [fmax_acq_rel_bfloat_global_cta_param_1];
-; SM70-FTZ-NEXT: ld.param.b64 %rd2, [fmax_acq_rel_bfloat_global_cta_param_0];
-; SM70-FTZ-NEXT: fence.acq_rel.cta;
-; SM70-FTZ-NEXT: and.b64 %rd1, %rd2, -4;
-; SM70-FTZ-NEXT: cvt.u32.u64 %r4, %rd2;
-; SM70-FTZ-NEXT: and.b32 %r5, %r4, 3;
-; SM70-FTZ-NEXT: shl.b32 %r1, %r5, 3;
-; SM70-FTZ-NEXT: mov.b32 %r6, 65535;
-; SM70-FTZ-NEXT: shl.b32 %r7, %r6, %r1;
-; SM70-FTZ-NEXT: not.b32 %r2, %r7;
-; SM70-FTZ-NEXT: ld.relaxed.cta.global.b32 %r23, [%rd1];
-; SM70-FTZ-NEXT: cvt.u32.u16 %r10, %rs1;
-; SM70-FTZ-NEXT: shl.b32 %r11, %r10, 16;
-; SM70-FTZ-NEXT: $L__BB81_1: // %atomicrmw.start
-; SM70-FTZ-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-FTZ-NEXT: shr.u32 %r8, %r23, %r1;
-; SM70-FTZ-NEXT: shl.b32 %r9, %r8, 16;
-; SM70-FTZ-NEXT: max.ftz.f32 %r12, %r9, %r11;
-; SM70-FTZ-NEXT: bfe.u32 %r13, %r12, 16, 1;
-; SM70-FTZ-NEXT: add.s32 %r14, %r13, %r12;
-; SM70-FTZ-NEXT: add.s32 %r15, %r14, 32767;
-; SM70-FTZ-NEXT: setp.nan.ftz.f32 %p1, %r12, %r12;
-; SM70-FTZ-NEXT: or.b32 %r16, %r12, 4194304;
-; SM70-FTZ-NEXT: selp.b32 %r17, %r16, %r15, %p1;
-; SM70-FTZ-NEXT: shr.u32 %r18, %r17, 16;
-; SM70-FTZ-NEXT: shl.b32 %r19, %r18, %r1;
-; SM70-FTZ-NEXT: and.b32 %r20, %r23, %r2;
-; SM70-FTZ-NEXT: or.b32 %r21, %r20, %r19;
-; SM70-FTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r23, %r21;
-; SM70-FTZ-NEXT: setp.ne.b32 %p2, %r3, %r23;
-; SM70-FTZ-NEXT: mov.b32 %r23, %r3;
-; SM70-FTZ-NEXT: @%p2 bra $L__BB81_1;
-; SM70-FTZ-NEXT: // %bb.2: // %atomicrmw.end
-; SM70-FTZ-NEXT: shr.u32 %r22, %r3, %r1;
-; SM70-FTZ-NEXT: fence.acq_rel.cta;
-; SM70-FTZ-NEXT: st.param.b16 [func_retval0], %r22;
-; SM70-FTZ-NEXT: ret;
+; SM70-FTZ-ALLOW-LABEL: fmax_acq_rel_bfloat_global_cta(
+; SM70-FTZ-ALLOW: {
+; SM70-FTZ-ALLOW-NEXT: .reg .pred %p<3>;
+; SM70-FTZ-ALLOW-NEXT: .reg .b16 %rs<2>;
+; SM70-FTZ-ALLOW-NEXT: .reg .b32 %r<24>;
+; SM70-FTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
+; SM70-FTZ-ALLOW-EMPTY:
+; SM70-FTZ-ALLOW-NEXT: // %bb.0:
+; SM70-FTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fmax_acq_rel_bfloat_global_cta_param_1];
+; SM70-FTZ-ALLOW-NEXT: ld.param.b64 %rd2, [fmax_acq_rel_bfloat_global_cta_param_0];
+; SM70-FTZ-ALLOW-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-ALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-FTZ-ALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-FTZ-ALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM70-FTZ-ALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-FTZ-ALLOW-NEXT: mov.b32 %r6, 65535;
+; SM70-FTZ-ALLOW-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-FTZ-ALLOW-NEXT: not.b32 %r2, %r7;
+; SM70-FTZ-ALLOW-NEXT: ld.relaxed.cta.global.b32 %r23, [%rd1];
+; SM70-FTZ-ALLOW-NEXT: cvt.u32.u16 %r10, %rs1;
+; SM70-FTZ-ALLOW-NEXT: shl.b32 %r11, %r10, 16;
+; SM70-FTZ-ALLOW-NEXT: $L__BB81_1: // %atomicrmw.start
+; SM70-FTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-FTZ-ALLOW-NEXT: shr.u32 %r8, %r23, %r1;
+; SM70-FTZ-ALLOW-NEXT: shl.b32 %r9, %r8, 16;
+; SM70-FTZ-ALLOW-NEXT: max.ftz.f32 %r12, %r9, %r11;
+; SM70-FTZ-ALLOW-NEXT: bfe.u32 %r13, %r12, 16, 1;
+; SM70-FTZ-ALLOW-NEXT: add.s32 %r14, %r13, %r12;
+; SM70-FTZ-ALLOW-NEXT: add.s32 %r15, %r14, 32767;
+; SM70-FTZ-ALLOW-NEXT: setp.nan.ftz.f32 %p1, %r12, %r12;
+; SM70-FTZ-ALLOW-NEXT: or.b32 %r16, %r12, 4194304;
+; SM70-FTZ-ALLOW-NEXT: selp.b32 %r17, %r16, %r15, %p1;
+; SM70-FTZ-ALLOW-NEXT: shr.u32 %r18, %r17, 16;
+; SM70-FTZ-ALLOW-NEXT: shl.b32 %r19, %r18, %r1;
+; SM70-FTZ-ALLOW-NEXT: and.b32 %r20, %r23, %r2;
+; SM70-FTZ-ALLOW-NEXT: or.b32 %r21, %r20, %r19;
+; SM70-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r23, %r21;
+; SM70-FTZ-ALLOW-NEXT: setp.ne.b32 %p2, %r3, %r23;
+; SM70-FTZ-ALLOW-NEXT: mov.b32 %r23, %r3;
+; SM70-FTZ-ALLOW-NEXT: @%p2 bra $L__BB81_1;
+; SM70-FTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-FTZ-ALLOW-NEXT: shr.u32 %r22, %r3, %r1;
+; SM70-FTZ-ALLOW-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-ALLOW-NEXT: st.param.b16 [func_retval0], %r22;
+; SM70-FTZ-ALLOW-NEXT: ret;
%retval = atomicrmw fmax ptr addrspace(1) %addr, bfloat %val syncscope("block") acq_rel
ret bfloat %retval
}
define bfloat @fminimum_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat %val) {
-; SM70-NOFTZ-LABEL: fminimum_acq_rel_bfloat_global_cta(
-; SM70-NOFTZ: {
-; SM70-NOFTZ-NEXT: .reg .pred %p<6>;
-; SM70-NOFTZ-NEXT: .reg .b16 %rs<7>;
-; SM70-NOFTZ-NEXT: .reg .b32 %r<18>;
-; SM70-NOFTZ-NEXT: .reg .b64 %rd<3>;
-; SM70-NOFTZ-EMPTY:
-; SM70-NOFTZ-NEXT: // %bb.0:
-; SM70-NOFTZ-NEXT: ld.param.b16 %rs1, [fminimum_acq_rel_bfloat_global_cta_param_1];
-; SM70-NOFTZ-NEXT: ld.param.b64 %rd2, [fminimum_acq_rel_bfloat_global_cta_param_0];
-; SM70-NOFTZ-NEXT: fence.acq_rel.cta;
-; SM70-NOFTZ-NEXT: and.b64 %rd1, %rd2, -4;
-; SM70-NOFTZ-NEXT: cvt.u32.u64 %r4, %rd2;
-; SM70-NOFTZ-NEXT: and.b32 %r5, %r4, 3;
-; SM70-NOFTZ-NEXT: shl.b32 %r1, %r5, 3;
-; SM70-NOFTZ-NEXT: mov.b32 %r6, 65535;
-; SM70-NOFTZ-NEXT: shl.b32 %r7, %r6, %r1;
-; SM70-NOFTZ-NEXT: not.b32 %r2, %r7;
-; SM70-NOFTZ-NEXT: ld.relaxed.cta.global.b32 %r17, [%rd1];
-; SM70-NOFTZ-NEXT: cvt.u32.u16 %r10, %rs1;
-; SM70-NOFTZ-NEXT: shl.b32 %r11, %r10, 16;
-; SM70-NOFTZ-NEXT: $L__BB82_1: // %atomicrmw.start
-; SM70-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NOFTZ-NEXT: shr.u32 %r8, %r17, %r1;
-; SM70-NOFTZ-NEXT: cvt.u16.u32 %rs2, %r8;
-; SM70-NOFTZ-NEXT: setp.eq.b16 %p1, %rs2, -32768;
-; SM70-NOFTZ-NEXT: selp.b16 %rs3, %rs2, %rs1, %p1;
-; SM70-NOFTZ-NEXT: shl.b32 %r9, %r8, 16;
-; SM70-NOFTZ-NEXT: setp.lt.f32 %p2, %r9, %r11;
-; SM70-NOFTZ-NEXT: selp.b16 %rs4, %rs2, %rs1, %p2;
-; SM70-NOFTZ-NEXT: setp.nan.f32 %p3, %r9, %r11;
-; SM70-NOFTZ-NEXT: selp.b16 %rs5, 0x7FC0, %rs4, %p3;
-; SM70-NOFTZ-NEXT: setp.eq.f32 %p4, %r9, %r11;
-; SM70-NOFTZ-NEXT: selp.b16 %rs6, %rs3, %rs5, %p4;
-; SM70-NOFTZ-NEXT: cvt.u32.u16 %r12, %rs6;
-; SM70-NOFTZ-NEXT: shl.b32 %r13, %r12, %r1;
-; SM70-NOFTZ-NEXT: and.b32 %r14, %r17, %r2;
-; SM70-NOFTZ-NEXT: or.b32 %r15, %r14, %r13;
-; SM70-NOFTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r17, %r15;
-; SM70-NOFTZ-NEXT: setp.ne.b32 %p5, %r3, %r17;
-; SM70-NOFTZ-NEXT: mov.b32 %r17, %r3;
-; SM70-NOFTZ-NEXT: @%p5 bra $L__BB82_1;
-; SM70-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end
-; SM70-NOFTZ-NEXT: shr.u32 %r16, %r3, %r1;
-; SM70-NOFTZ-NEXT: fence.acq_rel.cta;
-; SM70-NOFTZ-NEXT: st.param.b16 [func_retval0], %r16;
-; SM70-NOFTZ-NEXT: ret;
+; SM70-NOFTZ-DISALLOW-LABEL: fminimum_acq_rel_bfloat_global_cta(
+; SM70-NOFTZ-DISALLOW: {
+; SM70-NOFTZ-DISALLOW-NEXT: .reg .pred %p<6>;
+; SM70-NOFTZ-DISALLOW-NEXT: .reg .b16 %rs<7>;
+; SM70-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<18>;
+; SM70-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
+; SM70-NOFTZ-DISALLOW-EMPTY:
+; SM70-NOFTZ-DISALLOW-NEXT: // %bb.0:
+; SM70-NOFTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fminimum_acq_rel_bfloat_global_cta_param_1];
+; SM70-NOFTZ-DISALLOW-NEXT: ld.param.b64 %rd2, [fminimum_acq_rel_bfloat_global_cta_param_0];
+; SM70-NOFTZ-DISALLOW-NEXT: fence.acq_rel.cta;
+; SM70-NOFTZ-DISALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-NOFTZ-DISALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM70-NOFTZ-DISALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-NOFTZ-DISALLOW-NEXT: mov.b32 %r6, 65535;
+; SM70-NOFTZ-DISALLOW-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-NOFTZ-DISALLOW-NEXT: not.b32 %r2, %r7;
+; SM70-NOFTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b32 %r17, [%rd1];
+; SM70-NOFTZ-DISALLOW-NEXT: cvt.u32.u16 %r10, %rs1;
+; SM70-NOFTZ-DISALLOW-NEXT: shl.b32 %r11, %r10, 16;
+; SM70-NOFTZ-DISALLOW-NEXT: $L__BB82_1: // %atomicrmw.start
+; SM70-NOFTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NOFTZ-DISALLOW-NEXT: shr.u32 %r8, %r17, %r1;
+; SM70-NOFTZ-DISALLOW-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM70-NOFTZ-DISALLOW-NEXT: setp.eq.b16 %p1, %rs2, -32768;
+; SM70-NOFTZ-DISALLOW-NEXT: selp.b16 %rs3, %rs2, %rs1, %p1;
+; SM70-NOFTZ-DISALLOW-NEXT: shl.b32 %r9, %r8, 16;
+; SM70-NOFTZ-DISALLOW-NEXT: setp.lt.f32 %p2, %r9, %r11;
+; SM70-NOFTZ-DISALLOW-NEXT: selp.b16 %rs4, %rs2, %rs1, %p2;
+; SM70-NOFTZ-DISALLOW-NEXT: setp.nan.f32 %p3, %r9, %r11;
+; SM70-NOFTZ-DISALLOW-NEXT: selp.b16 %rs5, 0x7FC0, %rs4, %p3;
+; SM70-NOFTZ-DISALLOW-NEXT: setp.eq.f32 %p4, %r9, %r11;
+; SM70-NOFTZ-DISALLOW-NEXT: selp.b16 %rs6, %rs3, %rs5, %p4;
+; SM70-NOFTZ-DISALLOW-NEXT: cvt.u32.u16 %r12, %rs6;
+; SM70-NOFTZ-DISALLOW-NEXT: shl.b32 %r13, %r12, %r1;
+; SM70-NOFTZ-DISALLOW-NEXT: and.b32 %r14, %r17, %r2;
+; SM70-NOFTZ-DISALLOW-NEXT: or.b32 %r15, %r14, %r13;
+; SM70-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r17, %r15;
+; SM70-NOFTZ-DISALLOW-NEXT: setp.ne.b32 %p5, %r3, %r17;
+; SM70-NOFTZ-DISALLOW-NEXT: mov.b32 %r17, %r3;
+; SM70-NOFTZ-DISALLOW-NEXT: @%p5 bra $L__BB82_1;
+; SM70-NOFTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NOFTZ-DISALLOW-NEXT: shr.u32 %r16, %r3, %r1;
+; SM70-NOFTZ-DISALLOW-NEXT: fence.acq_rel.cta;
+; SM70-NOFTZ-DISALLOW-NEXT: st.param.b16 [func_retval0], %r16;
+; SM70-NOFTZ-DISALLOW-NEXT: ret;
;
-; SM70-FTZ-LABEL: fminimum_acq_rel_bfloat_global_cta(
-; SM70-FTZ: {
-; SM70-FTZ-NEXT: .reg .pred %p<6>;
-; SM70-FTZ-NEXT: .reg .b16 %rs<7>;
-; SM70-FTZ-NEXT: .reg .b32 %r<18>;
-; SM70-FTZ-NEXT: .reg .b64 %rd<3>;
-; SM70-FTZ-EMPTY:
-; SM70-FTZ-NEXT: // %bb.0:
-; SM70-FTZ-NEXT: ld.param.b16 %rs1, [fminimum_acq_rel_bfloat_global_cta_param_1];
-; SM70-FTZ-NEXT: ld.param.b64 %rd2, [fminimum_acq_rel_bfloat_global_cta_param_0];
-; SM70-FTZ-NEXT: fence.acq_rel.cta;
-; SM70-FTZ-NEXT: and.b64 %rd1, %rd2, -4;
-; SM70-FTZ-NEXT: cvt.u32.u64 %r4, %rd2;
-; SM70-FTZ-NEXT: and.b32 %r5, %r4, 3;
-; SM70-FTZ-NEXT: shl.b32 %r1, %r5, 3;
-; SM70-FTZ-NEXT: mov.b32 %r6, 65535;
-; SM70-FTZ-NEXT: shl.b32 %r7, %r6, %r1;
-; SM70-FTZ-NEXT: not.b32 %r2, %r7;
-; SM70-FTZ-NEXT: ld.relaxed.cta.global.b32 %r17, [%rd1];
-; SM70-FTZ-NEXT: cvt.u32.u16 %r10, %rs1;
-; SM70-FTZ-NEXT: shl.b32 %r11, %r10, 16;
-; SM70-FTZ-NEXT: $L__BB82_1: // %atomicrmw.start
-; SM70-FTZ-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-FTZ-NEXT: shr.u32 %r8, %r17, %r1;
-; SM70-FTZ-NEXT: cvt.u16.u32 %rs2, %r8;
-; SM70-FTZ-NEXT: setp.eq.b16 %p1, %rs2, -32768;
-; SM70-FTZ-NEXT: selp.b16 %rs3, %rs2, %rs1, %p1;
-; SM70-FTZ-NEXT: shl.b32 %r9, %r8, 16;
-; SM70-FTZ-NEXT: setp.lt.ftz.f32 %p2, %r9, %r11;
-; SM70-FTZ-NEXT: selp.b16 %rs4, %rs2, %rs1, %p2;
-; SM70-FTZ-NEXT: setp.nan.ftz.f32 %p3, %r9, %r11;
-; SM70-FTZ-NEXT: selp.b16 %rs5, 0x7FC0, %rs4, %p3;
-; SM70-FTZ-NEXT: setp.eq.ftz.f32 %p4, %r9, %r11;
-; SM70-FTZ-NEXT: selp.b16 %rs6, %rs3, %rs5, %p4;
-; SM70-FTZ-NEXT: cvt.u32.u16 %r12, %rs6;
-; SM70-FTZ-NEXT: shl.b32 %r13, %r12, %r1;
-; SM70-FTZ-NEXT: and.b32 %r14, %r17, %r2;
-; SM70-FTZ-NEXT: or.b32 %r15, %r14, %r13;
-; SM70-FTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r17, %r15;
-; SM70-FTZ-NEXT: setp.ne.b32 %p5, %r3, %r17;
-; SM70-FTZ-NEXT: mov.b32 %r17, %r3;
-; SM70-FTZ-NEXT: @%p5 bra $L__BB82_1;
-; SM70-FTZ-NEXT: // %bb.2: // %atomicrmw.end
-; SM70-FTZ-NEXT: shr.u32 %r16, %r3, %r1;
-; SM70-FTZ-NEXT: fence.acq_rel.cta;
-; SM70-FTZ-NEXT: st.param.b16 [func_retval0], %r16;
-; SM70-FTZ-NEXT: ret;
+; SM70-NOFTZ-ALLOW-LABEL: fminimum_acq_rel_bfloat_global_cta(
+; SM70-NOFTZ-ALLOW: {
+; SM70-NOFTZ-ALLOW-NEXT: .reg .pred %p<6>;
+; SM70-NOFTZ-ALLOW-NEXT: .reg .b16 %rs<7>;
+; SM70-NOFTZ-ALLOW-NEXT: .reg .b32 %r<18>;
+; SM70-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
+; SM70-NOFTZ-ALLOW-EMPTY:
+; SM70-NOFTZ-ALLOW-NEXT: // %bb.0:
+; SM70-NOFTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fminimum_acq_rel_bfloat_global_cta_param_1];
+; SM70-NOFTZ-ALLOW-NEXT: ld.param.b64 %rd2, [fminimum_acq_rel_bfloat_global_cta_param_0];
+; SM70-NOFTZ-ALLOW-NEXT: fence.acq_rel.cta;
+; SM70-NOFTZ-ALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NOFTZ-ALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-NOFTZ-ALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM70-NOFTZ-ALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-NOFTZ-ALLOW-NEXT: mov.b32 %r6, 65535;
+; SM70-NOFTZ-ALLOW-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-NOFTZ-ALLOW-NEXT: not.b32 %r2, %r7;
+; SM70-NOFTZ-ALLOW-NEXT: ld.relaxed.cta.global.b32 %r17, [%rd1];
+; SM70-NOFTZ-ALLOW-NEXT: cvt.u32.u16 %r10, %rs1;
+; SM70-NOFTZ-ALLOW-NEXT: shl.b32 %r11, %r10, 16;
+; SM70-NOFTZ-ALLOW-NEXT: $L__BB82_1: // %atomicrmw.start
+; SM70-NOFTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NOFTZ-ALLOW-NEXT: shr.u32 %r8, %r17, %r1;
+; SM70-NOFTZ-ALLOW-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM70-NOFTZ-ALLOW-NEXT: setp.eq.b16 %p1, %rs2, -32768;
+; SM70-NOFTZ-ALLOW-NEXT: selp.b16 %rs3, %rs2, %rs1, %p1;
+; SM70-NOFTZ-ALLOW-NEXT: shl.b32 %r9, %r8, 16;
+; SM70-NOFTZ-ALLOW-NEXT: setp.lt.f32 %p2, %r9, %r11;
+; SM70-NOFTZ-ALLOW-NEXT: selp.b16 %rs4, %rs2, %rs1, %p2;
+; SM70-NOFTZ-ALLOW-NEXT: setp.nan.f32 %p3, %r9, %r11;
+; SM70-NOFTZ-ALLOW-NEXT: selp.b16 %rs5, 0x7FC0, %rs4, %p3;
+; SM70-NOFTZ-ALLOW-NEXT: setp.eq.f32 %p4, %r9, %r11;
+; SM70-NOFTZ-ALLOW-NEXT: selp.b16 %rs6, %rs3, %rs5, %p4;
+; SM70-NOFTZ-ALLOW-NEXT: cvt.u32.u16 %r12, %rs6;
+; SM70-NOFTZ-ALLOW-NEXT: shl.b32 %r13, %r12, %r1;
+; SM70-NOFTZ-ALLOW-NEXT: and.b32 %r14, %r17, %r2;
+; SM70-NOFTZ-ALLOW-NEXT: or.b32 %r15, %r14, %r13;
+; SM70-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r17, %r15;
+; SM70-NOFTZ-ALLOW-NEXT: setp.ne.b32 %p5, %r3, %r17;
+; SM70-NOFTZ-ALLOW-NEXT: mov.b32 %r17, %r3;
+; SM70-NOFTZ-ALLOW-NEXT: @%p5 bra $L__BB82_1;
+; SM70-NOFTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NOFTZ-ALLOW-NEXT: shr.u32 %r16, %r3, %r1;
+; SM70-NOFTZ-ALLOW-NEXT: fence.acq_rel.cta;
+; SM70-NOFTZ-ALLOW-NEXT: st.param.b16 [func_retval0], %r16;
+; SM70-NOFTZ-ALLOW-NEXT: ret;
+;
+; SM70-FTZ-DISALLOW-LABEL: fminimum_acq_rel_bfloat_global_cta(
+; SM70-FTZ-DISALLOW: {
+; SM70-FTZ-DISALLOW-NEXT: .reg .pred %p<6>;
+; SM70-FTZ-DISALLOW-NEXT: .reg .b16 %rs<7>;
+; SM70-FTZ-DISALLOW-NEXT: .reg .b32 %r<18>;
+; SM70-FTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
+; SM70-FTZ-DISALLOW-EMPTY:
+; SM70-FTZ-DISALLOW-NEXT: // %bb.0:
+; SM70-FTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fminimum_acq_rel_bfloat_global_cta_param_1];
+; SM70-FTZ-DISALLOW-NEXT: ld.param.b64 %rd2, [fminimum_acq_rel_bfloat_global_cta_param_0];
+; SM70-FTZ-DISALLOW-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-DISALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-FTZ-DISALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-FTZ-DISALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM70-FTZ-DISALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-FTZ-DISALLOW-NEXT: mov.b32 %r6, 65535;
+; SM70-FTZ-DISALLOW-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-FTZ-DISALLOW-NEXT: not.b32 %r2, %r7;
+; SM70-FTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b32 %r17, [%rd1];
+; SM70-FTZ-DISALLOW-NEXT: cvt.u32.u16 %r10, %rs1;
+; SM70-FTZ-DISALLOW-NEXT: shl.b32 %r11, %r10, 16;
+; SM70-FTZ-DISALLOW-NEXT: $L__BB82_1: // %atomicrmw.start
+; SM70-FTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-FTZ-DISALLOW-NEXT: shr.u32 %r8, %r17, %r1;
+; SM70-FTZ-DISALLOW-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM70-FTZ-DISALLOW-NEXT: setp.eq.b16 %p1, %rs2, -32768;
+; SM70-FTZ-DISALLOW-NEXT: selp.b16 %rs3, %rs2, %rs1, %p1;
+; SM70-FTZ-DISALLOW-NEXT: shl.b32 %r9, %r8, 16;
+; SM70-FTZ-DISALLOW-NEXT: setp.lt.ftz.f32 %p2, %r9, %r11;
+; SM70-FTZ-DISALLOW-NEXT: selp.b16 %rs4, %rs2, %rs1, %p2;
+; SM70-FTZ-DISALLOW-NEXT: setp.nan.ftz.f32 %p3, %r9, %r11;
+; SM70-FTZ-DISALLOW-NEXT: selp.b16 %rs5, 0x7FC0, %rs4, %p3;
+; SM70-FTZ-DISALLOW-NEXT: setp.eq.ftz.f32 %p4, %r9, %r11;
+; SM70-FTZ-DISALLOW-NEXT: selp.b16 %rs6, %rs3, %rs5, %p4;
+; SM70-FTZ-DISALLOW-NEXT: cvt.u32.u16 %r12, %rs6;
+; SM70-FTZ-DISALLOW-NEXT: shl.b32 %r13, %r12, %r1;
+; SM70-FTZ-DISALLOW-NEXT: and.b32 %r14, %r17, %r2;
+; SM70-FTZ-DISALLOW-NEXT: or.b32 %r15, %r14, %r13;
+; SM70-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r17, %r15;
+; SM70-FTZ-DISALLOW-NEXT: setp.ne.b32 %p5, %r3, %r17;
+; SM70-FTZ-DISALLOW-NEXT: mov.b32 %r17, %r3;
+; SM70-FTZ-DISALLOW-NEXT: @%p5 bra $L__BB82_1;
+; SM70-FTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-FTZ-DISALLOW-NEXT: shr.u32 %r16, %r3, %r1;
+; SM70-FTZ-DISALLOW-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-DISALLOW-NEXT: st.param.b16 [func_retval0], %r16;
+; SM70-FTZ-DISALLOW-NEXT: ret;
+;
+; SM70-FTZ-ALLOW-LABEL: fminimum_acq_rel_bfloat_global_cta(
+; SM70-FTZ-ALLOW: {
+; SM70-FTZ-ALLOW-NEXT: .reg .pred %p<6>;
+; SM70-FTZ-ALLOW-NEXT: .reg .b16 %rs<7>;
+; SM70-FTZ-ALLOW-NEXT: .reg .b32 %r<18>;
+; SM70-FTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
+; SM70-FTZ-ALLOW-EMPTY:
+; SM70-FTZ-ALLOW-NEXT: // %bb.0:
+; SM70-FTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fminimum_acq_rel_bfloat_global_cta_param_1];
+; SM70-FTZ-ALLOW-NEXT: ld.param.b64 %rd2, [fminimum_acq_rel_bfloat_global_cta_param_0];
+; SM70-FTZ-ALLOW-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-ALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-FTZ-ALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-FTZ-ALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM70-FTZ-ALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-FTZ-ALLOW-NEXT: mov.b32 %r6, 65535;
+; SM70-FTZ-ALLOW-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-FTZ-ALLOW-NEXT: not.b32 %r2, %r7;
+; SM70-FTZ-ALLOW-NEXT: ld.relaxed.cta.global.b32 %r17, [%rd1];
+; SM70-FTZ-ALLOW-NEXT: cvt.u32.u16 %r10, %rs1;
+; SM70-FTZ-ALLOW-NEXT: shl.b32 %r11, %r10, 16;
+; SM70-FTZ-ALLOW-NEXT: $L__BB82_1: // %atomicrmw.start
+; SM70-FTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-FTZ-ALLOW-NEXT: shr.u32 %r8, %r17, %r1;
+; SM70-FTZ-ALLOW-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM70-FTZ-ALLOW-NEXT: setp.eq.b16 %p1, %rs2, -32768;
+; SM70-FTZ-ALLOW-NEXT: selp.b16 %rs3, %rs2, %rs1, %p1;
+; SM70-FTZ-ALLOW-NEXT: shl.b32 %r9, %r8, 16;
+; SM70-FTZ-ALLOW-NEXT: setp.lt.ftz.f32 %p2, %r9, %r11;
+; SM70-FTZ-ALLOW-NEXT: selp.b16 %rs4, %rs2, %rs1, %p2;
+; SM70-FTZ-ALLOW-NEXT: setp.nan.ftz.f32 %p3, %r9, %r11;
+; SM70-FTZ-ALLOW-NEXT: selp.b16 %rs5, 0x7FC0, %rs4, %p3;
+; SM70-FTZ-ALLOW-NEXT: setp.eq.ftz.f32 %p4, %r9, %r11;
+; SM70-FTZ-ALLOW-NEXT: selp.b16 %rs6, %rs3, %rs5, %p4;
+; SM70-FTZ-ALLOW-NEXT: cvt.u32.u16 %r12, %rs6;
+; SM70-FTZ-ALLOW-NEXT: shl.b32 %r13, %r12, %r1;
+; SM70-FTZ-ALLOW-NEXT: and.b32 %r14, %r17, %r2;
+; SM70-FTZ-ALLOW-NEXT: or.b32 %r15, %r14, %r13;
+; SM70-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r17, %r15;
+; SM70-FTZ-ALLOW-NEXT: setp.ne.b32 %p5, %r3, %r17;
+; SM70-FTZ-ALLOW-NEXT: mov.b32 %r17, %r3;
+; SM70-FTZ-ALLOW-NEXT: @%p5 bra $L__BB82_1;
+; SM70-FTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-FTZ-ALLOW-NEXT: shr.u32 %r16, %r3, %r1;
+; SM70-FTZ-ALLOW-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-ALLOW-NEXT: st.param.b16 [func_retval0], %r16;
+; SM70-FTZ-ALLOW-NEXT: ret;
%retval = atomicrmw fminimum ptr addrspace(1) %addr, bfloat %val syncscope("block") acq_rel
ret bfloat %retval
}
define bfloat @fmaximum_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat %val) {
-; SM70-NOFTZ-LABEL: fmaximum_acq_rel_bfloat_global_cta(
-; SM70-NOFTZ: {
-; SM70-NOFTZ-NEXT: .reg .pred %p<6>;
-; SM70-NOFTZ-NEXT: .reg .b16 %rs<7>;
-; SM70-NOFTZ-NEXT: .reg .b32 %r<18>;
-; SM70-NOFTZ-NEXT: .reg .b64 %rd<3>;
-; SM70-NOFTZ-EMPTY:
-; SM70-NOFTZ-NEXT: // %bb.0:
-; SM70-NOFTZ-NEXT: ld.param.b16 %rs1, [fmaximum_acq_rel_bfloat_global_cta_param_1];
-; SM70-NOFTZ-NEXT: ld.param.b64 %rd2, [fmaximum_acq_rel_bfloat_global_cta_param_0];
-; SM70-NOFTZ-NEXT: fence.acq_rel.cta;
-; SM70-NOFTZ-NEXT: and.b64 %rd1, %rd2, -4;
-; SM70-NOFTZ-NEXT: cvt.u32.u64 %r4, %rd2;
-; SM70-NOFTZ-NEXT: and.b32 %r5, %r4, 3;
-; SM70-NOFTZ-NEXT: shl.b32 %r1, %r5, 3;
-; SM70-NOFTZ-NEXT: mov.b32 %r6, 65535;
-; SM70-NOFTZ-NEXT: shl.b32 %r7, %r6, %r1;
-; SM70-NOFTZ-NEXT: not.b32 %r2, %r7;
-; SM70-NOFTZ-NEXT: ld.relaxed.cta.global.b32 %r17, [%rd1];
-; SM70-NOFTZ-NEXT: cvt.u32.u16 %r10, %rs1;
-; SM70-NOFTZ-NEXT: shl.b32 %r11, %r10, 16;
-; SM70-NOFTZ-NEXT: $L__BB83_1: // %atomicrmw.start
-; SM70-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NOFTZ-NEXT: shr.u32 %r8, %r17, %r1;
-; SM70-NOFTZ-NEXT: cvt.u16.u32 %rs2, %r8;
-; SM70-NOFTZ-NEXT: setp.eq.b16 %p1, %rs2, 0;
-; SM70-NOFTZ-NEXT: selp.b16 %rs3, %rs2, %rs1, %p1;
-; SM70-NOFTZ-NEXT: shl.b32 %r9, %r8, 16;
-; SM70-NOFTZ-NEXT: setp.gt.f32 %p2, %r9, %r11;
-; SM70-NOFTZ-NEXT: selp.b16 %rs4, %rs2, %rs1, %p2;
-; SM70-NOFTZ-NEXT: setp.nan.f32 %p3, %r9, %r11;
-; SM70-NOFTZ-NEXT: selp.b16 %rs5, 0x7FC0, %rs4, %p3;
-; SM70-NOFTZ-NEXT: setp.eq.f32 %p4, %r9, %r11;
-; SM70-NOFTZ-NEXT: selp.b16 %rs6, %rs3, %rs5, %p4;
-; SM70-NOFTZ-NEXT: cvt.u32.u16 %r12, %rs6;
-; SM70-NOFTZ-NEXT: shl.b32 %r13, %r12, %r1;
-; SM70-NOFTZ-NEXT: and.b32 %r14, %r17, %r2;
-; SM70-NOFTZ-NEXT: or.b32 %r15, %r14, %r13;
-; SM70-NOFTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r17, %r15;
-; SM70-NOFTZ-NEXT: setp.ne.b32 %p5, %r3, %r17;
-; SM70-NOFTZ-NEXT: mov.b32 %r17, %r3;
-; SM70-NOFTZ-NEXT: @%p5 bra $L__BB83_1;
-; SM70-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end
-; SM70-NOFTZ-NEXT: shr.u32 %r16, %r3, %r1;
-; SM70-NOFTZ-NEXT: fence.acq_rel.cta;
-; SM70-NOFTZ-NEXT: st.param.b16 [func_retval0], %r16;
-; SM70-NOFTZ-NEXT: ret;
+; SM70-NOFTZ-DISALLOW-LABEL: fmaximum_acq_rel_bfloat_global_cta(
+; SM70-NOFTZ-DISALLOW: {
+; SM70-NOFTZ-DISALLOW-NEXT: .reg .pred %p<6>;
+; SM70-NOFTZ-DISALLOW-NEXT: .reg .b16 %rs<7>;
+; SM70-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<18>;
+; SM70-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
+; SM70-NOFTZ-DISALLOW-EMPTY:
+; SM70-NOFTZ-DISALLOW-NEXT: // %bb.0:
+; SM70-NOFTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fmaximum_acq_rel_bfloat_global_cta_param_1];
+; SM70-NOFTZ-DISALLOW-NEXT: ld.param.b64 %rd2, [fmaximum_acq_rel_bfloat_global_cta_param_0];
+; SM70-NOFTZ-DISALLOW-NEXT: fence.acq_rel.cta;
+; SM70-NOFTZ-DISALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-NOFTZ-DISALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM70-NOFTZ-DISALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-NOFTZ-DISALLOW-NEXT: mov.b32 %r6, 65535;
+; SM70-NOFTZ-DISALLOW-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-NOFTZ-DISALLOW-NEXT: not.b32 %r2, %r7;
+; SM70-NOFTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b32 %r17, [%rd1];
+; SM70-NOFTZ-DISALLOW-NEXT: cvt.u32.u16 %r10, %rs1;
+; SM70-NOFTZ-DISALLOW-NEXT: shl.b32 %r11, %r10, 16;
+; SM70-NOFTZ-DISALLOW-NEXT: $L__BB83_1: // %atomicrmw.start
+; SM70-NOFTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NOFTZ-DISALLOW-NEXT: shr.u32 %r8, %r17, %r1;
+; SM70-NOFTZ-DISALLOW-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM70-NOFTZ-DISALLOW-NEXT: setp.eq.b16 %p1, %rs2, 0;
+; SM70-NOFTZ-DISALLOW-NEXT: selp.b16 %rs3, %rs2, %rs1, %p1;
+; SM70-NOFTZ-DISALLOW-NEXT: shl.b32 %r9, %r8, 16;
+; SM70-NOFTZ-DISALLOW-NEXT: setp.gt.f32 %p2, %r9, %r11;
+; SM70-NOFTZ-DISALLOW-NEXT: selp.b16 %rs4, %rs2, %rs1, %p2;
+; SM70-NOFTZ-DISALLOW-NEXT: setp.nan.f32 %p3, %r9, %r11;
+; SM70-NOFTZ-DISALLOW-NEXT: selp.b16 %rs5, 0x7FC0, %rs4, %p3;
+; SM70-NOFTZ-DISALLOW-NEXT: setp.eq.f32 %p4, %r9, %r11;
+; SM70-NOFTZ-DISALLOW-NEXT: selp.b16 %rs6, %rs3, %rs5, %p4;
+; SM70-NOFTZ-DISALLOW-NEXT: cvt.u32.u16 %r12, %rs6;
+; SM70-NOFTZ-DISALLOW-NEXT: shl.b32 %r13, %r12, %r1;
+; SM70-NOFTZ-DISALLOW-NEXT: and.b32 %r14, %r17, %r2;
+; SM70-NOFTZ-DISALLOW-NEXT: or.b32 %r15, %r14, %r13;
+; SM70-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r17, %r15;
+; SM70-NOFTZ-DISALLOW-NEXT: setp.ne.b32 %p5, %r3, %r17;
+; SM70-NOFTZ-DISALLOW-NEXT: mov.b32 %r17, %r3;
+; SM70-NOFTZ-DISALLOW-NEXT: @%p5 bra $L__BB83_1;
+; SM70-NOFTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NOFTZ-DISALLOW-NEXT: shr.u32 %r16, %r3, %r1;
+; SM70-NOFTZ-DISALLOW-NEXT: fence.acq_rel.cta;
+; SM70-NOFTZ-DISALLOW-NEXT: st.param.b16 [func_retval0], %r16;
+; SM70-NOFTZ-DISALLOW-NEXT: ret;
+;
+; SM70-NOFTZ-ALLOW-LABEL: fmaximum_acq_rel_bfloat_global_cta(
+; SM70-NOFTZ-ALLOW: {
+; SM70-NOFTZ-ALLOW-NEXT: .reg .pred %p<6>;
+; SM70-NOFTZ-ALLOW-NEXT: .reg .b16 %rs<7>;
+; SM70-NOFTZ-ALLOW-NEXT: .reg .b32 %r<18>;
+; SM70-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
+; SM70-NOFTZ-ALLOW-EMPTY:
+; SM70-NOFTZ-ALLOW-NEXT: // %bb.0:
+; SM70-NOFTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fmaximum_acq_rel_bfloat_global_cta_param_1];
+; SM70-NOFTZ-ALLOW-NEXT: ld.param.b64 %rd2, [fmaximum_acq_rel_bfloat_global_cta_param_0];
+; SM70-NOFTZ-ALLOW-NEXT: fence.acq_rel.cta;
+; SM70-NOFTZ-ALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-NOFTZ-ALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-NOFTZ-ALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM70-NOFTZ-ALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-NOFTZ-ALLOW-NEXT: mov.b32 %r6, 65535;
+; SM70-NOFTZ-ALLOW-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-NOFTZ-ALLOW-NEXT: not.b32 %r2, %r7;
+; SM70-NOFTZ-ALLOW-NEXT: ld.relaxed.cta.global.b32 %r17, [%rd1];
+; SM70-NOFTZ-ALLOW-NEXT: cvt.u32.u16 %r10, %rs1;
+; SM70-NOFTZ-ALLOW-NEXT: shl.b32 %r11, %r10, 16;
+; SM70-NOFTZ-ALLOW-NEXT: $L__BB83_1: // %atomicrmw.start
+; SM70-NOFTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NOFTZ-ALLOW-NEXT: shr.u32 %r8, %r17, %r1;
+; SM70-NOFTZ-ALLOW-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM70-NOFTZ-ALLOW-NEXT: setp.eq.b16 %p1, %rs2, 0;
+; SM70-NOFTZ-ALLOW-NEXT: selp.b16 %rs3, %rs2, %rs1, %p1;
+; SM70-NOFTZ-ALLOW-NEXT: shl.b32 %r9, %r8, 16;
+; SM70-NOFTZ-ALLOW-NEXT: setp.gt.f32 %p2, %r9, %r11;
+; SM70-NOFTZ-ALLOW-NEXT: selp.b16 %rs4, %rs2, %rs1, %p2;
+; SM70-NOFTZ-ALLOW-NEXT: setp.nan.f32 %p3, %r9, %r11;
+; SM70-NOFTZ-ALLOW-NEXT: selp.b16 %rs5, 0x7FC0, %rs4, %p3;
+; SM70-NOFTZ-ALLOW-NEXT: setp.eq.f32 %p4, %r9, %r11;
+; SM70-NOFTZ-ALLOW-NEXT: selp.b16 %rs6, %rs3, %rs5, %p4;
+; SM70-NOFTZ-ALLOW-NEXT: cvt.u32.u16 %r12, %rs6;
+; SM70-NOFTZ-ALLOW-NEXT: shl.b32 %r13, %r12, %r1;
+; SM70-NOFTZ-ALLOW-NEXT: and.b32 %r14, %r17, %r2;
+; SM70-NOFTZ-ALLOW-NEXT: or.b32 %r15, %r14, %r13;
+; SM70-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r17, %r15;
+; SM70-NOFTZ-ALLOW-NEXT: setp.ne.b32 %p5, %r3, %r17;
+; SM70-NOFTZ-ALLOW-NEXT: mov.b32 %r17, %r3;
+; SM70-NOFTZ-ALLOW-NEXT: @%p5 bra $L__BB83_1;
+; SM70-NOFTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NOFTZ-ALLOW-NEXT: shr.u32 %r16, %r3, %r1;
+; SM70-NOFTZ-ALLOW-NEXT: fence.acq_rel.cta;
+; SM70-NOFTZ-ALLOW-NEXT: st.param.b16 [func_retval0], %r16;
+; SM70-NOFTZ-ALLOW-NEXT: ret;
;
-; SM70-FTZ-LABEL: fmaximum_acq_rel_bfloat_global_cta(
-; SM70-FTZ: {
-; SM70-FTZ-NEXT: .reg .pred %p<6>;
-; SM70-FTZ-NEXT: .reg .b16 %rs<7>;
-; SM70-FTZ-NEXT: .reg .b32 %r<18>;
-; SM70-FTZ-NEXT: .reg .b64 %rd<3>;
-; SM70-FTZ-EMPTY:
-; SM70-FTZ-NEXT: // %bb.0:
-; SM70-FTZ-NEXT: ld.param.b16 %rs1, [fmaximum_acq_rel_bfloat_global_cta_param_1];
-; SM70-FTZ-NEXT: ld.param.b64 %rd2, [fmaximum_acq_rel_bfloat_global_cta_param_0];
-; SM70-FTZ-NEXT: fence.acq_rel.cta;
-; SM70-FTZ-NEXT: and.b64 %rd1, %rd2, -4;
-; SM70-FTZ-NEXT: cvt.u32.u64 %r4, %rd2;
-; SM70-FTZ-NEXT: and.b32 %r5, %r4, 3;
-; SM70-FTZ-NEXT: shl.b32 %r1, %r5, 3;
-; SM70-FTZ-NEXT: mov.b32 %r6, 65535;
-; SM70-FTZ-NEXT: shl.b32 %r7, %r6, %r1;
-; SM70-FTZ-NEXT: not.b32 %r2, %r7;
-; SM70-FTZ-NEXT: ld.relaxed.cta.global.b32 %r17, [%rd1];
-; SM70-FTZ-NEXT: cvt.u32.u16 %r10, %rs1;
-; SM70-FTZ-NEXT: shl.b32 %r11, %r10, 16;
-; SM70-FTZ-NEXT: $L__BB83_1: // %atomicrmw.start
-; SM70-FTZ-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-FTZ-NEXT: shr.u32 %r8, %r17, %r1;
-; SM70-FTZ-NEXT: cvt.u16.u32 %rs2, %r8;
-; SM70-FTZ-NEXT: setp.eq.b16 %p1, %rs2, 0;
-; SM70-FTZ-NEXT: selp.b16 %rs3, %rs2, %rs1, %p1;
-; SM70-FTZ-NEXT: shl.b32 %r9, %r8, 16;
-; SM70-FTZ-NEXT: setp.gt.ftz.f32 %p2, %r9, %r11;
-; SM70-FTZ-NEXT: selp.b16 %rs4, %rs2, %rs1, %p2;
-; SM70-FTZ-NEXT: setp.nan.ftz.f32 %p3, %r9, %r11;
-; SM70-FTZ-NEXT: selp.b16 %rs5, 0x7FC0, %rs4, %p3;
-; SM70-FTZ-NEXT: setp.eq.ftz.f32 %p4, %r9, %r11;
-; SM70-FTZ-NEXT: selp.b16 %rs6, %rs3, %rs5, %p4;
-; SM70-FTZ-NEXT: cvt.u32.u16 %r12, %rs6;
-; SM70-FTZ-NEXT: shl.b32 %r13, %r12, %r1;
-; SM70-FTZ-NEXT: and.b32 %r14, %r17, %r2;
-; SM70-FTZ-NEXT: or.b32 %r15, %r14, %r13;
-; SM70-FTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r17, %r15;
-; SM70-FTZ-NEXT: setp.ne.b32 %p5, %r3, %r17;
-; SM70-FTZ-NEXT: mov.b32 %r17, %r3;
-; SM70-FTZ-NEXT: @%p5 bra $L__BB83_1;
-; SM70-FTZ-NEXT: // %bb.2: // %atomicrmw.end
-; SM70-FTZ-NEXT: shr.u32 %r16, %r3, %r1;
-; SM70-FTZ-NEXT: fence.acq_rel.cta;
-; SM70-FTZ-NEXT: st.param.b16 [func_retval0], %r16;
-; SM70-FTZ-NEXT: ret;
+; SM70-FTZ-DISALLOW-LABEL: fmaximum_acq_rel_bfloat_global_cta(
+; SM70-FTZ-DISALLOW: {
+; SM70-FTZ-DISALLOW-NEXT: .reg .pred %p<6>;
+; SM70-FTZ-DISALLOW-NEXT: .reg .b16 %rs<7>;
+; SM70-FTZ-DISALLOW-NEXT: .reg .b32 %r<18>;
+; SM70-FTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
+; SM70-FTZ-DISALLOW-EMPTY:
+; SM70-FTZ-DISALLOW-NEXT: // %bb.0:
+; SM70-FTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fmaximum_acq_rel_bfloat_global_cta_param_1];
+; SM70-FTZ-DISALLOW-NEXT: ld.param.b64 %rd2, [fmaximum_acq_rel_bfloat_global_cta_param_0];
+; SM70-FTZ-DISALLOW-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-DISALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-FTZ-DISALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-FTZ-DISALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM70-FTZ-DISALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-FTZ-DISALLOW-NEXT: mov.b32 %r6, 65535;
+; SM70-FTZ-DISALLOW-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-FTZ-DISALLOW-NEXT: not.b32 %r2, %r7;
+; SM70-FTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b32 %r17, [%rd1];
+; SM70-FTZ-DISALLOW-NEXT: cvt.u32.u16 %r10, %rs1;
+; SM70-FTZ-DISALLOW-NEXT: shl.b32 %r11, %r10, 16;
+; SM70-FTZ-DISALLOW-NEXT: $L__BB83_1: // %atomicrmw.start
+; SM70-FTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-FTZ-DISALLOW-NEXT: shr.u32 %r8, %r17, %r1;
+; SM70-FTZ-DISALLOW-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM70-FTZ-DISALLOW-NEXT: setp.eq.b16 %p1, %rs2, 0;
+; SM70-FTZ-DISALLOW-NEXT: selp.b16 %rs3, %rs2, %rs1, %p1;
+; SM70-FTZ-DISALLOW-NEXT: shl.b32 %r9, %r8, 16;
+; SM70-FTZ-DISALLOW-NEXT: setp.gt.ftz.f32 %p2, %r9, %r11;
+; SM70-FTZ-DISALLOW-NEXT: selp.b16 %rs4, %rs2, %rs1, %p2;
+; SM70-FTZ-DISALLOW-NEXT: setp.nan.ftz.f32 %p3, %r9, %r11;
+; SM70-FTZ-DISALLOW-NEXT: selp.b16 %rs5, 0x7FC0, %rs4, %p3;
+; SM70-FTZ-DISALLOW-NEXT: setp.eq.ftz.f32 %p4, %r9, %r11;
+; SM70-FTZ-DISALLOW-NEXT: selp.b16 %rs6, %rs3, %rs5, %p4;
+; SM70-FTZ-DISALLOW-NEXT: cvt.u32.u16 %r12, %rs6;
+; SM70-FTZ-DISALLOW-NEXT: shl.b32 %r13, %r12, %r1;
+; SM70-FTZ-DISALLOW-NEXT: and.b32 %r14, %r17, %r2;
+; SM70-FTZ-DISALLOW-NEXT: or.b32 %r15, %r14, %r13;
+; SM70-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r17, %r15;
+; SM70-FTZ-DISALLOW-NEXT: setp.ne.b32 %p5, %r3, %r17;
+; SM70-FTZ-DISALLOW-NEXT: mov.b32 %r17, %r3;
+; SM70-FTZ-DISALLOW-NEXT: @%p5 bra $L__BB83_1;
+; SM70-FTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-FTZ-DISALLOW-NEXT: shr.u32 %r16, %r3, %r1;
+; SM70-FTZ-DISALLOW-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-DISALLOW-NEXT: st.param.b16 [func_retval0], %r16;
+; SM70-FTZ-DISALLOW-NEXT: ret;
+;
+; SM70-FTZ-ALLOW-LABEL: fmaximum_acq_rel_bfloat_global_cta(
+; SM70-FTZ-ALLOW: {
+; SM70-FTZ-ALLOW-NEXT: .reg .pred %p<6>;
+; SM70-FTZ-ALLOW-NEXT: .reg .b16 %rs<7>;
+; SM70-FTZ-ALLOW-NEXT: .reg .b32 %r<18>;
+; SM70-FTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
+; SM70-FTZ-ALLOW-EMPTY:
+; SM70-FTZ-ALLOW-NEXT: // %bb.0:
+; SM70-FTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fmaximum_acq_rel_bfloat_global_cta_param_1];
+; SM70-FTZ-ALLOW-NEXT: ld.param.b64 %rd2, [fmaximum_acq_rel_bfloat_global_cta_param_0];
+; SM70-FTZ-ALLOW-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-ALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM70-FTZ-ALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM70-FTZ-ALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM70-FTZ-ALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM70-FTZ-ALLOW-NEXT: mov.b32 %r6, 65535;
+; SM70-FTZ-ALLOW-NEXT: shl.b32 %r7, %r6, %r1;
+; SM70-FTZ-ALLOW-NEXT: not.b32 %r2, %r7;
+; SM70-FTZ-ALLOW-NEXT: ld.relaxed.cta.global.b32 %r17, [%rd1];
+; SM70-FTZ-ALLOW-NEXT: cvt.u32.u16 %r10, %rs1;
+; SM70-FTZ-ALLOW-NEXT: shl.b32 %r11, %r10, 16;
+; SM70-FTZ-ALLOW-NEXT: $L__BB83_1: // %atomicrmw.start
+; SM70-FTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-FTZ-ALLOW-NEXT: shr.u32 %r8, %r17, %r1;
+; SM70-FTZ-ALLOW-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM70-FTZ-ALLOW-NEXT: setp.eq.b16 %p1, %rs2, 0;
+; SM70-FTZ-ALLOW-NEXT: selp.b16 %rs3, %rs2, %rs1, %p1;
+; SM70-FTZ-ALLOW-NEXT: shl.b32 %r9, %r8, 16;
+; SM70-FTZ-ALLOW-NEXT: setp.gt.ftz.f32 %p2, %r9, %r11;
+; SM70-FTZ-ALLOW-NEXT: selp.b16 %rs4, %rs2, %rs1, %p2;
+; SM70-FTZ-ALLOW-NEXT: setp.nan.ftz.f32 %p3, %r9, %r11;
+; SM70-FTZ-ALLOW-NEXT: selp.b16 %rs5, 0x7FC0, %rs4, %p3;
+; SM70-FTZ-ALLOW-NEXT: setp.eq.ftz.f32 %p4, %r9, %r11;
+; SM70-FTZ-ALLOW-NEXT: selp.b16 %rs6, %rs3, %rs5, %p4;
+; SM70-FTZ-ALLOW-NEXT: cvt.u32.u16 %r12, %rs6;
+; SM70-FTZ-ALLOW-NEXT: shl.b32 %r13, %r12, %r1;
+; SM70-FTZ-ALLOW-NEXT: and.b32 %r14, %r17, %r2;
+; SM70-FTZ-ALLOW-NEXT: or.b32 %r15, %r14, %r13;
+; SM70-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r17, %r15;
+; SM70-FTZ-ALLOW-NEXT: setp.ne.b32 %p5, %r3, %r17;
+; SM70-FTZ-ALLOW-NEXT: mov.b32 %r17, %r3;
+; SM70-FTZ-ALLOW-NEXT: @%p5 bra $L__BB83_1;
+; SM70-FTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-FTZ-ALLOW-NEXT: shr.u32 %r16, %r3, %r1;
+; SM70-FTZ-ALLOW-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-ALLOW-NEXT: st.param.b16 [func_retval0], %r16;
+; SM70-FTZ-ALLOW-NEXT: ret;
%retval = atomicrmw fmaximum ptr addrspace(1) %addr, bfloat %val syncscope("block") acq_rel
ret bfloat %retval
}
@@ -3886,40 +5176,64 @@ define i64 @nand_acq_rel_i64_global_sys(ptr addrspace(1) %addr, i64 %val) {
}
define float @fadd_acq_rel_float_global_sys(ptr addrspace(1) %addr, float %val) {
-; SM70-NOFTZ-LABEL: fadd_acq_rel_float_global_sys(
-; SM70-NOFTZ: {
-; SM70-NOFTZ-NEXT: .reg .pred %p<2>;
-; SM70-NOFTZ-NEXT: .reg .b32 %r<5>;
-; SM70-NOFTZ-NEXT: .reg .b64 %rd<2>;
-; SM70-NOFTZ-EMPTY:
-; SM70-NOFTZ-NEXT: // %bb.0:
-; SM70-NOFTZ-NEXT: ld.param.b32 %r2, [fadd_acq_rel_float_global_sys_param_1];
-; SM70-NOFTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_sys_param_0];
-; SM70-NOFTZ-NEXT: fence.acq_rel.sys;
-; SM70-NOFTZ-NEXT: ld.relaxed.sys.global.b32 %r4, [%rd1];
-; SM70-NOFTZ-NEXT: $L__BB106_1: // %atomicrmw.start
-; SM70-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NOFTZ-NEXT: add.rn.f32 %r3, %r4, %r2;
-; SM70-NOFTZ-NEXT: atom.relaxed.sys.global.cas.b32 %r1, [%rd1], %r4, %r3;
-; SM70-NOFTZ-NEXT: setp.ne.b32 %p1, %r1, %r4;
-; SM70-NOFTZ-NEXT: mov.b32 %r4, %r1;
-; SM70-NOFTZ-NEXT: @%p1 bra $L__BB106_1;
-; SM70-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end
-; SM70-NOFTZ-NEXT: fence.acq_rel.sys;
-; SM70-NOFTZ-NEXT: st.param.b32 [func_retval0], %r1;
-; SM70-NOFTZ-NEXT: ret;
+; SM70-NOFTZ-DISALLOW-LABEL: fadd_acq_rel_float_global_sys(
+; SM70-NOFTZ-DISALLOW: {
+; SM70-NOFTZ-DISALLOW-NEXT: .reg .pred %p<2>;
+; SM70-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<5>;
+; SM70-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
+; SM70-NOFTZ-DISALLOW-EMPTY:
+; SM70-NOFTZ-DISALLOW-NEXT: // %bb.0:
+; SM70-NOFTZ-DISALLOW-NEXT: ld.param.b32 %r2, [fadd_acq_rel_float_global_sys_param_1];
+; SM70-NOFTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_sys_param_0];
+; SM70-NOFTZ-DISALLOW-NEXT: fence.acq_rel.sys;
+; SM70-NOFTZ-DISALLOW-NEXT: ld.relaxed.sys.global.b32 %r4, [%rd1];
+; SM70-NOFTZ-DISALLOW-NEXT: $L__BB106_1: // %atomicrmw.start
+; SM70-NOFTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NOFTZ-DISALLOW-NEXT: add.rn.f32 %r3, %r4, %r2;
+; SM70-NOFTZ-DISALLOW-NEXT: atom.relaxed.sys.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM70-NOFTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM70-NOFTZ-DISALLOW-NEXT: mov.b32 %r4, %r1;
+; SM70-NOFTZ-DISALLOW-NEXT: @%p1 bra $L__BB106_1;
+; SM70-NOFTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NOFTZ-DISALLOW-NEXT: fence.acq_rel.sys;
+; SM70-NOFTZ-DISALLOW-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NOFTZ-DISALLOW-NEXT: ret;
+;
+; SM70-NOFTZ-ALLOW-LABEL: fadd_acq_rel_float_global_sys(
+; SM70-NOFTZ-ALLOW: {
+; SM70-NOFTZ-ALLOW-NEXT: .reg .b32 %r<3>;
+; SM70-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
+; SM70-NOFTZ-ALLOW-EMPTY:
+; SM70-NOFTZ-ALLOW-NEXT: // %bb.0:
+; SM70-NOFTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_sys_param_0];
+; SM70-NOFTZ-ALLOW-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_global_sys_param_1];
+; SM70-NOFTZ-ALLOW-NEXT: atom.acq_rel.sys.global.add.f32 %r2, [%rd1], %r1;
+; SM70-NOFTZ-ALLOW-NEXT: st.param.b32 [func_retval0], %r2;
+; SM70-NOFTZ-ALLOW-NEXT: ret;
+;
+; SM70-FTZ-DISALLOW-LABEL: fadd_acq_rel_float_global_sys(
+; SM70-FTZ-DISALLOW: {
+; SM70-FTZ-DISALLOW-NEXT: .reg .b32 %r<3>;
+; SM70-FTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
+; SM70-FTZ-DISALLOW-EMPTY:
+; SM70-FTZ-DISALLOW-NEXT: // %bb.0:
+; SM70-FTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_sys_param_0];
+; SM70-FTZ-DISALLOW-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_global_sys_param_1];
+; SM70-FTZ-DISALLOW-NEXT: atom.acq_rel.sys.global.add.f32 %r2, [%rd1], %r1;
+; SM70-FTZ-DISALLOW-NEXT: st.param.b32 [func_retval0], %r2;
+; SM70-FTZ-DISALLOW-NEXT: ret;
;
-; SM70-FTZ-LABEL: fadd_acq_rel_float_global_sys(
-; SM70-FTZ: {
-; SM70-FTZ-NEXT: .reg .b32 %r<3>;
-; SM70-FTZ-NEXT: .reg .b64 %rd<2>;
-; SM70-FTZ-EMPTY:
-; SM70-FTZ-NEXT: // %bb.0:
-; SM70-FTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_sys_param_0];
-; SM70-FTZ-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_global_sys_param_1];
-; SM70-FTZ-NEXT: atom.acq_rel.sys.global.add.f32 %r2, [%rd1], %r1;
-; SM70-FTZ-NEXT: st.param.b32 [func_retval0], %r2;
-; SM70-FTZ-NEXT: ret;
+; SM70-FTZ-ALLOW-LABEL: fadd_acq_rel_float_global_sys(
+; SM70-FTZ-ALLOW: {
+; SM70-FTZ-ALLOW-NEXT: .reg .b32 %r<3>;
+; SM70-FTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
+; SM70-FTZ-ALLOW-EMPTY:
+; SM70-FTZ-ALLOW-NEXT: // %bb.0:
+; SM70-FTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_sys_param_0];
+; SM70-FTZ-ALLOW-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_global_sys_param_1];
+; SM70-FTZ-ALLOW-NEXT: atom.acq_rel.sys.global.add.f32 %r2, [%rd1], %r1;
+; SM70-FTZ-ALLOW-NEXT: st.param.b32 [func_retval0], %r2;
+; SM70-FTZ-ALLOW-NEXT: ret;
%retval = atomicrmw fadd ptr addrspace(1) %addr, float %val syncscope("") acq_rel
ret float %retval
}
@@ -4058,40 +5372,64 @@ define i64 @nand_acq_rel_i64_global_cluster(ptr addrspace(1) %addr, i64 %val) {
}
define float @fadd_acq_rel_float_global_cluster(ptr addrspace(1) %addr, float %val) {
-; SM70-NOFTZ-LABEL: fadd_acq_rel_float_global_cluster(
-; SM70-NOFTZ: {
-; SM70-NOFTZ-NEXT: .reg .pred %p<2>;
-; SM70-NOFTZ-NEXT: .reg .b32 %r<5>;
-; SM70-NOFTZ-NEXT: .reg .b64 %rd<2>;
-; SM70-NOFTZ-EMPTY:
-; SM70-NOFTZ-NEXT: // %bb.0:
-; SM70-NOFTZ-NEXT: ld.param.b32 %r2, [fadd_acq_rel_float_global_cluster_param_1];
-; SM70-NOFTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_cluster_param_0];
-; SM70-NOFTZ-NEXT: fence.acq_rel.cta;
-; SM70-NOFTZ-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
-; SM70-NOFTZ-NEXT: $L__BB114_1: // %atomicrmw.start
-; SM70-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NOFTZ-NEXT: add.rn.f32 %r3, %r4, %r2;
-; SM70-NOFTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
-; SM70-NOFTZ-NEXT: setp.ne.b32 %p1, %r1, %r4;
-; SM70-NOFTZ-NEXT: mov.b32 %r4, %r1;
-; SM70-NOFTZ-NEXT: @%p1 bra $L__BB114_1;
-; SM70-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end
-; SM70-NOFTZ-NEXT: fence.acq_rel.cta;
-; SM70-NOFTZ-NEXT: st.param.b32 [func_retval0], %r1;
-; SM70-NOFTZ-NEXT: ret;
+; SM70-NOFTZ-DISALLOW-LABEL: fadd_acq_rel_float_global_cluster(
+; SM70-NOFTZ-DISALLOW: {
+; SM70-NOFTZ-DISALLOW-NEXT: .reg .pred %p<2>;
+; SM70-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<5>;
+; SM70-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
+; SM70-NOFTZ-DISALLOW-EMPTY:
+; SM70-NOFTZ-DISALLOW-NEXT: // %bb.0:
+; SM70-NOFTZ-DISALLOW-NEXT: ld.param.b32 %r2, [fadd_acq_rel_float_global_cluster_param_1];
+; SM70-NOFTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_cluster_param_0];
+; SM70-NOFTZ-DISALLOW-NEXT: fence.acq_rel.cta;
+; SM70-NOFTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM70-NOFTZ-DISALLOW-NEXT: $L__BB114_1: // %atomicrmw.start
+; SM70-NOFTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NOFTZ-DISALLOW-NEXT: add.rn.f32 %r3, %r4, %r2;
+; SM70-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM70-NOFTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM70-NOFTZ-DISALLOW-NEXT: mov.b32 %r4, %r1;
+; SM70-NOFTZ-DISALLOW-NEXT: @%p1 bra $L__BB114_1;
+; SM70-NOFTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NOFTZ-DISALLOW-NEXT: fence.acq_rel.cta;
+; SM70-NOFTZ-DISALLOW-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NOFTZ-DISALLOW-NEXT: ret;
;
-; SM70-FTZ-LABEL: fadd_acq_rel_float_global_cluster(
-; SM70-FTZ: {
-; SM70-FTZ-NEXT: .reg .b32 %r<3>;
-; SM70-FTZ-NEXT: .reg .b64 %rd<2>;
-; SM70-FTZ-EMPTY:
-; SM70-FTZ-NEXT: // %bb.0:
-; SM70-FTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_cluster_param_0];
-; SM70-FTZ-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_global_cluster_param_1];
-; SM70-FTZ-NEXT: atom.acq_rel.cta.global.add.f32 %r2, [%rd1], %r1;
-; SM70-FTZ-NEXT: st.param.b32 [func_retval0], %r2;
-; SM70-FTZ-NEXT: ret;
+; SM70-NOFTZ-ALLOW-LABEL: fadd_acq_rel_float_global_cluster(
+; SM70-NOFTZ-ALLOW: {
+; SM70-NOFTZ-ALLOW-NEXT: .reg .b32 %r<3>;
+; SM70-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
+; SM70-NOFTZ-ALLOW-EMPTY:
+; SM70-NOFTZ-ALLOW-NEXT: // %bb.0:
+; SM70-NOFTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_cluster_param_0];
+; SM70-NOFTZ-ALLOW-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_global_cluster_param_1];
+; SM70-NOFTZ-ALLOW-NEXT: atom.acq_rel.cta.global.add.f32 %r2, [%rd1], %r1;
+; SM70-NOFTZ-ALLOW-NEXT: st.param.b32 [func_retval0], %r2;
+; SM70-NOFTZ-ALLOW-NEXT: ret;
+;
+; SM70-FTZ-DISALLOW-LABEL: fadd_acq_rel_float_global_cluster(
+; SM70-FTZ-DISALLOW: {
+; SM70-FTZ-DISALLOW-NEXT: .reg .b32 %r<3>;
+; SM70-FTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
+; SM70-FTZ-DISALLOW-EMPTY:
+; SM70-FTZ-DISALLOW-NEXT: // %bb.0:
+; SM70-FTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_cluster_param_0];
+; SM70-FTZ-DISALLOW-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_global_cluster_param_1];
+; SM70-FTZ-DISALLOW-NEXT: atom.acq_rel.cta.global.add.f32 %r2, [%rd1], %r1;
+; SM70-FTZ-DISALLOW-NEXT: st.param.b32 [func_retval0], %r2;
+; SM70-FTZ-DISALLOW-NEXT: ret;
+;
+; SM70-FTZ-ALLOW-LABEL: fadd_acq_rel_float_global_cluster(
+; SM70-FTZ-ALLOW: {
+; SM70-FTZ-ALLOW-NEXT: .reg .b32 %r<3>;
+; SM70-FTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
+; SM70-FTZ-ALLOW-EMPTY:
+; SM70-FTZ-ALLOW-NEXT: // %bb.0:
+; SM70-FTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_cluster_param_0];
+; SM70-FTZ-ALLOW-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_global_cluster_param_1];
+; SM70-FTZ-ALLOW-NEXT: atom.acq_rel.cta.global.add.f32 %r2, [%rd1], %r1;
+; SM70-FTZ-ALLOW-NEXT: st.param.b32 [func_retval0], %r2;
+; SM70-FTZ-ALLOW-NEXT: ret;
%retval = atomicrmw fadd ptr addrspace(1) %addr, float %val syncscope("cluster") acq_rel
ret float %retval
}
@@ -4230,40 +5568,64 @@ define i64 @nand_acq_rel_i64_global_gpu(ptr addrspace(1) %addr, i64 %val) {
}
define float @fadd_acq_rel_float_global_gpu(ptr addrspace(1) %addr, float %val) {
-; SM70-NOFTZ-LABEL: fadd_acq_rel_float_global_gpu(
-; SM70-NOFTZ: {
-; SM70-NOFTZ-NEXT: .reg .pred %p<2>;
-; SM70-NOFTZ-NEXT: .reg .b32 %r<5>;
-; SM70-NOFTZ-NEXT: .reg .b64 %rd<2>;
-; SM70-NOFTZ-EMPTY:
-; SM70-NOFTZ-NEXT: // %bb.0:
-; SM70-NOFTZ-NEXT: ld.param.b32 %r2, [fadd_acq_rel_float_global_gpu_param_1];
-; SM70-NOFTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_gpu_param_0];
-; SM70-NOFTZ-NEXT: fence.acq_rel.gpu;
-; SM70-NOFTZ-NEXT: ld.relaxed.gpu.global.b32 %r4, [%rd1];
-; SM70-NOFTZ-NEXT: $L__BB122_1: // %atomicrmw.start
-; SM70-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NOFTZ-NEXT: add.rn.f32 %r3, %r4, %r2;
-; SM70-NOFTZ-NEXT: atom.relaxed.gpu.global.cas.b32 %r1, [%rd1], %r4, %r3;
-; SM70-NOFTZ-NEXT: setp.ne.b32 %p1, %r1, %r4;
-; SM70-NOFTZ-NEXT: mov.b32 %r4, %r1;
-; SM70-NOFTZ-NEXT: @%p1 bra $L__BB122_1;
-; SM70-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end
-; SM70-NOFTZ-NEXT: fence.acq_rel.gpu;
-; SM70-NOFTZ-NEXT: st.param.b32 [func_retval0], %r1;
-; SM70-NOFTZ-NEXT: ret;
+; SM70-NOFTZ-DISALLOW-LABEL: fadd_acq_rel_float_global_gpu(
+; SM70-NOFTZ-DISALLOW: {
+; SM70-NOFTZ-DISALLOW-NEXT: .reg .pred %p<2>;
+; SM70-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<5>;
+; SM70-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
+; SM70-NOFTZ-DISALLOW-EMPTY:
+; SM70-NOFTZ-DISALLOW-NEXT: // %bb.0:
+; SM70-NOFTZ-DISALLOW-NEXT: ld.param.b32 %r2, [fadd_acq_rel_float_global_gpu_param_1];
+; SM70-NOFTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_gpu_param_0];
+; SM70-NOFTZ-DISALLOW-NEXT: fence.acq_rel.gpu;
+; SM70-NOFTZ-DISALLOW-NEXT: ld.relaxed.gpu.global.b32 %r4, [%rd1];
+; SM70-NOFTZ-DISALLOW-NEXT: $L__BB122_1: // %atomicrmw.start
+; SM70-NOFTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NOFTZ-DISALLOW-NEXT: add.rn.f32 %r3, %r4, %r2;
+; SM70-NOFTZ-DISALLOW-NEXT: atom.relaxed.gpu.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM70-NOFTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM70-NOFTZ-DISALLOW-NEXT: mov.b32 %r4, %r1;
+; SM70-NOFTZ-DISALLOW-NEXT: @%p1 bra $L__BB122_1;
+; SM70-NOFTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NOFTZ-DISALLOW-NEXT: fence.acq_rel.gpu;
+; SM70-NOFTZ-DISALLOW-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NOFTZ-DISALLOW-NEXT: ret;
+;
+; SM70-NOFTZ-ALLOW-LABEL: fadd_acq_rel_float_global_gpu(
+; SM70-NOFTZ-ALLOW: {
+; SM70-NOFTZ-ALLOW-NEXT: .reg .b32 %r<3>;
+; SM70-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
+; SM70-NOFTZ-ALLOW-EMPTY:
+; SM70-NOFTZ-ALLOW-NEXT: // %bb.0:
+; SM70-NOFTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_gpu_param_0];
+; SM70-NOFTZ-ALLOW-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_global_gpu_param_1];
+; SM70-NOFTZ-ALLOW-NEXT: atom.acq_rel.gpu.global.add.f32 %r2, [%rd1], %r1;
+; SM70-NOFTZ-ALLOW-NEXT: st.param.b32 [func_retval0], %r2;
+; SM70-NOFTZ-ALLOW-NEXT: ret;
;
-; SM70-FTZ-LABEL: fadd_acq_rel_float_global_gpu(
-; SM70-FTZ: {
-; SM70-FTZ-NEXT: .reg .b32 %r<3>;
-; SM70-FTZ-NEXT: .reg .b64 %rd<2>;
-; SM70-FTZ-EMPTY:
-; SM70-FTZ-NEXT: // %bb.0:
-; SM70-FTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_gpu_param_0];
-; SM70-FTZ-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_global_gpu_param_1];
-; SM70-FTZ-NEXT: atom.acq_rel.gpu.global.add.f32 %r2, [%rd1], %r1;
-; SM70-FTZ-NEXT: st.param.b32 [func_retval0], %r2;
-; SM70-FTZ-NEXT: ret;
+; SM70-FTZ-DISALLOW-LABEL: fadd_acq_rel_float_global_gpu(
+; SM70-FTZ-DISALLOW: {
+; SM70-FTZ-DISALLOW-NEXT: .reg .b32 %r<3>;
+; SM70-FTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
+; SM70-FTZ-DISALLOW-EMPTY:
+; SM70-FTZ-DISALLOW-NEXT: // %bb.0:
+; SM70-FTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_gpu_param_0];
+; SM70-FTZ-DISALLOW-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_global_gpu_param_1];
+; SM70-FTZ-DISALLOW-NEXT: atom.acq_rel.gpu.global.add.f32 %r2, [%rd1], %r1;
+; SM70-FTZ-DISALLOW-NEXT: st.param.b32 [func_retval0], %r2;
+; SM70-FTZ-DISALLOW-NEXT: ret;
+;
+; SM70-FTZ-ALLOW-LABEL: fadd_acq_rel_float_global_gpu(
+; SM70-FTZ-ALLOW: {
+; SM70-FTZ-ALLOW-NEXT: .reg .b32 %r<3>;
+; SM70-FTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
+; SM70-FTZ-ALLOW-EMPTY:
+; SM70-FTZ-ALLOW-NEXT: // %bb.0:
+; SM70-FTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_gpu_param_0];
+; SM70-FTZ-ALLOW-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_global_gpu_param_1];
+; SM70-FTZ-ALLOW-NEXT: atom.acq_rel.gpu.global.add.f32 %r2, [%rd1], %r1;
+; SM70-FTZ-ALLOW-NEXT: st.param.b32 [func_retval0], %r2;
+; SM70-FTZ-ALLOW-NEXT: ret;
%retval = atomicrmw fadd ptr addrspace(1) %addr, float %val syncscope("device") acq_rel
ret float %retval
}
@@ -4402,51 +5764,75 @@ define i64 @nand_acq_rel_i64_generic_cta(ptr %addr, i64 %val) {
}
define float @fadd_acq_rel_float_generic_cta(ptr %addr, float %val) {
-; SM70-NOFTZ-LABEL: fadd_acq_rel_float_generic_cta(
-; SM70-NOFTZ: {
-; SM70-NOFTZ-NEXT: .reg .pred %p<2>;
-; SM70-NOFTZ-NEXT: .reg .b32 %r<5>;
-; SM70-NOFTZ-NEXT: .reg .b64 %rd<2>;
-; SM70-NOFTZ-EMPTY:
-; SM70-NOFTZ-NEXT: // %bb.0:
-; SM70-NOFTZ-NEXT: ld.param.b32 %r2, [fadd_acq_rel_float_generic_cta_param_1];
-; SM70-NOFTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_generic_cta_param_0];
-; SM70-NOFTZ-NEXT: fence.acq_rel.cta;
-; SM70-NOFTZ-NEXT: ld.relaxed.cta.b32 %r4, [%rd1];
-; SM70-NOFTZ-NEXT: $L__BB130_1: // %atomicrmw.start
-; SM70-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NOFTZ-NEXT: add.rn.f32 %r3, %r4, %r2;
-; SM70-NOFTZ-NEXT: atom.relaxed.cta.cas.b32 %r1, [%rd1], %r4, %r3;
-; SM70-NOFTZ-NEXT: setp.ne.b32 %p1, %r1, %r4;
-; SM70-NOFTZ-NEXT: mov.b32 %r4, %r1;
-; SM70-NOFTZ-NEXT: @%p1 bra $L__BB130_1;
-; SM70-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end
-; SM70-NOFTZ-NEXT: fence.acq_rel.cta;
-; SM70-NOFTZ-NEXT: st.param.b32 [func_retval0], %r1;
-; SM70-NOFTZ-NEXT: ret;
+; SM70-NOFTZ-DISALLOW-LABEL: fadd_acq_rel_float_generic_cta(
+; SM70-NOFTZ-DISALLOW: {
+; SM70-NOFTZ-DISALLOW-NEXT: .reg .pred %p<2>;
+; SM70-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<5>;
+; SM70-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
+; SM70-NOFTZ-DISALLOW-EMPTY:
+; SM70-NOFTZ-DISALLOW-NEXT: // %bb.0:
+; SM70-NOFTZ-DISALLOW-NEXT: ld.param.b32 %r2, [fadd_acq_rel_float_generic_cta_param_1];
+; SM70-NOFTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_generic_cta_param_0];
+; SM70-NOFTZ-DISALLOW-NEXT: fence.acq_rel.cta;
+; SM70-NOFTZ-DISALLOW-NEXT: ld.relaxed.cta.b32 %r4, [%rd1];
+; SM70-NOFTZ-DISALLOW-NEXT: $L__BB130_1: // %atomicrmw.start
+; SM70-NOFTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-NOFTZ-DISALLOW-NEXT: add.rn.f32 %r3, %r4, %r2;
+; SM70-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM70-NOFTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM70-NOFTZ-DISALLOW-NEXT: mov.b32 %r4, %r1;
+; SM70-NOFTZ-DISALLOW-NEXT: @%p1 bra $L__BB130_1;
+; SM70-NOFTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-NOFTZ-DISALLOW-NEXT: fence.acq_rel.cta;
+; SM70-NOFTZ-DISALLOW-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-NOFTZ-DISALLOW-NEXT: ret;
+;
+; SM70-NOFTZ-ALLOW-LABEL: fadd_acq_rel_float_generic_cta(
+; SM70-NOFTZ-ALLOW: {
+; SM70-NOFTZ-ALLOW-NEXT: .reg .b32 %r<3>;
+; SM70-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
+; SM70-NOFTZ-ALLOW-EMPTY:
+; SM70-NOFTZ-ALLOW-NEXT: // %bb.0:
+; SM70-NOFTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_generic_cta_param_0];
+; SM70-NOFTZ-ALLOW-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_generic_cta_param_1];
+; SM70-NOFTZ-ALLOW-NEXT: atom.acq_rel.cta.add.f32 %r2, [%rd1], %r1;
+; SM70-NOFTZ-ALLOW-NEXT: st.param.b32 [func_retval0], %r2;
+; SM70-NOFTZ-ALLOW-NEXT: ret;
+;
+; SM70-FTZ-DISALLOW-LABEL: fadd_acq_rel_float_generic_cta(
+; SM70-FTZ-DISALLOW: {
+; SM70-FTZ-DISALLOW-NEXT: .reg .pred %p<2>;
+; SM70-FTZ-DISALLOW-NEXT: .reg .b32 %r<5>;
+; SM70-FTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
+; SM70-FTZ-DISALLOW-EMPTY:
+; SM70-FTZ-DISALLOW-NEXT: // %bb.0:
+; SM70-FTZ-DISALLOW-NEXT: ld.param.b32 %r2, [fadd_acq_rel_float_generic_cta_param_1];
+; SM70-FTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_generic_cta_param_0];
+; SM70-FTZ-DISALLOW-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-DISALLOW-NEXT: ld.relaxed.cta.b32 %r4, [%rd1];
+; SM70-FTZ-DISALLOW-NEXT: $L__BB130_1: // %atomicrmw.start
+; SM70-FTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-FTZ-DISALLOW-NEXT: add.rn.ftz.f32 %r3, %r4, %r2;
+; SM70-FTZ-DISALLOW-NEXT: atom.relaxed.cta.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM70-FTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM70-FTZ-DISALLOW-NEXT: mov.b32 %r4, %r1;
+; SM70-FTZ-DISALLOW-NEXT: @%p1 bra $L__BB130_1;
+; SM70-FTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-FTZ-DISALLOW-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-DISALLOW-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-FTZ-DISALLOW-NEXT: ret;
;
-; SM70-FTZ-LABEL: fadd_acq_rel_float_generic_cta(
-; SM70-FTZ: {
-; SM70-FTZ-NEXT: .reg .pred %p<2>;
-; SM70-FTZ-NEXT: .reg .b32 %r<5>;
-; SM70-FTZ-NEXT: .reg .b64 %rd<2>;
-; SM70-FTZ-EMPTY:
-; SM70-FTZ-NEXT: // %bb.0:
-; SM70-FTZ-NEXT: ld.param.b32 %r2, [fadd_acq_rel_float_generic_cta_param_1];
-; SM70-FTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_generic_cta_param_0];
-; SM70-FTZ-NEXT: fence.acq_rel.cta;
-; SM70-FTZ-NEXT: ld.relaxed.cta.b32 %r4, [%rd1];
-; SM70-FTZ-NEXT: $L__BB130_1: // %atomicrmw.start
-; SM70-FTZ-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-FTZ-NEXT: add.rn.ftz.f32 %r3, %r4, %r2;
-; SM70-FTZ-NEXT: atom.relaxed.cta.cas.b32 %r1, [%rd1], %r4, %r3;
-; SM70-FTZ-NEXT: setp.ne.b32 %p1, %r1, %r4;
-; SM70-FTZ-NEXT: mov.b32 %r4, %r1;
-; SM70-FTZ-NEXT: @%p1 bra $L__BB130_1;
-; SM70-FTZ-NEXT: // %bb.2: // %atomicrmw.end
-; SM70-FTZ-NEXT: fence.acq_rel.cta;
-; SM70-FTZ-NEXT: st.param.b32 [func_retval0], %r1;
-; SM70-FTZ-NEXT: ret;
+; SM70-FTZ-ALLOW-LABEL: fadd_acq_rel_float_generic_cta(
+; SM70-FTZ-ALLOW: {
+; SM70-FTZ-ALLOW-NEXT: .reg .b32 %r<3>;
+; SM70-FTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
+; SM70-FTZ-ALLOW-EMPTY:
+; SM70-FTZ-ALLOW-NEXT: // %bb.0:
+; SM70-FTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_generic_cta_param_0];
+; SM70-FTZ-ALLOW-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_generic_cta_param_1];
+; SM70-FTZ-ALLOW-NEXT: atom.acq_rel.cta.add.f32 %r2, [%rd1], %r1;
+; SM70-FTZ-ALLOW-NEXT: st.param.b32 [func_retval0], %r2;
+; SM70-FTZ-ALLOW-NEXT: ret;
%retval = atomicrmw fadd ptr %addr, float %val syncscope("block") acq_rel
ret float %retval
}
@@ -4585,40 +5971,64 @@ define i64 @nand_acq_rel_i64_shared_cta(ptr addrspace(3) %addr, i64 %val) {
}
define float @fadd_acq_rel_float_shared_cta(ptr addrspace(3) %addr, float %val) {
-; SM70-NOFTZ-LABEL: fadd_acq_rel_float_shared_cta(
-; SM70-NOFTZ: {
-; SM70-NOFTZ-NEXT: .reg .b32 %r<3>;
-; SM70-NOFTZ-NEXT: .reg .b64 %rd<2>;
-; SM70-NOFTZ-EMPTY:
-; SM70-NOFTZ-NEXT: // %bb.0:
-; SM70-NOFTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_shared_cta_param_0];
-; SM70-NOFTZ-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_shared_cta_param_1];
-; SM70-NOFTZ-NEXT: atom.acq_rel.cta.shared.add.f32 %r2, [%rd1], %r1;
-; SM70-NOFTZ-NEXT: st.param.b32 [func_retval0], %r2;
-; SM70-NOFTZ-NEXT: ret;
+; SM70-NOFTZ-DISALLOW-LABEL: fadd_acq_rel_float_shared_cta(
+; SM70-NOFTZ-DISALLOW: {
+; SM70-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<3>;
+; SM70-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
+; SM70-NOFTZ-DISALLOW-EMPTY:
+; SM70-NOFTZ-DISALLOW-NEXT: // %bb.0:
+; SM70-NOFTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_shared_cta_param_0];
+; SM70-NOFTZ-DISALLOW-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_shared_cta_param_1];
+; SM70-NOFTZ-DISALLOW-NEXT: atom.acq_rel.cta.shared.add.f32 %r2, [%rd1], %r1;
+; SM70-NOFTZ-DISALLOW-NEXT: st.param.b32 [func_retval0], %r2;
+; SM70-NOFTZ-DISALLOW-NEXT: ret;
+;
+; SM70-NOFTZ-ALLOW-LABEL: fadd_acq_rel_float_shared_cta(
+; SM70-NOFTZ-ALLOW: {
+; SM70-NOFTZ-ALLOW-NEXT: .reg .b32 %r<3>;
+; SM70-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
+; SM70-NOFTZ-ALLOW-EMPTY:
+; SM70-NOFTZ-ALLOW-NEXT: // %bb.0:
+; SM70-NOFTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_shared_cta_param_0];
+; SM70-NOFTZ-ALLOW-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_shared_cta_param_1];
+; SM70-NOFTZ-ALLOW-NEXT: atom.acq_rel.cta.shared.add.f32 %r2, [%rd1], %r1;
+; SM70-NOFTZ-ALLOW-NEXT: st.param.b32 [func_retval0], %r2;
+; SM70-NOFTZ-ALLOW-NEXT: ret;
+;
+; SM70-FTZ-DISALLOW-LABEL: fadd_acq_rel_float_shared_cta(
+; SM70-FTZ-DISALLOW: {
+; SM70-FTZ-DISALLOW-NEXT: .reg .pred %p<2>;
+; SM70-FTZ-DISALLOW-NEXT: .reg .b32 %r<5>;
+; SM70-FTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
+; SM70-FTZ-DISALLOW-EMPTY:
+; SM70-FTZ-DISALLOW-NEXT: // %bb.0:
+; SM70-FTZ-DISALLOW-NEXT: ld.param.b32 %r2, [fadd_acq_rel_float_shared_cta_param_1];
+; SM70-FTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_shared_cta_param_0];
+; SM70-FTZ-DISALLOW-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-DISALLOW-NEXT: ld.relaxed.cta.shared.b32 %r4, [%rd1];
+; SM70-FTZ-DISALLOW-NEXT: $L__BB138_1: // %atomicrmw.start
+; SM70-FTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM70-FTZ-DISALLOW-NEXT: add.rn.ftz.f32 %r3, %r4, %r2;
+; SM70-FTZ-DISALLOW-NEXT: atom.relaxed.cta.shared.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM70-FTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM70-FTZ-DISALLOW-NEXT: mov.b32 %r4, %r1;
+; SM70-FTZ-DISALLOW-NEXT: @%p1 bra $L__BB138_1;
+; SM70-FTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM70-FTZ-DISALLOW-NEXT: fence.acq_rel.cta;
+; SM70-FTZ-DISALLOW-NEXT: st.param.b32 [func_retval0], %r1;
+; SM70-FTZ-DISALLOW-NEXT: ret;
;
-; SM70-FTZ-LABEL: fadd_acq_rel_float_shared_cta(
-; SM70-FTZ: {
-; SM70-FTZ-NEXT: .reg .pred %p<2>;
-; SM70-FTZ-NEXT: .reg .b32 %r<5>;
-; SM70-FTZ-NEXT: .reg .b64 %rd<2>;
-; SM70-FTZ-EMPTY:
-; SM70-FTZ-NEXT: // %bb.0:
-; SM70-FTZ-NEXT: ld.param.b32 %r2, [fadd_acq_rel_float_shared_cta_param_1];
-; SM70-FTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_shared_cta_param_0];
-; SM70-FTZ-NEXT: fence.acq_rel.cta;
-; SM70-FTZ-NEXT: ld.relaxed.cta.shared.b32 %r4, [%rd1];
-; SM70-FTZ-NEXT: $L__BB138_1: // %atomicrmw.start
-; SM70-FTZ-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-FTZ-NEXT: add.rn.ftz.f32 %r3, %r4, %r2;
-; SM70-FTZ-NEXT: atom.relaxed.cta.shared.cas.b32 %r1, [%rd1], %r4, %r3;
-; SM70-FTZ-NEXT: setp.ne.b32 %p1, %r1, %r4;
-; SM70-FTZ-NEXT: mov.b32 %r4, %r1;
-; SM70-FTZ-NEXT: @%p1 bra $L__BB138_1;
-; SM70-FTZ-NEXT: // %bb.2: // %atomicrmw.end
-; SM70-FTZ-NEXT: fence.acq_rel.cta;
-; SM70-FTZ-NEXT: st.param.b32 [func_retval0], %r1;
-; SM70-FTZ-NEXT: ret;
+; SM70-FTZ-ALLOW-LABEL: fadd_acq_rel_float_shared_cta(
+; SM70-FTZ-ALLOW: {
+; SM70-FTZ-ALLOW-NEXT: .reg .b32 %r<3>;
+; SM70-FTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
+; SM70-FTZ-ALLOW-EMPTY:
+; SM70-FTZ-ALLOW-NEXT: // %bb.0:
+; SM70-FTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_shared_cta_param_0];
+; SM70-FTZ-ALLOW-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_shared_cta_param_1];
+; SM70-FTZ-ALLOW-NEXT: atom.acq_rel.cta.shared.add.f32 %r2, [%rd1], %r1;
+; SM70-FTZ-ALLOW-NEXT: st.param.b32 [func_retval0], %r2;
+; SM70-FTZ-ALLOW-NEXT: ret;
%retval = atomicrmw fadd ptr addrspace(3) %addr, float %val syncscope("block") acq_rel
ret float %retval
}
diff --git a/llvm/test/CodeGen/NVPTX/atomicrmw-sm90.ll b/llvm/test/CodeGen/NVPTX/atomicrmw-sm90.ll
index 7bc834674ecc7..65e9daef2165d 100644
--- a/llvm/test/CodeGen/NVPTX/atomicrmw-sm90.ll
+++ b/llvm/test/CodeGen/NVPTX/atomicrmw-sm90.ll
@@ -1,7 +1,15 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
-; RUN: llc < %s -march=nvptx64 -mcpu=sm_90 -mattr=+ptx87 | FileCheck %s --check-prefixes=SM90,SM90-NOFTZ
-; RUN: llc < %s -march=nvptx64 -mcpu=sm_90 -mattr=+ptx87 -denormal-fp-math-f32=preserve-sign | FileCheck %s --check-prefixes=SM90,SM90-FTZ
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_90 -mattr=+ptx87 -nvptx-allow-ftz-atomics=false | FileCheck %s --check-prefixes=SM90,SM90-NOFTZ-DISALLOW
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_90 -mattr=+ptx87 -nvptx-allow-ftz-atomics=true | FileCheck %s --check-prefixes=SM90,SM90-NOFTZ-ALLOW
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_90 -mattr=+ptx87 | FileCheck %s --check-prefixes=SM90,SM90-NOFTZ-ALLOW
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_90 -mattr=+ptx87 -denormal-fp-math-f32=preserve-sign -nvptx-allow-ftz-atomics=false | FileCheck %s --check-prefixes=SM90,SM90-FTZ-DISALLOW
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_90 -mattr=+ptx87 -denormal-fp-math-f32=preserve-sign -nvptx-allow-ftz-atomics=true | FileCheck %s --check-prefixes=SM90,SM90-FTZ-ALLOW
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_90 -mattr=+ptx87 -denormal-fp-math-f32=preserve-sign | FileCheck %s --check-prefixes=SM90,SM90-FTZ-ALLOW
+; RUN: %if ptxas-sm_90 && ptxas-isa-8.7 %{ llc < %s -march=nvptx64 -mcpu=sm_90 -mattr=+ptx87 -nvptx-allow-ftz-atomics=false | %ptxas-verify -arch=sm_90 %}
+; RUN: %if ptxas-sm_90 && ptxas-isa-8.7 %{ llc < %s -march=nvptx64 -mcpu=sm_90 -mattr=+ptx87 -nvptx-allow-ftz-atomics=true | %ptxas-verify -arch=sm_90 %}
; RUN: %if ptxas-sm_90 && ptxas-isa-8.7 %{ llc < %s -march=nvptx64 -mcpu=sm_90 -mattr=+ptx87 | %ptxas-verify -arch=sm_90 %}
+; RUN: %if ptxas-sm_90 && ptxas-isa-8.7 %{ llc < %s -march=nvptx64 -mcpu=sm_90 -mattr=+ptx87 -denormal-fp-math-f32=preserve-sign -nvptx-allow-ftz-atomics=false | %ptxas-verify -arch=sm_90 %}
+; RUN: %if ptxas-sm_90 && ptxas-isa-8.7 %{ llc < %s -march=nvptx64 -mcpu=sm_90 -mattr=+ptx87 -denormal-fp-math-f32=preserve-sign -nvptx-allow-ftz-atomics=true | %ptxas-verify -arch=sm_90 %}
; RUN: %if ptxas-sm_90 && ptxas-isa-8.7 %{ llc < %s -march=nvptx64 -mcpu=sm_90 -mattr=+ptx87 -denormal-fp-math-f32=preserve-sign | %ptxas-verify -arch=sm_90 %}
define i8 @xchg_acq_rel_i8_global_cta(ptr addrspace(1) %addr, i8 %val) {
@@ -1739,290 +1747,544 @@ define i64 @usub_sat_acq_rel_i64_global_cta(ptr addrspace(1) %addr, i64 %val) {
}
define float @fadd_acq_rel_float_global_cta(ptr addrspace(1) %addr, float %val) {
-; SM90-NOFTZ-LABEL: fadd_acq_rel_float_global_cta(
-; SM90-NOFTZ: {
-; SM90-NOFTZ-NEXT: .reg .pred %p<2>;
-; SM90-NOFTZ-NEXT: .reg .b32 %r<5>;
-; SM90-NOFTZ-NEXT: .reg .b64 %rd<2>;
-; SM90-NOFTZ-EMPTY:
-; SM90-NOFTZ-NEXT: // %bb.0:
-; SM90-NOFTZ-NEXT: ld.param.b32 %r2, [fadd_acq_rel_float_global_cta_param_1];
-; SM90-NOFTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_cta_param_0];
-; SM90-NOFTZ-NEXT: fence.release.cta;
-; SM90-NOFTZ-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
-; SM90-NOFTZ-NEXT: $L__BB60_1: // %atomicrmw.start
-; SM90-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NOFTZ-NEXT: add.rn.f32 %r3, %r4, %r2;
-; SM90-NOFTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
-; SM90-NOFTZ-NEXT: setp.ne.b32 %p1, %r1, %r4;
-; SM90-NOFTZ-NEXT: mov.b32 %r4, %r1;
-; SM90-NOFTZ-NEXT: @%p1 bra $L__BB60_1;
-; SM90-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NOFTZ-NEXT: fence.acquire.cta;
-; SM90-NOFTZ-NEXT: st.param.b32 [func_retval0], %r1;
-; SM90-NOFTZ-NEXT: ret;
+; SM90-NOFTZ-DISALLOW-LABEL: fadd_acq_rel_float_global_cta(
+; SM90-NOFTZ-DISALLOW: {
+; SM90-NOFTZ-DISALLOW-NEXT: .reg .pred %p<2>;
+; SM90-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<5>;
+; SM90-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
+; SM90-NOFTZ-DISALLOW-EMPTY:
+; SM90-NOFTZ-DISALLOW-NEXT: // %bb.0:
+; SM90-NOFTZ-DISALLOW-NEXT: ld.param.b32 %r2, [fadd_acq_rel_float_global_cta_param_1];
+; SM90-NOFTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_cta_param_0];
+; SM90-NOFTZ-DISALLOW-NEXT: fence.release.cta;
+; SM90-NOFTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM90-NOFTZ-DISALLOW-NEXT: $L__BB60_1: // %atomicrmw.start
+; SM90-NOFTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NOFTZ-DISALLOW-NEXT: add.rn.f32 %r3, %r4, %r2;
+; SM90-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM90-NOFTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM90-NOFTZ-DISALLOW-NEXT: mov.b32 %r4, %r1;
+; SM90-NOFTZ-DISALLOW-NEXT: @%p1 bra $L__BB60_1;
+; SM90-NOFTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NOFTZ-DISALLOW-NEXT: fence.acquire.cta;
+; SM90-NOFTZ-DISALLOW-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NOFTZ-DISALLOW-NEXT: ret;
;
-; SM90-FTZ-LABEL: fadd_acq_rel_float_global_cta(
-; SM90-FTZ: {
-; SM90-FTZ-NEXT: .reg .b32 %r<3>;
-; SM90-FTZ-NEXT: .reg .b64 %rd<2>;
-; SM90-FTZ-EMPTY:
-; SM90-FTZ-NEXT: // %bb.0:
-; SM90-FTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_cta_param_0];
-; SM90-FTZ-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_global_cta_param_1];
-; SM90-FTZ-NEXT: atom.acq_rel.cta.global.add.f32 %r2, [%rd1], %r1;
-; SM90-FTZ-NEXT: st.param.b32 [func_retval0], %r2;
-; SM90-FTZ-NEXT: ret;
+; SM90-NOFTZ-ALLOW-LABEL: fadd_acq_rel_float_global_cta(
+; SM90-NOFTZ-ALLOW: {
+; SM90-NOFTZ-ALLOW-NEXT: .reg .b32 %r<3>;
+; SM90-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
+; SM90-NOFTZ-ALLOW-EMPTY:
+; SM90-NOFTZ-ALLOW-NEXT: // %bb.0:
+; SM90-NOFTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_cta_param_0];
+; SM90-NOFTZ-ALLOW-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_global_cta_param_1];
+; SM90-NOFTZ-ALLOW-NEXT: atom.acq_rel.cta.global.add.f32 %r2, [%rd1], %r1;
+; SM90-NOFTZ-ALLOW-NEXT: st.param.b32 [func_retval0], %r2;
+; SM90-NOFTZ-ALLOW-NEXT: ret;
+;
+; SM90-FTZ-DISALLOW-LABEL: fadd_acq_rel_float_global_cta(
+; SM90-FTZ-DISALLOW: {
+; SM90-FTZ-DISALLOW-NEXT: .reg .b32 %r<3>;
+; SM90-FTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
+; SM90-FTZ-DISALLOW-EMPTY:
+; SM90-FTZ-DISALLOW-NEXT: // %bb.0:
+; SM90-FTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_cta_param_0];
+; SM90-FTZ-DISALLOW-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_global_cta_param_1];
+; SM90-FTZ-DISALLOW-NEXT: atom.acq_rel.cta.global.add.f32 %r2, [%rd1], %r1;
+; SM90-FTZ-DISALLOW-NEXT: st.param.b32 [func_retval0], %r2;
+; SM90-FTZ-DISALLOW-NEXT: ret;
+;
+; SM90-FTZ-ALLOW-LABEL: fadd_acq_rel_float_global_cta(
+; SM90-FTZ-ALLOW: {
+; SM90-FTZ-ALLOW-NEXT: .reg .b32 %r<3>;
+; SM90-FTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
+; SM90-FTZ-ALLOW-EMPTY:
+; SM90-FTZ-ALLOW-NEXT: // %bb.0:
+; SM90-FTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_cta_param_0];
+; SM90-FTZ-ALLOW-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_global_cta_param_1];
+; SM90-FTZ-ALLOW-NEXT: atom.acq_rel.cta.global.add.f32 %r2, [%rd1], %r1;
+; SM90-FTZ-ALLOW-NEXT: st.param.b32 [func_retval0], %r2;
+; SM90-FTZ-ALLOW-NEXT: ret;
%retval = atomicrmw fadd ptr addrspace(1) %addr, float %val syncscope("block") acq_rel
ret float %retval
}
define float @fsub_acq_rel_float_global_cta(ptr addrspace(1) %addr, float %val) {
-; SM90-NOFTZ-LABEL: fsub_acq_rel_float_global_cta(
-; SM90-NOFTZ: {
-; SM90-NOFTZ-NEXT: .reg .pred %p<2>;
-; SM90-NOFTZ-NEXT: .reg .b32 %r<5>;
-; SM90-NOFTZ-NEXT: .reg .b64 %rd<2>;
-; SM90-NOFTZ-EMPTY:
-; SM90-NOFTZ-NEXT: // %bb.0:
-; SM90-NOFTZ-NEXT: ld.param.b32 %r2, [fsub_acq_rel_float_global_cta_param_1];
-; SM90-NOFTZ-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_float_global_cta_param_0];
-; SM90-NOFTZ-NEXT: fence.release.cta;
-; SM90-NOFTZ-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
-; SM90-NOFTZ-NEXT: $L__BB61_1: // %atomicrmw.start
-; SM90-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NOFTZ-NEXT: sub.rn.f32 %r3, %r4, %r2;
-; SM90-NOFTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
-; SM90-NOFTZ-NEXT: setp.ne.b32 %p1, %r1, %r4;
-; SM90-NOFTZ-NEXT: mov.b32 %r4, %r1;
-; SM90-NOFTZ-NEXT: @%p1 bra $L__BB61_1;
-; SM90-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NOFTZ-NEXT: fence.acquire.cta;
-; SM90-NOFTZ-NEXT: st.param.b32 [func_retval0], %r1;
-; SM90-NOFTZ-NEXT: ret;
+; SM90-NOFTZ-DISALLOW-LABEL: fsub_acq_rel_float_global_cta(
+; SM90-NOFTZ-DISALLOW: {
+; SM90-NOFTZ-DISALLOW-NEXT: .reg .pred %p<2>;
+; SM90-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<5>;
+; SM90-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
+; SM90-NOFTZ-DISALLOW-EMPTY:
+; SM90-NOFTZ-DISALLOW-NEXT: // %bb.0:
+; SM90-NOFTZ-DISALLOW-NEXT: ld.param.b32 %r2, [fsub_acq_rel_float_global_cta_param_1];
+; SM90-NOFTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_float_global_cta_param_0];
+; SM90-NOFTZ-DISALLOW-NEXT: fence.release.cta;
+; SM90-NOFTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM90-NOFTZ-DISALLOW-NEXT: $L__BB61_1: // %atomicrmw.start
+; SM90-NOFTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NOFTZ-DISALLOW-NEXT: sub.rn.f32 %r3, %r4, %r2;
+; SM90-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM90-NOFTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM90-NOFTZ-DISALLOW-NEXT: mov.b32 %r4, %r1;
+; SM90-NOFTZ-DISALLOW-NEXT: @%p1 bra $L__BB61_1;
+; SM90-NOFTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NOFTZ-DISALLOW-NEXT: fence.acquire.cta;
+; SM90-NOFTZ-DISALLOW-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NOFTZ-DISALLOW-NEXT: ret;
+;
+; SM90-NOFTZ-ALLOW-LABEL: fsub_acq_rel_float_global_cta(
+; SM90-NOFTZ-ALLOW: {
+; SM90-NOFTZ-ALLOW-NEXT: .reg .pred %p<2>;
+; SM90-NOFTZ-ALLOW-NEXT: .reg .b32 %r<5>;
+; SM90-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
+; SM90-NOFTZ-ALLOW-EMPTY:
+; SM90-NOFTZ-ALLOW-NEXT: // %bb.0:
+; SM90-NOFTZ-ALLOW-NEXT: ld.param.b32 %r2, [fsub_acq_rel_float_global_cta_param_1];
+; SM90-NOFTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_float_global_cta_param_0];
+; SM90-NOFTZ-ALLOW-NEXT: fence.release.cta;
+; SM90-NOFTZ-ALLOW-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM90-NOFTZ-ALLOW-NEXT: $L__BB61_1: // %atomicrmw.start
+; SM90-NOFTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NOFTZ-ALLOW-NEXT: sub.rn.f32 %r3, %r4, %r2;
+; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM90-NOFTZ-ALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM90-NOFTZ-ALLOW-NEXT: mov.b32 %r4, %r1;
+; SM90-NOFTZ-ALLOW-NEXT: @%p1 bra $L__BB61_1;
+; SM90-NOFTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NOFTZ-ALLOW-NEXT: fence.acquire.cta;
+; SM90-NOFTZ-ALLOW-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NOFTZ-ALLOW-NEXT: ret;
;
-; SM90-FTZ-LABEL: fsub_acq_rel_float_global_cta(
-; SM90-FTZ: {
-; SM90-FTZ-NEXT: .reg .pred %p<2>;
-; SM90-FTZ-NEXT: .reg .b32 %r<5>;
-; SM90-FTZ-NEXT: .reg .b64 %rd<2>;
-; SM90-FTZ-EMPTY:
-; SM90-FTZ-NEXT: // %bb.0:
-; SM90-FTZ-NEXT: ld.param.b32 %r2, [fsub_acq_rel_float_global_cta_param_1];
-; SM90-FTZ-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_float_global_cta_param_0];
-; SM90-FTZ-NEXT: fence.release.cta;
-; SM90-FTZ-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
-; SM90-FTZ-NEXT: $L__BB61_1: // %atomicrmw.start
-; SM90-FTZ-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-FTZ-NEXT: sub.rn.ftz.f32 %r3, %r4, %r2;
-; SM90-FTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
-; SM90-FTZ-NEXT: setp.ne.b32 %p1, %r1, %r4;
-; SM90-FTZ-NEXT: mov.b32 %r4, %r1;
-; SM90-FTZ-NEXT: @%p1 bra $L__BB61_1;
-; SM90-FTZ-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-FTZ-NEXT: fence.acquire.cta;
-; SM90-FTZ-NEXT: st.param.b32 [func_retval0], %r1;
-; SM90-FTZ-NEXT: ret;
+; SM90-FTZ-DISALLOW-LABEL: fsub_acq_rel_float_global_cta(
+; SM90-FTZ-DISALLOW: {
+; SM90-FTZ-DISALLOW-NEXT: .reg .pred %p<2>;
+; SM90-FTZ-DISALLOW-NEXT: .reg .b32 %r<5>;
+; SM90-FTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
+; SM90-FTZ-DISALLOW-EMPTY:
+; SM90-FTZ-DISALLOW-NEXT: // %bb.0:
+; SM90-FTZ-DISALLOW-NEXT: ld.param.b32 %r2, [fsub_acq_rel_float_global_cta_param_1];
+; SM90-FTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_float_global_cta_param_0];
+; SM90-FTZ-DISALLOW-NEXT: fence.release.cta;
+; SM90-FTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM90-FTZ-DISALLOW-NEXT: $L__BB61_1: // %atomicrmw.start
+; SM90-FTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-FTZ-DISALLOW-NEXT: sub.rn.ftz.f32 %r3, %r4, %r2;
+; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM90-FTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM90-FTZ-DISALLOW-NEXT: mov.b32 %r4, %r1;
+; SM90-FTZ-DISALLOW-NEXT: @%p1 bra $L__BB61_1;
+; SM90-FTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-FTZ-DISALLOW-NEXT: fence.acquire.cta;
+; SM90-FTZ-DISALLOW-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-FTZ-DISALLOW-NEXT: ret;
+;
+; SM90-FTZ-ALLOW-LABEL: fsub_acq_rel_float_global_cta(
+; SM90-FTZ-ALLOW: {
+; SM90-FTZ-ALLOW-NEXT: .reg .pred %p<2>;
+; SM90-FTZ-ALLOW-NEXT: .reg .b32 %r<5>;
+; SM90-FTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
+; SM90-FTZ-ALLOW-EMPTY:
+; SM90-FTZ-ALLOW-NEXT: // %bb.0:
+; SM90-FTZ-ALLOW-NEXT: ld.param.b32 %r2, [fsub_acq_rel_float_global_cta_param_1];
+; SM90-FTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_float_global_cta_param_0];
+; SM90-FTZ-ALLOW-NEXT: fence.release.cta;
+; SM90-FTZ-ALLOW-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM90-FTZ-ALLOW-NEXT: $L__BB61_1: // %atomicrmw.start
+; SM90-FTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-FTZ-ALLOW-NEXT: sub.rn.ftz.f32 %r3, %r4, %r2;
+; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM90-FTZ-ALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM90-FTZ-ALLOW-NEXT: mov.b32 %r4, %r1;
+; SM90-FTZ-ALLOW-NEXT: @%p1 bra $L__BB61_1;
+; SM90-FTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-FTZ-ALLOW-NEXT: fence.acquire.cta;
+; SM90-FTZ-ALLOW-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-FTZ-ALLOW-NEXT: ret;
%retval = atomicrmw fsub ptr addrspace(1) %addr, float %val syncscope("block") acq_rel
ret float %retval
}
define float @fmin_acq_rel_float_global_cta(ptr addrspace(1) %addr, float %val) {
-; SM90-NOFTZ-LABEL: fmin_acq_rel_float_global_cta(
-; SM90-NOFTZ: {
-; SM90-NOFTZ-NEXT: .reg .pred %p<2>;
-; SM90-NOFTZ-NEXT: .reg .b32 %r<5>;
-; SM90-NOFTZ-NEXT: .reg .b64 %rd<2>;
-; SM90-NOFTZ-EMPTY:
-; SM90-NOFTZ-NEXT: // %bb.0:
-; SM90-NOFTZ-NEXT: ld.param.b32 %r2, [fmin_acq_rel_float_global_cta_param_1];
-; SM90-NOFTZ-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_float_global_cta_param_0];
-; SM90-NOFTZ-NEXT: fence.release.cta;
-; SM90-NOFTZ-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
-; SM90-NOFTZ-NEXT: $L__BB62_1: // %atomicrmw.start
-; SM90-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NOFTZ-NEXT: min.f32 %r3, %r4, %r2;
-; SM90-NOFTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
-; SM90-NOFTZ-NEXT: setp.ne.b32 %p1, %r1, %r4;
-; SM90-NOFTZ-NEXT: mov.b32 %r4, %r1;
-; SM90-NOFTZ-NEXT: @%p1 bra $L__BB62_1;
-; SM90-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NOFTZ-NEXT: fence.acquire.cta;
-; SM90-NOFTZ-NEXT: st.param.b32 [func_retval0], %r1;
-; SM90-NOFTZ-NEXT: ret;
+; SM90-NOFTZ-DISALLOW-LABEL: fmin_acq_rel_float_global_cta(
+; SM90-NOFTZ-DISALLOW: {
+; SM90-NOFTZ-DISALLOW-NEXT: .reg .pred %p<2>;
+; SM90-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<5>;
+; SM90-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
+; SM90-NOFTZ-DISALLOW-EMPTY:
+; SM90-NOFTZ-DISALLOW-NEXT: // %bb.0:
+; SM90-NOFTZ-DISALLOW-NEXT: ld.param.b32 %r2, [fmin_acq_rel_float_global_cta_param_1];
+; SM90-NOFTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_float_global_cta_param_0];
+; SM90-NOFTZ-DISALLOW-NEXT: fence.release.cta;
+; SM90-NOFTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM90-NOFTZ-DISALLOW-NEXT: $L__BB62_1: // %atomicrmw.start
+; SM90-NOFTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NOFTZ-DISALLOW-NEXT: min.f32 %r3, %r4, %r2;
+; SM90-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM90-NOFTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM90-NOFTZ-DISALLOW-NEXT: mov.b32 %r4, %r1;
+; SM90-NOFTZ-DISALLOW-NEXT: @%p1 bra $L__BB62_1;
+; SM90-NOFTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NOFTZ-DISALLOW-NEXT: fence.acquire.cta;
+; SM90-NOFTZ-DISALLOW-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NOFTZ-DISALLOW-NEXT: ret;
+;
+; SM90-NOFTZ-ALLOW-LABEL: fmin_acq_rel_float_global_cta(
+; SM90-NOFTZ-ALLOW: {
+; SM90-NOFTZ-ALLOW-NEXT: .reg .pred %p<2>;
+; SM90-NOFTZ-ALLOW-NEXT: .reg .b32 %r<5>;
+; SM90-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
+; SM90-NOFTZ-ALLOW-EMPTY:
+; SM90-NOFTZ-ALLOW-NEXT: // %bb.0:
+; SM90-NOFTZ-ALLOW-NEXT: ld.param.b32 %r2, [fmin_acq_rel_float_global_cta_param_1];
+; SM90-NOFTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_float_global_cta_param_0];
+; SM90-NOFTZ-ALLOW-NEXT: fence.release.cta;
+; SM90-NOFTZ-ALLOW-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM90-NOFTZ-ALLOW-NEXT: $L__BB62_1: // %atomicrmw.start
+; SM90-NOFTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NOFTZ-ALLOW-NEXT: min.f32 %r3, %r4, %r2;
+; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM90-NOFTZ-ALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM90-NOFTZ-ALLOW-NEXT: mov.b32 %r4, %r1;
+; SM90-NOFTZ-ALLOW-NEXT: @%p1 bra $L__BB62_1;
+; SM90-NOFTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NOFTZ-ALLOW-NEXT: fence.acquire.cta;
+; SM90-NOFTZ-ALLOW-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NOFTZ-ALLOW-NEXT: ret;
;
-; SM90-FTZ-LABEL: fmin_acq_rel_float_global_cta(
-; SM90-FTZ: {
-; SM90-FTZ-NEXT: .reg .pred %p<2>;
-; SM90-FTZ-NEXT: .reg .b32 %r<5>;
-; SM90-FTZ-NEXT: .reg .b64 %rd<2>;
-; SM90-FTZ-EMPTY:
-; SM90-FTZ-NEXT: // %bb.0:
-; SM90-FTZ-NEXT: ld.param.b32 %r2, [fmin_acq_rel_float_global_cta_param_1];
-; SM90-FTZ-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_float_global_cta_param_0];
-; SM90-FTZ-NEXT: fence.release.cta;
-; SM90-FTZ-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
-; SM90-FTZ-NEXT: $L__BB62_1: // %atomicrmw.start
-; SM90-FTZ-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-FTZ-NEXT: min.ftz.f32 %r3, %r4, %r2;
-; SM90-FTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
-; SM90-FTZ-NEXT: setp.ne.b32 %p1, %r1, %r4;
-; SM90-FTZ-NEXT: mov.b32 %r4, %r1;
-; SM90-FTZ-NEXT: @%p1 bra $L__BB62_1;
-; SM90-FTZ-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-FTZ-NEXT: fence.acquire.cta;
-; SM90-FTZ-NEXT: st.param.b32 [func_retval0], %r1;
-; SM90-FTZ-NEXT: ret;
+; SM90-FTZ-DISALLOW-LABEL: fmin_acq_rel_float_global_cta(
+; SM90-FTZ-DISALLOW: {
+; SM90-FTZ-DISALLOW-NEXT: .reg .pred %p<2>;
+; SM90-FTZ-DISALLOW-NEXT: .reg .b32 %r<5>;
+; SM90-FTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
+; SM90-FTZ-DISALLOW-EMPTY:
+; SM90-FTZ-DISALLOW-NEXT: // %bb.0:
+; SM90-FTZ-DISALLOW-NEXT: ld.param.b32 %r2, [fmin_acq_rel_float_global_cta_param_1];
+; SM90-FTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_float_global_cta_param_0];
+; SM90-FTZ-DISALLOW-NEXT: fence.release.cta;
+; SM90-FTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM90-FTZ-DISALLOW-NEXT: $L__BB62_1: // %atomicrmw.start
+; SM90-FTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-FTZ-DISALLOW-NEXT: min.ftz.f32 %r3, %r4, %r2;
+; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM90-FTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM90-FTZ-DISALLOW-NEXT: mov.b32 %r4, %r1;
+; SM90-FTZ-DISALLOW-NEXT: @%p1 bra $L__BB62_1;
+; SM90-FTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-FTZ-DISALLOW-NEXT: fence.acquire.cta;
+; SM90-FTZ-DISALLOW-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-FTZ-DISALLOW-NEXT: ret;
+;
+; SM90-FTZ-ALLOW-LABEL: fmin_acq_rel_float_global_cta(
+; SM90-FTZ-ALLOW: {
+; SM90-FTZ-ALLOW-NEXT: .reg .pred %p<2>;
+; SM90-FTZ-ALLOW-NEXT: .reg .b32 %r<5>;
+; SM90-FTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
+; SM90-FTZ-ALLOW-EMPTY:
+; SM90-FTZ-ALLOW-NEXT: // %bb.0:
+; SM90-FTZ-ALLOW-NEXT: ld.param.b32 %r2, [fmin_acq_rel_float_global_cta_param_1];
+; SM90-FTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_float_global_cta_param_0];
+; SM90-FTZ-ALLOW-NEXT: fence.release.cta;
+; SM90-FTZ-ALLOW-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM90-FTZ-ALLOW-NEXT: $L__BB62_1: // %atomicrmw.start
+; SM90-FTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-FTZ-ALLOW-NEXT: min.ftz.f32 %r3, %r4, %r2;
+; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM90-FTZ-ALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM90-FTZ-ALLOW-NEXT: mov.b32 %r4, %r1;
+; SM90-FTZ-ALLOW-NEXT: @%p1 bra $L__BB62_1;
+; SM90-FTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-FTZ-ALLOW-NEXT: fence.acquire.cta;
+; SM90-FTZ-ALLOW-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-FTZ-ALLOW-NEXT: ret;
%retval = atomicrmw fmin ptr addrspace(1) %addr, float %val syncscope("block") acq_rel
ret float %retval
}
define float @fmax_acq_rel_float_global_cta(ptr addrspace(1) %addr, float %val) {
-; SM90-NOFTZ-LABEL: fmax_acq_rel_float_global_cta(
-; SM90-NOFTZ: {
-; SM90-NOFTZ-NEXT: .reg .pred %p<2>;
-; SM90-NOFTZ-NEXT: .reg .b32 %r<5>;
-; SM90-NOFTZ-NEXT: .reg .b64 %rd<2>;
-; SM90-NOFTZ-EMPTY:
-; SM90-NOFTZ-NEXT: // %bb.0:
-; SM90-NOFTZ-NEXT: ld.param.b32 %r2, [fmax_acq_rel_float_global_cta_param_1];
-; SM90-NOFTZ-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_float_global_cta_param_0];
-; SM90-NOFTZ-NEXT: fence.release.cta;
-; SM90-NOFTZ-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
-; SM90-NOFTZ-NEXT: $L__BB63_1: // %atomicrmw.start
-; SM90-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NOFTZ-NEXT: max.f32 %r3, %r4, %r2;
-; SM90-NOFTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
-; SM90-NOFTZ-NEXT: setp.ne.b32 %p1, %r1, %r4;
-; SM90-NOFTZ-NEXT: mov.b32 %r4, %r1;
-; SM90-NOFTZ-NEXT: @%p1 bra $L__BB63_1;
-; SM90-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NOFTZ-NEXT: fence.acquire.cta;
-; SM90-NOFTZ-NEXT: st.param.b32 [func_retval0], %r1;
-; SM90-NOFTZ-NEXT: ret;
+; SM90-NOFTZ-DISALLOW-LABEL: fmax_acq_rel_float_global_cta(
+; SM90-NOFTZ-DISALLOW: {
+; SM90-NOFTZ-DISALLOW-NEXT: .reg .pred %p<2>;
+; SM90-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<5>;
+; SM90-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
+; SM90-NOFTZ-DISALLOW-EMPTY:
+; SM90-NOFTZ-DISALLOW-NEXT: // %bb.0:
+; SM90-NOFTZ-DISALLOW-NEXT: ld.param.b32 %r2, [fmax_acq_rel_float_global_cta_param_1];
+; SM90-NOFTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_float_global_cta_param_0];
+; SM90-NOFTZ-DISALLOW-NEXT: fence.release.cta;
+; SM90-NOFTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM90-NOFTZ-DISALLOW-NEXT: $L__BB63_1: // %atomicrmw.start
+; SM90-NOFTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NOFTZ-DISALLOW-NEXT: max.f32 %r3, %r4, %r2;
+; SM90-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM90-NOFTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM90-NOFTZ-DISALLOW-NEXT: mov.b32 %r4, %r1;
+; SM90-NOFTZ-DISALLOW-NEXT: @%p1 bra $L__BB63_1;
+; SM90-NOFTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NOFTZ-DISALLOW-NEXT: fence.acquire.cta;
+; SM90-NOFTZ-DISALLOW-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NOFTZ-DISALLOW-NEXT: ret;
+;
+; SM90-NOFTZ-ALLOW-LABEL: fmax_acq_rel_float_global_cta(
+; SM90-NOFTZ-ALLOW: {
+; SM90-NOFTZ-ALLOW-NEXT: .reg .pred %p<2>;
+; SM90-NOFTZ-ALLOW-NEXT: .reg .b32 %r<5>;
+; SM90-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
+; SM90-NOFTZ-ALLOW-EMPTY:
+; SM90-NOFTZ-ALLOW-NEXT: // %bb.0:
+; SM90-NOFTZ-ALLOW-NEXT: ld.param.b32 %r2, [fmax_acq_rel_float_global_cta_param_1];
+; SM90-NOFTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_float_global_cta_param_0];
+; SM90-NOFTZ-ALLOW-NEXT: fence.release.cta;
+; SM90-NOFTZ-ALLOW-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM90-NOFTZ-ALLOW-NEXT: $L__BB63_1: // %atomicrmw.start
+; SM90-NOFTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NOFTZ-ALLOW-NEXT: max.f32 %r3, %r4, %r2;
+; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM90-NOFTZ-ALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM90-NOFTZ-ALLOW-NEXT: mov.b32 %r4, %r1;
+; SM90-NOFTZ-ALLOW-NEXT: @%p1 bra $L__BB63_1;
+; SM90-NOFTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NOFTZ-ALLOW-NEXT: fence.acquire.cta;
+; SM90-NOFTZ-ALLOW-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NOFTZ-ALLOW-NEXT: ret;
;
-; SM90-FTZ-LABEL: fmax_acq_rel_float_global_cta(
-; SM90-FTZ: {
-; SM90-FTZ-NEXT: .reg .pred %p<2>;
-; SM90-FTZ-NEXT: .reg .b32 %r<5>;
-; SM90-FTZ-NEXT: .reg .b64 %rd<2>;
-; SM90-FTZ-EMPTY:
-; SM90-FTZ-NEXT: // %bb.0:
-; SM90-FTZ-NEXT: ld.param.b32 %r2, [fmax_acq_rel_float_global_cta_param_1];
-; SM90-FTZ-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_float_global_cta_param_0];
-; SM90-FTZ-NEXT: fence.release.cta;
-; SM90-FTZ-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
-; SM90-FTZ-NEXT: $L__BB63_1: // %atomicrmw.start
-; SM90-FTZ-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-FTZ-NEXT: max.ftz.f32 %r3, %r4, %r2;
-; SM90-FTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
-; SM90-FTZ-NEXT: setp.ne.b32 %p1, %r1, %r4;
-; SM90-FTZ-NEXT: mov.b32 %r4, %r1;
-; SM90-FTZ-NEXT: @%p1 bra $L__BB63_1;
-; SM90-FTZ-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-FTZ-NEXT: fence.acquire.cta;
-; SM90-FTZ-NEXT: st.param.b32 [func_retval0], %r1;
-; SM90-FTZ-NEXT: ret;
+; SM90-FTZ-DISALLOW-LABEL: fmax_acq_rel_float_global_cta(
+; SM90-FTZ-DISALLOW: {
+; SM90-FTZ-DISALLOW-NEXT: .reg .pred %p<2>;
+; SM90-FTZ-DISALLOW-NEXT: .reg .b32 %r<5>;
+; SM90-FTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
+; SM90-FTZ-DISALLOW-EMPTY:
+; SM90-FTZ-DISALLOW-NEXT: // %bb.0:
+; SM90-FTZ-DISALLOW-NEXT: ld.param.b32 %r2, [fmax_acq_rel_float_global_cta_param_1];
+; SM90-FTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_float_global_cta_param_0];
+; SM90-FTZ-DISALLOW-NEXT: fence.release.cta;
+; SM90-FTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM90-FTZ-DISALLOW-NEXT: $L__BB63_1: // %atomicrmw.start
+; SM90-FTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-FTZ-DISALLOW-NEXT: max.ftz.f32 %r3, %r4, %r2;
+; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM90-FTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM90-FTZ-DISALLOW-NEXT: mov.b32 %r4, %r1;
+; SM90-FTZ-DISALLOW-NEXT: @%p1 bra $L__BB63_1;
+; SM90-FTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-FTZ-DISALLOW-NEXT: fence.acquire.cta;
+; SM90-FTZ-DISALLOW-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-FTZ-DISALLOW-NEXT: ret;
+;
+; SM90-FTZ-ALLOW-LABEL: fmax_acq_rel_float_global_cta(
+; SM90-FTZ-ALLOW: {
+; SM90-FTZ-ALLOW-NEXT: .reg .pred %p<2>;
+; SM90-FTZ-ALLOW-NEXT: .reg .b32 %r<5>;
+; SM90-FTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
+; SM90-FTZ-ALLOW-EMPTY:
+; SM90-FTZ-ALLOW-NEXT: // %bb.0:
+; SM90-FTZ-ALLOW-NEXT: ld.param.b32 %r2, [fmax_acq_rel_float_global_cta_param_1];
+; SM90-FTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_float_global_cta_param_0];
+; SM90-FTZ-ALLOW-NEXT: fence.release.cta;
+; SM90-FTZ-ALLOW-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM90-FTZ-ALLOW-NEXT: $L__BB63_1: // %atomicrmw.start
+; SM90-FTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-FTZ-ALLOW-NEXT: max.ftz.f32 %r3, %r4, %r2;
+; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM90-FTZ-ALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM90-FTZ-ALLOW-NEXT: mov.b32 %r4, %r1;
+; SM90-FTZ-ALLOW-NEXT: @%p1 bra $L__BB63_1;
+; SM90-FTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-FTZ-ALLOW-NEXT: fence.acquire.cta;
+; SM90-FTZ-ALLOW-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-FTZ-ALLOW-NEXT: ret;
%retval = atomicrmw fmax ptr addrspace(1) %addr, float %val syncscope("block") acq_rel
ret float %retval
}
define float @fminimum_acq_rel_float_global_cta(ptr addrspace(1) %addr, float %val) {
-; SM90-NOFTZ-LABEL: fminimum_acq_rel_float_global_cta(
-; SM90-NOFTZ: {
-; SM90-NOFTZ-NEXT: .reg .pred %p<2>;
-; SM90-NOFTZ-NEXT: .reg .b32 %r<5>;
-; SM90-NOFTZ-NEXT: .reg .b64 %rd<2>;
-; SM90-NOFTZ-EMPTY:
-; SM90-NOFTZ-NEXT: // %bb.0:
-; SM90-NOFTZ-NEXT: ld.param.b32 %r2, [fminimum_acq_rel_float_global_cta_param_1];
-; SM90-NOFTZ-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_float_global_cta_param_0];
-; SM90-NOFTZ-NEXT: fence.release.cta;
-; SM90-NOFTZ-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
-; SM90-NOFTZ-NEXT: $L__BB64_1: // %atomicrmw.start
-; SM90-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NOFTZ-NEXT: min.NaN.f32 %r3, %r4, %r2;
-; SM90-NOFTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
-; SM90-NOFTZ-NEXT: setp.ne.b32 %p1, %r1, %r4;
-; SM90-NOFTZ-NEXT: mov.b32 %r4, %r1;
-; SM90-NOFTZ-NEXT: @%p1 bra $L__BB64_1;
-; SM90-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NOFTZ-NEXT: fence.acquire.cta;
-; SM90-NOFTZ-NEXT: st.param.b32 [func_retval0], %r1;
-; SM90-NOFTZ-NEXT: ret;
+; SM90-NOFTZ-DISALLOW-LABEL: fminimum_acq_rel_float_global_cta(
+; SM90-NOFTZ-DISALLOW: {
+; SM90-NOFTZ-DISALLOW-NEXT: .reg .pred %p<2>;
+; SM90-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<5>;
+; SM90-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
+; SM90-NOFTZ-DISALLOW-EMPTY:
+; SM90-NOFTZ-DISALLOW-NEXT: // %bb.0:
+; SM90-NOFTZ-DISALLOW-NEXT: ld.param.b32 %r2, [fminimum_acq_rel_float_global_cta_param_1];
+; SM90-NOFTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_float_global_cta_param_0];
+; SM90-NOFTZ-DISALLOW-NEXT: fence.release.cta;
+; SM90-NOFTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM90-NOFTZ-DISALLOW-NEXT: $L__BB64_1: // %atomicrmw.start
+; SM90-NOFTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NOFTZ-DISALLOW-NEXT: min.NaN.f32 %r3, %r4, %r2;
+; SM90-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM90-NOFTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM90-NOFTZ-DISALLOW-NEXT: mov.b32 %r4, %r1;
+; SM90-NOFTZ-DISALLOW-NEXT: @%p1 bra $L__BB64_1;
+; SM90-NOFTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NOFTZ-DISALLOW-NEXT: fence.acquire.cta;
+; SM90-NOFTZ-DISALLOW-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NOFTZ-DISALLOW-NEXT: ret;
+;
+; SM90-NOFTZ-ALLOW-LABEL: fminimum_acq_rel_float_global_cta(
+; SM90-NOFTZ-ALLOW: {
+; SM90-NOFTZ-ALLOW-NEXT: .reg .pred %p<2>;
+; SM90-NOFTZ-ALLOW-NEXT: .reg .b32 %r<5>;
+; SM90-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
+; SM90-NOFTZ-ALLOW-EMPTY:
+; SM90-NOFTZ-ALLOW-NEXT: // %bb.0:
+; SM90-NOFTZ-ALLOW-NEXT: ld.param.b32 %r2, [fminimum_acq_rel_float_global_cta_param_1];
+; SM90-NOFTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_float_global_cta_param_0];
+; SM90-NOFTZ-ALLOW-NEXT: fence.release.cta;
+; SM90-NOFTZ-ALLOW-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM90-NOFTZ-ALLOW-NEXT: $L__BB64_1: // %atomicrmw.start
+; SM90-NOFTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NOFTZ-ALLOW-NEXT: min.NaN.f32 %r3, %r4, %r2;
+; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM90-NOFTZ-ALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM90-NOFTZ-ALLOW-NEXT: mov.b32 %r4, %r1;
+; SM90-NOFTZ-ALLOW-NEXT: @%p1 bra $L__BB64_1;
+; SM90-NOFTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NOFTZ-ALLOW-NEXT: fence.acquire.cta;
+; SM90-NOFTZ-ALLOW-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NOFTZ-ALLOW-NEXT: ret;
;
-; SM90-FTZ-LABEL: fminimum_acq_rel_float_global_cta(
-; SM90-FTZ: {
-; SM90-FTZ-NEXT: .reg .pred %p<2>;
-; SM90-FTZ-NEXT: .reg .b32 %r<5>;
-; SM90-FTZ-NEXT: .reg .b64 %rd<2>;
-; SM90-FTZ-EMPTY:
-; SM90-FTZ-NEXT: // %bb.0:
-; SM90-FTZ-NEXT: ld.param.b32 %r2, [fminimum_acq_rel_float_global_cta_param_1];
-; SM90-FTZ-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_float_global_cta_param_0];
-; SM90-FTZ-NEXT: fence.release.cta;
-; SM90-FTZ-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
-; SM90-FTZ-NEXT: $L__BB64_1: // %atomicrmw.start
-; SM90-FTZ-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-FTZ-NEXT: min.ftz.NaN.f32 %r3, %r4, %r2;
-; SM90-FTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
-; SM90-FTZ-NEXT: setp.ne.b32 %p1, %r1, %r4;
-; SM90-FTZ-NEXT: mov.b32 %r4, %r1;
-; SM90-FTZ-NEXT: @%p1 bra $L__BB64_1;
-; SM90-FTZ-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-FTZ-NEXT: fence.acquire.cta;
-; SM90-FTZ-NEXT: st.param.b32 [func_retval0], %r1;
-; SM90-FTZ-NEXT: ret;
+; SM90-FTZ-DISALLOW-LABEL: fminimum_acq_rel_float_global_cta(
+; SM90-FTZ-DISALLOW: {
+; SM90-FTZ-DISALLOW-NEXT: .reg .pred %p<2>;
+; SM90-FTZ-DISALLOW-NEXT: .reg .b32 %r<5>;
+; SM90-FTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
+; SM90-FTZ-DISALLOW-EMPTY:
+; SM90-FTZ-DISALLOW-NEXT: // %bb.0:
+; SM90-FTZ-DISALLOW-NEXT: ld.param.b32 %r2, [fminimum_acq_rel_float_global_cta_param_1];
+; SM90-FTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_float_global_cta_param_0];
+; SM90-FTZ-DISALLOW-NEXT: fence.release.cta;
+; SM90-FTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM90-FTZ-DISALLOW-NEXT: $L__BB64_1: // %atomicrmw.start
+; SM90-FTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-FTZ-DISALLOW-NEXT: min.ftz.NaN.f32 %r3, %r4, %r2;
+; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM90-FTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM90-FTZ-DISALLOW-NEXT: mov.b32 %r4, %r1;
+; SM90-FTZ-DISALLOW-NEXT: @%p1 bra $L__BB64_1;
+; SM90-FTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-FTZ-DISALLOW-NEXT: fence.acquire.cta;
+; SM90-FTZ-DISALLOW-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-FTZ-DISALLOW-NEXT: ret;
+;
+; SM90-FTZ-ALLOW-LABEL: fminimum_acq_rel_float_global_cta(
+; SM90-FTZ-ALLOW: {
+; SM90-FTZ-ALLOW-NEXT: .reg .pred %p<2>;
+; SM90-FTZ-ALLOW-NEXT: .reg .b32 %r<5>;
+; SM90-FTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
+; SM90-FTZ-ALLOW-EMPTY:
+; SM90-FTZ-ALLOW-NEXT: // %bb.0:
+; SM90-FTZ-ALLOW-NEXT: ld.param.b32 %r2, [fminimum_acq_rel_float_global_cta_param_1];
+; SM90-FTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_float_global_cta_param_0];
+; SM90-FTZ-ALLOW-NEXT: fence.release.cta;
+; SM90-FTZ-ALLOW-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM90-FTZ-ALLOW-NEXT: $L__BB64_1: // %atomicrmw.start
+; SM90-FTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-FTZ-ALLOW-NEXT: min.ftz.NaN.f32 %r3, %r4, %r2;
+; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM90-FTZ-ALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM90-FTZ-ALLOW-NEXT: mov.b32 %r4, %r1;
+; SM90-FTZ-ALLOW-NEXT: @%p1 bra $L__BB64_1;
+; SM90-FTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-FTZ-ALLOW-NEXT: fence.acquire.cta;
+; SM90-FTZ-ALLOW-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-FTZ-ALLOW-NEXT: ret;
%retval = atomicrmw fminimum ptr addrspace(1) %addr, float %val syncscope("block") acq_rel
ret float %retval
}
define float @fmaximum_acq_rel_float_global_cta(ptr addrspace(1) %addr, float %val) {
-; SM90-NOFTZ-LABEL: fmaximum_acq_rel_float_global_cta(
-; SM90-NOFTZ: {
-; SM90-NOFTZ-NEXT: .reg .pred %p<2>;
-; SM90-NOFTZ-NEXT: .reg .b32 %r<5>;
-; SM90-NOFTZ-NEXT: .reg .b64 %rd<2>;
-; SM90-NOFTZ-EMPTY:
-; SM90-NOFTZ-NEXT: // %bb.0:
-; SM90-NOFTZ-NEXT: ld.param.b32 %r2, [fmaximum_acq_rel_float_global_cta_param_1];
-; SM90-NOFTZ-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_float_global_cta_param_0];
-; SM90-NOFTZ-NEXT: fence.release.cta;
-; SM90-NOFTZ-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
-; SM90-NOFTZ-NEXT: $L__BB65_1: // %atomicrmw.start
-; SM90-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NOFTZ-NEXT: max.NaN.f32 %r3, %r4, %r2;
-; SM90-NOFTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
-; SM90-NOFTZ-NEXT: setp.ne.b32 %p1, %r1, %r4;
-; SM90-NOFTZ-NEXT: mov.b32 %r4, %r1;
-; SM90-NOFTZ-NEXT: @%p1 bra $L__BB65_1;
-; SM90-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NOFTZ-NEXT: fence.acquire.cta;
-; SM90-NOFTZ-NEXT: st.param.b32 [func_retval0], %r1;
-; SM90-NOFTZ-NEXT: ret;
+; SM90-NOFTZ-DISALLOW-LABEL: fmaximum_acq_rel_float_global_cta(
+; SM90-NOFTZ-DISALLOW: {
+; SM90-NOFTZ-DISALLOW-NEXT: .reg .pred %p<2>;
+; SM90-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<5>;
+; SM90-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
+; SM90-NOFTZ-DISALLOW-EMPTY:
+; SM90-NOFTZ-DISALLOW-NEXT: // %bb.0:
+; SM90-NOFTZ-DISALLOW-NEXT: ld.param.b32 %r2, [fmaximum_acq_rel_float_global_cta_param_1];
+; SM90-NOFTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_float_global_cta_param_0];
+; SM90-NOFTZ-DISALLOW-NEXT: fence.release.cta;
+; SM90-NOFTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM90-NOFTZ-DISALLOW-NEXT: $L__BB65_1: // %atomicrmw.start
+; SM90-NOFTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NOFTZ-DISALLOW-NEXT: max.NaN.f32 %r3, %r4, %r2;
+; SM90-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM90-NOFTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM90-NOFTZ-DISALLOW-NEXT: mov.b32 %r4, %r1;
+; SM90-NOFTZ-DISALLOW-NEXT: @%p1 bra $L__BB65_1;
+; SM90-NOFTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NOFTZ-DISALLOW-NEXT: fence.acquire.cta;
+; SM90-NOFTZ-DISALLOW-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NOFTZ-DISALLOW-NEXT: ret;
+;
+; SM90-NOFTZ-ALLOW-LABEL: fmaximum_acq_rel_float_global_cta(
+; SM90-NOFTZ-ALLOW: {
+; SM90-NOFTZ-ALLOW-NEXT: .reg .pred %p<2>;
+; SM90-NOFTZ-ALLOW-NEXT: .reg .b32 %r<5>;
+; SM90-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
+; SM90-NOFTZ-ALLOW-EMPTY:
+; SM90-NOFTZ-ALLOW-NEXT: // %bb.0:
+; SM90-NOFTZ-ALLOW-NEXT: ld.param.b32 %r2, [fmaximum_acq_rel_float_global_cta_param_1];
+; SM90-NOFTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_float_global_cta_param_0];
+; SM90-NOFTZ-ALLOW-NEXT: fence.release.cta;
+; SM90-NOFTZ-ALLOW-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM90-NOFTZ-ALLOW-NEXT: $L__BB65_1: // %atomicrmw.start
+; SM90-NOFTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NOFTZ-ALLOW-NEXT: max.NaN.f32 %r3, %r4, %r2;
+; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM90-NOFTZ-ALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM90-NOFTZ-ALLOW-NEXT: mov.b32 %r4, %r1;
+; SM90-NOFTZ-ALLOW-NEXT: @%p1 bra $L__BB65_1;
+; SM90-NOFTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NOFTZ-ALLOW-NEXT: fence.acquire.cta;
+; SM90-NOFTZ-ALLOW-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NOFTZ-ALLOW-NEXT: ret;
;
-; SM90-FTZ-LABEL: fmaximum_acq_rel_float_global_cta(
-; SM90-FTZ: {
-; SM90-FTZ-NEXT: .reg .pred %p<2>;
-; SM90-FTZ-NEXT: .reg .b32 %r<5>;
-; SM90-FTZ-NEXT: .reg .b64 %rd<2>;
-; SM90-FTZ-EMPTY:
-; SM90-FTZ-NEXT: // %bb.0:
-; SM90-FTZ-NEXT: ld.param.b32 %r2, [fmaximum_acq_rel_float_global_cta_param_1];
-; SM90-FTZ-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_float_global_cta_param_0];
-; SM90-FTZ-NEXT: fence.release.cta;
-; SM90-FTZ-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
-; SM90-FTZ-NEXT: $L__BB65_1: // %atomicrmw.start
-; SM90-FTZ-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-FTZ-NEXT: max.ftz.NaN.f32 %r3, %r4, %r2;
-; SM90-FTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
-; SM90-FTZ-NEXT: setp.ne.b32 %p1, %r1, %r4;
-; SM90-FTZ-NEXT: mov.b32 %r4, %r1;
-; SM90-FTZ-NEXT: @%p1 bra $L__BB65_1;
-; SM90-FTZ-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-FTZ-NEXT: fence.acquire.cta;
-; SM90-FTZ-NEXT: st.param.b32 [func_retval0], %r1;
-; SM90-FTZ-NEXT: ret;
+; SM90-FTZ-DISALLOW-LABEL: fmaximum_acq_rel_float_global_cta(
+; SM90-FTZ-DISALLOW: {
+; SM90-FTZ-DISALLOW-NEXT: .reg .pred %p<2>;
+; SM90-FTZ-DISALLOW-NEXT: .reg .b32 %r<5>;
+; SM90-FTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
+; SM90-FTZ-DISALLOW-EMPTY:
+; SM90-FTZ-DISALLOW-NEXT: // %bb.0:
+; SM90-FTZ-DISALLOW-NEXT: ld.param.b32 %r2, [fmaximum_acq_rel_float_global_cta_param_1];
+; SM90-FTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_float_global_cta_param_0];
+; SM90-FTZ-DISALLOW-NEXT: fence.release.cta;
+; SM90-FTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM90-FTZ-DISALLOW-NEXT: $L__BB65_1: // %atomicrmw.start
+; SM90-FTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-FTZ-DISALLOW-NEXT: max.ftz.NaN.f32 %r3, %r4, %r2;
+; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM90-FTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM90-FTZ-DISALLOW-NEXT: mov.b32 %r4, %r1;
+; SM90-FTZ-DISALLOW-NEXT: @%p1 bra $L__BB65_1;
+; SM90-FTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-FTZ-DISALLOW-NEXT: fence.acquire.cta;
+; SM90-FTZ-DISALLOW-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-FTZ-DISALLOW-NEXT: ret;
+;
+; SM90-FTZ-ALLOW-LABEL: fmaximum_acq_rel_float_global_cta(
+; SM90-FTZ-ALLOW: {
+; SM90-FTZ-ALLOW-NEXT: .reg .pred %p<2>;
+; SM90-FTZ-ALLOW-NEXT: .reg .b32 %r<5>;
+; SM90-FTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
+; SM90-FTZ-ALLOW-EMPTY:
+; SM90-FTZ-ALLOW-NEXT: // %bb.0:
+; SM90-FTZ-ALLOW-NEXT: ld.param.b32 %r2, [fmaximum_acq_rel_float_global_cta_param_1];
+; SM90-FTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_float_global_cta_param_0];
+; SM90-FTZ-ALLOW-NEXT: fence.release.cta;
+; SM90-FTZ-ALLOW-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1];
+; SM90-FTZ-ALLOW-NEXT: $L__BB65_1: // %atomicrmw.start
+; SM90-FTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-FTZ-ALLOW-NEXT: max.ftz.NaN.f32 %r3, %r4, %r2;
+; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM90-FTZ-ALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM90-FTZ-ALLOW-NEXT: mov.b32 %r4, %r1;
+; SM90-FTZ-ALLOW-NEXT: @%p1 bra $L__BB65_1;
+; SM90-FTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-FTZ-ALLOW-NEXT: fence.acquire.cta;
+; SM90-FTZ-ALLOW-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-FTZ-ALLOW-NEXT: ret;
%retval = atomicrmw fmaximum ptr addrspace(1) %addr, float %val syncscope("block") acq_rel
ret float %retval
}
@@ -2185,455 +2447,859 @@ define double @fmaximum_acq_rel_double_global_cta(ptr addrspace(1) %addr, double
}
define half @fadd_acq_rel_half_global_cta(ptr addrspace(1) %addr, half %val) {
-; SM90-NOFTZ-LABEL: fadd_acq_rel_half_global_cta(
-; SM90-NOFTZ: {
-; SM90-NOFTZ-NEXT: .reg .b16 %rs<3>;
-; SM90-NOFTZ-NEXT: .reg .b64 %rd<2>;
-; SM90-NOFTZ-EMPTY:
-; SM90-NOFTZ-NEXT: // %bb.0:
-; SM90-NOFTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_half_global_cta_param_0];
-; SM90-NOFTZ-NEXT: ld.param.b16 %rs1, [fadd_acq_rel_half_global_cta_param_1];
-; SM90-NOFTZ-NEXT: atom.acq_rel.cta.global.add.noftz.f16 %rs2, [%rd1], %rs1;
-; SM90-NOFTZ-NEXT: st.param.b16 [func_retval0], %rs2;
-; SM90-NOFTZ-NEXT: ret;
+; SM90-NOFTZ-DISALLOW-LABEL: fadd_acq_rel_half_global_cta(
+; SM90-NOFTZ-DISALLOW: {
+; SM90-NOFTZ-DISALLOW-NEXT: .reg .b16 %rs<3>;
+; SM90-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
+; SM90-NOFTZ-DISALLOW-EMPTY:
+; SM90-NOFTZ-DISALLOW-NEXT: // %bb.0:
+; SM90-NOFTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_half_global_cta_param_0];
+; SM90-NOFTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fadd_acq_rel_half_global_cta_param_1];
+; SM90-NOFTZ-DISALLOW-NEXT: atom.acq_rel.cta.global.add.noftz.f16 %rs2, [%rd1], %rs1;
+; SM90-NOFTZ-DISALLOW-NEXT: st.param.b16 [func_retval0], %rs2;
+; SM90-NOFTZ-DISALLOW-NEXT: ret;
+;
+; SM90-NOFTZ-ALLOW-LABEL: fadd_acq_rel_half_global_cta(
+; SM90-NOFTZ-ALLOW: {
+; SM90-NOFTZ-ALLOW-NEXT: .reg .b16 %rs<3>;
+; SM90-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
+; SM90-NOFTZ-ALLOW-EMPTY:
+; SM90-NOFTZ-ALLOW-NEXT: // %bb.0:
+; SM90-NOFTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_half_global_cta_param_0];
+; SM90-NOFTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fadd_acq_rel_half_global_cta_param_1];
+; SM90-NOFTZ-ALLOW-NEXT: atom.acq_rel.cta.global.add.noftz.f16 %rs2, [%rd1], %rs1;
+; SM90-NOFTZ-ALLOW-NEXT: st.param.b16 [func_retval0], %rs2;
+; SM90-NOFTZ-ALLOW-NEXT: ret;
;
-; SM90-FTZ-LABEL: fadd_acq_rel_half_global_cta(
-; SM90-FTZ: {
-; SM90-FTZ-NEXT: .reg .pred %p<2>;
-; SM90-FTZ-NEXT: .reg .b16 %rs<4>;
-; SM90-FTZ-NEXT: .reg .b32 %r<15>;
-; SM90-FTZ-NEXT: .reg .b64 %rd<3>;
-; SM90-FTZ-EMPTY:
-; SM90-FTZ-NEXT: // %bb.0:
-; SM90-FTZ-NEXT: ld.param.b16 %rs1, [fadd_acq_rel_half_global_cta_param_1];
-; SM90-FTZ-NEXT: ld.param.b64 %rd2, [fadd_acq_rel_half_global_cta_param_0];
-; SM90-FTZ-NEXT: fence.release.cta;
-; SM90-FTZ-NEXT: and.b64 %rd1, %rd2, -4;
-; SM90-FTZ-NEXT: cvt.u32.u64 %r4, %rd2;
-; SM90-FTZ-NEXT: and.b32 %r5, %r4, 3;
-; SM90-FTZ-NEXT: shl.b32 %r1, %r5, 3;
-; SM90-FTZ-NEXT: mov.b32 %r6, 65535;
-; SM90-FTZ-NEXT: shl.b32 %r7, %r6, %r1;
-; SM90-FTZ-NEXT: not.b32 %r2, %r7;
-; SM90-FTZ-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
-; SM90-FTZ-NEXT: $L__BB72_1: // %atomicrmw.start
-; SM90-FTZ-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-FTZ-NEXT: shr.u32 %r8, %r14, %r1;
-; SM90-FTZ-NEXT: cvt.u16.u32 %rs2, %r8;
-; SM90-FTZ-NEXT: add.rn.ftz.f16 %rs3, %rs2, %rs1;
-; SM90-FTZ-NEXT: cvt.u32.u16 %r9, %rs3;
-; SM90-FTZ-NEXT: shl.b32 %r10, %r9, %r1;
-; SM90-FTZ-NEXT: and.b32 %r11, %r14, %r2;
-; SM90-FTZ-NEXT: or.b32 %r12, %r11, %r10;
-; SM90-FTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
-; SM90-FTZ-NEXT: setp.ne.b32 %p1, %r3, %r14;
-; SM90-FTZ-NEXT: mov.b32 %r14, %r3;
-; SM90-FTZ-NEXT: @%p1 bra $L__BB72_1;
-; SM90-FTZ-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-FTZ-NEXT: shr.u32 %r13, %r3, %r1;
-; SM90-FTZ-NEXT: fence.acquire.cta;
-; SM90-FTZ-NEXT: st.param.b16 [func_retval0], %r13;
-; SM90-FTZ-NEXT: ret;
+; SM90-FTZ-DISALLOW-LABEL: fadd_acq_rel_half_global_cta(
+; SM90-FTZ-DISALLOW: {
+; SM90-FTZ-DISALLOW-NEXT: .reg .pred %p<2>;
+; SM90-FTZ-DISALLOW-NEXT: .reg .b16 %rs<4>;
+; SM90-FTZ-DISALLOW-NEXT: .reg .b32 %r<15>;
+; SM90-FTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
+; SM90-FTZ-DISALLOW-EMPTY:
+; SM90-FTZ-DISALLOW-NEXT: // %bb.0:
+; SM90-FTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fadd_acq_rel_half_global_cta_param_1];
+; SM90-FTZ-DISALLOW-NEXT: ld.param.b64 %rd2, [fadd_acq_rel_half_global_cta_param_0];
+; SM90-FTZ-DISALLOW-NEXT: fence.release.cta;
+; SM90-FTZ-DISALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-FTZ-DISALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM90-FTZ-DISALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM90-FTZ-DISALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM90-FTZ-DISALLOW-NEXT: mov.b32 %r6, 65535;
+; SM90-FTZ-DISALLOW-NEXT: shl.b32 %r7, %r6, %r1;
+; SM90-FTZ-DISALLOW-NEXT: not.b32 %r2, %r7;
+; SM90-FTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM90-FTZ-DISALLOW-NEXT: $L__BB72_1: // %atomicrmw.start
+; SM90-FTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-FTZ-DISALLOW-NEXT: shr.u32 %r8, %r14, %r1;
+; SM90-FTZ-DISALLOW-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM90-FTZ-DISALLOW-NEXT: add.rn.ftz.f16 %rs3, %rs2, %rs1;
+; SM90-FTZ-DISALLOW-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM90-FTZ-DISALLOW-NEXT: shl.b32 %r10, %r9, %r1;
+; SM90-FTZ-DISALLOW-NEXT: and.b32 %r11, %r14, %r2;
+; SM90-FTZ-DISALLOW-NEXT: or.b32 %r12, %r11, %r10;
+; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM90-FTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM90-FTZ-DISALLOW-NEXT: mov.b32 %r14, %r3;
+; SM90-FTZ-DISALLOW-NEXT: @%p1 bra $L__BB72_1;
+; SM90-FTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-FTZ-DISALLOW-NEXT: shr.u32 %r13, %r3, %r1;
+; SM90-FTZ-DISALLOW-NEXT: fence.acquire.cta;
+; SM90-FTZ-DISALLOW-NEXT: st.param.b16 [func_retval0], %r13;
+; SM90-FTZ-DISALLOW-NEXT: ret;
+;
+; SM90-FTZ-ALLOW-LABEL: fadd_acq_rel_half_global_cta(
+; SM90-FTZ-ALLOW: {
+; SM90-FTZ-ALLOW-NEXT: .reg .b16 %rs<3>;
+; SM90-FTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
+; SM90-FTZ-ALLOW-EMPTY:
+; SM90-FTZ-ALLOW-NEXT: // %bb.0:
+; SM90-FTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_half_global_cta_param_0];
+; SM90-FTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fadd_acq_rel_half_global_cta_param_1];
+; SM90-FTZ-ALLOW-NEXT: atom.acq_rel.cta.global.add.noftz.f16 %rs2, [%rd1], %rs1;
+; SM90-FTZ-ALLOW-NEXT: st.param.b16 [func_retval0], %rs2;
+; SM90-FTZ-ALLOW-NEXT: ret;
%retval = atomicrmw fadd ptr addrspace(1) %addr, half %val syncscope("block") acq_rel
ret half %retval
}
define half @fsub_acq_rel_half_global_cta(ptr addrspace(1) %addr, half %val) {
-; SM90-NOFTZ-LABEL: fsub_acq_rel_half_global_cta(
-; SM90-NOFTZ: {
-; SM90-NOFTZ-NEXT: .reg .pred %p<2>;
-; SM90-NOFTZ-NEXT: .reg .b16 %rs<4>;
-; SM90-NOFTZ-NEXT: .reg .b32 %r<15>;
-; SM90-NOFTZ-NEXT: .reg .b64 %rd<3>;
-; SM90-NOFTZ-EMPTY:
-; SM90-NOFTZ-NEXT: // %bb.0:
-; SM90-NOFTZ-NEXT: ld.param.b16 %rs1, [fsub_acq_rel_half_global_cta_param_1];
-; SM90-NOFTZ-NEXT: ld.param.b64 %rd2, [fsub_acq_rel_half_global_cta_param_0];
-; SM90-NOFTZ-NEXT: fence.release.cta;
-; SM90-NOFTZ-NEXT: and.b64 %rd1, %rd2, -4;
-; SM90-NOFTZ-NEXT: cvt.u32.u64 %r4, %rd2;
-; SM90-NOFTZ-NEXT: and.b32 %r5, %r4, 3;
-; SM90-NOFTZ-NEXT: shl.b32 %r1, %r5, 3;
-; SM90-NOFTZ-NEXT: mov.b32 %r6, 65535;
-; SM90-NOFTZ-NEXT: shl.b32 %r7, %r6, %r1;
-; SM90-NOFTZ-NEXT: not.b32 %r2, %r7;
-; SM90-NOFTZ-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
-; SM90-NOFTZ-NEXT: $L__BB73_1: // %atomicrmw.start
-; SM90-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NOFTZ-NEXT: shr.u32 %r8, %r14, %r1;
-; SM90-NOFTZ-NEXT: cvt.u16.u32 %rs2, %r8;
-; SM90-NOFTZ-NEXT: sub.rn.f16 %rs3, %rs2, %rs1;
-; SM90-NOFTZ-NEXT: cvt.u32.u16 %r9, %rs3;
-; SM90-NOFTZ-NEXT: shl.b32 %r10, %r9, %r1;
-; SM90-NOFTZ-NEXT: and.b32 %r11, %r14, %r2;
-; SM90-NOFTZ-NEXT: or.b32 %r12, %r11, %r10;
-; SM90-NOFTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
-; SM90-NOFTZ-NEXT: setp.ne.b32 %p1, %r3, %r14;
-; SM90-NOFTZ-NEXT: mov.b32 %r14, %r3;
-; SM90-NOFTZ-NEXT: @%p1 bra $L__BB73_1;
-; SM90-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NOFTZ-NEXT: shr.u32 %r13, %r3, %r1;
-; SM90-NOFTZ-NEXT: fence.acquire.cta;
-; SM90-NOFTZ-NEXT: st.param.b16 [func_retval0], %r13;
-; SM90-NOFTZ-NEXT: ret;
+; SM90-NOFTZ-DISALLOW-LABEL: fsub_acq_rel_half_global_cta(
+; SM90-NOFTZ-DISALLOW: {
+; SM90-NOFTZ-DISALLOW-NEXT: .reg .pred %p<2>;
+; SM90-NOFTZ-DISALLOW-NEXT: .reg .b16 %rs<4>;
+; SM90-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<15>;
+; SM90-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
+; SM90-NOFTZ-DISALLOW-EMPTY:
+; SM90-NOFTZ-DISALLOW-NEXT: // %bb.0:
+; SM90-NOFTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fsub_acq_rel_half_global_cta_param_1];
+; SM90-NOFTZ-DISALLOW-NEXT: ld.param.b64 %rd2, [fsub_acq_rel_half_global_cta_param_0];
+; SM90-NOFTZ-DISALLOW-NEXT: fence.release.cta;
+; SM90-NOFTZ-DISALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM90-NOFTZ-DISALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM90-NOFTZ-DISALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM90-NOFTZ-DISALLOW-NEXT: mov.b32 %r6, 65535;
+; SM90-NOFTZ-DISALLOW-NEXT: shl.b32 %r7, %r6, %r1;
+; SM90-NOFTZ-DISALLOW-NEXT: not.b32 %r2, %r7;
+; SM90-NOFTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM90-NOFTZ-DISALLOW-NEXT: $L__BB73_1: // %atomicrmw.start
+; SM90-NOFTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NOFTZ-DISALLOW-NEXT: shr.u32 %r8, %r14, %r1;
+; SM90-NOFTZ-DISALLOW-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM90-NOFTZ-DISALLOW-NEXT: sub.rn.f16 %rs3, %rs2, %rs1;
+; SM90-NOFTZ-DISALLOW-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM90-NOFTZ-DISALLOW-NEXT: shl.b32 %r10, %r9, %r1;
+; SM90-NOFTZ-DISALLOW-NEXT: and.b32 %r11, %r14, %r2;
+; SM90-NOFTZ-DISALLOW-NEXT: or.b32 %r12, %r11, %r10;
+; SM90-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM90-NOFTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM90-NOFTZ-DISALLOW-NEXT: mov.b32 %r14, %r3;
+; SM90-NOFTZ-DISALLOW-NEXT: @%p1 bra $L__BB73_1;
+; SM90-NOFTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NOFTZ-DISALLOW-NEXT: shr.u32 %r13, %r3, %r1;
+; SM90-NOFTZ-DISALLOW-NEXT: fence.acquire.cta;
+; SM90-NOFTZ-DISALLOW-NEXT: st.param.b16 [func_retval0], %r13;
+; SM90-NOFTZ-DISALLOW-NEXT: ret;
+;
+; SM90-NOFTZ-ALLOW-LABEL: fsub_acq_rel_half_global_cta(
+; SM90-NOFTZ-ALLOW: {
+; SM90-NOFTZ-ALLOW-NEXT: .reg .pred %p<2>;
+; SM90-NOFTZ-ALLOW-NEXT: .reg .b16 %rs<4>;
+; SM90-NOFTZ-ALLOW-NEXT: .reg .b32 %r<15>;
+; SM90-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
+; SM90-NOFTZ-ALLOW-EMPTY:
+; SM90-NOFTZ-ALLOW-NEXT: // %bb.0:
+; SM90-NOFTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fsub_acq_rel_half_global_cta_param_1];
+; SM90-NOFTZ-ALLOW-NEXT: ld.param.b64 %rd2, [fsub_acq_rel_half_global_cta_param_0];
+; SM90-NOFTZ-ALLOW-NEXT: fence.release.cta;
+; SM90-NOFTZ-ALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NOFTZ-ALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM90-NOFTZ-ALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM90-NOFTZ-ALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM90-NOFTZ-ALLOW-NEXT: mov.b32 %r6, 65535;
+; SM90-NOFTZ-ALLOW-NEXT: shl.b32 %r7, %r6, %r1;
+; SM90-NOFTZ-ALLOW-NEXT: not.b32 %r2, %r7;
+; SM90-NOFTZ-ALLOW-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM90-NOFTZ-ALLOW-NEXT: $L__BB73_1: // %atomicrmw.start
+; SM90-NOFTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NOFTZ-ALLOW-NEXT: shr.u32 %r8, %r14, %r1;
+; SM90-NOFTZ-ALLOW-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM90-NOFTZ-ALLOW-NEXT: sub.rn.f16 %rs3, %rs2, %rs1;
+; SM90-NOFTZ-ALLOW-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM90-NOFTZ-ALLOW-NEXT: shl.b32 %r10, %r9, %r1;
+; SM90-NOFTZ-ALLOW-NEXT: and.b32 %r11, %r14, %r2;
+; SM90-NOFTZ-ALLOW-NEXT: or.b32 %r12, %r11, %r10;
+; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM90-NOFTZ-ALLOW-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM90-NOFTZ-ALLOW-NEXT: mov.b32 %r14, %r3;
+; SM90-NOFTZ-ALLOW-NEXT: @%p1 bra $L__BB73_1;
+; SM90-NOFTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NOFTZ-ALLOW-NEXT: shr.u32 %r13, %r3, %r1;
+; SM90-NOFTZ-ALLOW-NEXT: fence.acquire.cta;
+; SM90-NOFTZ-ALLOW-NEXT: st.param.b16 [func_retval0], %r13;
+; SM90-NOFTZ-ALLOW-NEXT: ret;
;
-; SM90-FTZ-LABEL: fsub_acq_rel_half_global_cta(
-; SM90-FTZ: {
-; SM90-FTZ-NEXT: .reg .pred %p<2>;
-; SM90-FTZ-NEXT: .reg .b16 %rs<4>;
-; SM90-FTZ-NEXT: .reg .b32 %r<15>;
-; SM90-FTZ-NEXT: .reg .b64 %rd<3>;
-; SM90-FTZ-EMPTY:
-; SM90-FTZ-NEXT: // %bb.0:
-; SM90-FTZ-NEXT: ld.param.b16 %rs1, [fsub_acq_rel_half_global_cta_param_1];
-; SM90-FTZ-NEXT: ld.param.b64 %rd2, [fsub_acq_rel_half_global_cta_param_0];
-; SM90-FTZ-NEXT: fence.release.cta;
-; SM90-FTZ-NEXT: and.b64 %rd1, %rd2, -4;
-; SM90-FTZ-NEXT: cvt.u32.u64 %r4, %rd2;
-; SM90-FTZ-NEXT: and.b32 %r5, %r4, 3;
-; SM90-FTZ-NEXT: shl.b32 %r1, %r5, 3;
-; SM90-FTZ-NEXT: mov.b32 %r6, 65535;
-; SM90-FTZ-NEXT: shl.b32 %r7, %r6, %r1;
-; SM90-FTZ-NEXT: not.b32 %r2, %r7;
-; SM90-FTZ-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
-; SM90-FTZ-NEXT: $L__BB73_1: // %atomicrmw.start
-; SM90-FTZ-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-FTZ-NEXT: shr.u32 %r8, %r14, %r1;
-; SM90-FTZ-NEXT: cvt.u16.u32 %rs2, %r8;
-; SM90-FTZ-NEXT: sub.rn.ftz.f16 %rs3, %rs2, %rs1;
-; SM90-FTZ-NEXT: cvt.u32.u16 %r9, %rs3;
-; SM90-FTZ-NEXT: shl.b32 %r10, %r9, %r1;
-; SM90-FTZ-NEXT: and.b32 %r11, %r14, %r2;
-; SM90-FTZ-NEXT: or.b32 %r12, %r11, %r10;
-; SM90-FTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
-; SM90-FTZ-NEXT: setp.ne.b32 %p1, %r3, %r14;
-; SM90-FTZ-NEXT: mov.b32 %r14, %r3;
-; SM90-FTZ-NEXT: @%p1 bra $L__BB73_1;
-; SM90-FTZ-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-FTZ-NEXT: shr.u32 %r13, %r3, %r1;
-; SM90-FTZ-NEXT: fence.acquire.cta;
-; SM90-FTZ-NEXT: st.param.b16 [func_retval0], %r13;
-; SM90-FTZ-NEXT: ret;
+; SM90-FTZ-DISALLOW-LABEL: fsub_acq_rel_half_global_cta(
+; SM90-FTZ-DISALLOW: {
+; SM90-FTZ-DISALLOW-NEXT: .reg .pred %p<2>;
+; SM90-FTZ-DISALLOW-NEXT: .reg .b16 %rs<4>;
+; SM90-FTZ-DISALLOW-NEXT: .reg .b32 %r<15>;
+; SM90-FTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
+; SM90-FTZ-DISALLOW-EMPTY:
+; SM90-FTZ-DISALLOW-NEXT: // %bb.0:
+; SM90-FTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fsub_acq_rel_half_global_cta_param_1];
+; SM90-FTZ-DISALLOW-NEXT: ld.param.b64 %rd2, [fsub_acq_rel_half_global_cta_param_0];
+; SM90-FTZ-DISALLOW-NEXT: fence.release.cta;
+; SM90-FTZ-DISALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-FTZ-DISALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM90-FTZ-DISALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM90-FTZ-DISALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM90-FTZ-DISALLOW-NEXT: mov.b32 %r6, 65535;
+; SM90-FTZ-DISALLOW-NEXT: shl.b32 %r7, %r6, %r1;
+; SM90-FTZ-DISALLOW-NEXT: not.b32 %r2, %r7;
+; SM90-FTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM90-FTZ-DISALLOW-NEXT: $L__BB73_1: // %atomicrmw.start
+; SM90-FTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-FTZ-DISALLOW-NEXT: shr.u32 %r8, %r14, %r1;
+; SM90-FTZ-DISALLOW-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM90-FTZ-DISALLOW-NEXT: sub.rn.ftz.f16 %rs3, %rs2, %rs1;
+; SM90-FTZ-DISALLOW-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM90-FTZ-DISALLOW-NEXT: shl.b32 %r10, %r9, %r1;
+; SM90-FTZ-DISALLOW-NEXT: and.b32 %r11, %r14, %r2;
+; SM90-FTZ-DISALLOW-NEXT: or.b32 %r12, %r11, %r10;
+; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM90-FTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM90-FTZ-DISALLOW-NEXT: mov.b32 %r14, %r3;
+; SM90-FTZ-DISALLOW-NEXT: @%p1 bra $L__BB73_1;
+; SM90-FTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-FTZ-DISALLOW-NEXT: shr.u32 %r13, %r3, %r1;
+; SM90-FTZ-DISALLOW-NEXT: fence.acquire.cta;
+; SM90-FTZ-DISALLOW-NEXT: st.param.b16 [func_retval0], %r13;
+; SM90-FTZ-DISALLOW-NEXT: ret;
+;
+; SM90-FTZ-ALLOW-LABEL: fsub_acq_rel_half_global_cta(
+; SM90-FTZ-ALLOW: {
+; SM90-FTZ-ALLOW-NEXT: .reg .pred %p<2>;
+; SM90-FTZ-ALLOW-NEXT: .reg .b16 %rs<4>;
+; SM90-FTZ-ALLOW-NEXT: .reg .b32 %r<15>;
+; SM90-FTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
+; SM90-FTZ-ALLOW-EMPTY:
+; SM90-FTZ-ALLOW-NEXT: // %bb.0:
+; SM90-FTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fsub_acq_rel_half_global_cta_param_1];
+; SM90-FTZ-ALLOW-NEXT: ld.param.b64 %rd2, [fsub_acq_rel_half_global_cta_param_0];
+; SM90-FTZ-ALLOW-NEXT: fence.release.cta;
+; SM90-FTZ-ALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-FTZ-ALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM90-FTZ-ALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM90-FTZ-ALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM90-FTZ-ALLOW-NEXT: mov.b32 %r6, 65535;
+; SM90-FTZ-ALLOW-NEXT: shl.b32 %r7, %r6, %r1;
+; SM90-FTZ-ALLOW-NEXT: not.b32 %r2, %r7;
+; SM90-FTZ-ALLOW-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM90-FTZ-ALLOW-NEXT: $L__BB73_1: // %atomicrmw.start
+; SM90-FTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-FTZ-ALLOW-NEXT: shr.u32 %r8, %r14, %r1;
+; SM90-FTZ-ALLOW-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM90-FTZ-ALLOW-NEXT: sub.rn.ftz.f16 %rs3, %rs2, %rs1;
+; SM90-FTZ-ALLOW-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM90-FTZ-ALLOW-NEXT: shl.b32 %r10, %r9, %r1;
+; SM90-FTZ-ALLOW-NEXT: and.b32 %r11, %r14, %r2;
+; SM90-FTZ-ALLOW-NEXT: or.b32 %r12, %r11, %r10;
+; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM90-FTZ-ALLOW-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM90-FTZ-ALLOW-NEXT: mov.b32 %r14, %r3;
+; SM90-FTZ-ALLOW-NEXT: @%p1 bra $L__BB73_1;
+; SM90-FTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-FTZ-ALLOW-NEXT: shr.u32 %r13, %r3, %r1;
+; SM90-FTZ-ALLOW-NEXT: fence.acquire.cta;
+; SM90-FTZ-ALLOW-NEXT: st.param.b16 [func_retval0], %r13;
+; SM90-FTZ-ALLOW-NEXT: ret;
%retval = atomicrmw fsub ptr addrspace(1) %addr, half %val syncscope("block") acq_rel
ret half %retval
}
define half @fmin_acq_rel_half_global_cta(ptr addrspace(1) %addr, half %val) {
-; SM90-NOFTZ-LABEL: fmin_acq_rel_half_global_cta(
-; SM90-NOFTZ: {
-; SM90-NOFTZ-NEXT: .reg .pred %p<2>;
-; SM90-NOFTZ-NEXT: .reg .b16 %rs<4>;
-; SM90-NOFTZ-NEXT: .reg .b32 %r<15>;
-; SM90-NOFTZ-NEXT: .reg .b64 %rd<3>;
-; SM90-NOFTZ-EMPTY:
-; SM90-NOFTZ-NEXT: // %bb.0:
-; SM90-NOFTZ-NEXT: ld.param.b16 %rs1, [fmin_acq_rel_half_global_cta_param_1];
-; SM90-NOFTZ-NEXT: ld.param.b64 %rd2, [fmin_acq_rel_half_global_cta_param_0];
-; SM90-NOFTZ-NEXT: fence.release.cta;
-; SM90-NOFTZ-NEXT: and.b64 %rd1, %rd2, -4;
-; SM90-NOFTZ-NEXT: cvt.u32.u64 %r4, %rd2;
-; SM90-NOFTZ-NEXT: and.b32 %r5, %r4, 3;
-; SM90-NOFTZ-NEXT: shl.b32 %r1, %r5, 3;
-; SM90-NOFTZ-NEXT: mov.b32 %r6, 65535;
-; SM90-NOFTZ-NEXT: shl.b32 %r7, %r6, %r1;
-; SM90-NOFTZ-NEXT: not.b32 %r2, %r7;
-; SM90-NOFTZ-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
-; SM90-NOFTZ-NEXT: $L__BB74_1: // %atomicrmw.start
-; SM90-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NOFTZ-NEXT: shr.u32 %r8, %r14, %r1;
-; SM90-NOFTZ-NEXT: cvt.u16.u32 %rs2, %r8;
-; SM90-NOFTZ-NEXT: min.f16 %rs3, %rs2, %rs1;
-; SM90-NOFTZ-NEXT: cvt.u32.u16 %r9, %rs3;
-; SM90-NOFTZ-NEXT: shl.b32 %r10, %r9, %r1;
-; SM90-NOFTZ-NEXT: and.b32 %r11, %r14, %r2;
-; SM90-NOFTZ-NEXT: or.b32 %r12, %r11, %r10;
-; SM90-NOFTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
-; SM90-NOFTZ-NEXT: setp.ne.b32 %p1, %r3, %r14;
-; SM90-NOFTZ-NEXT: mov.b32 %r14, %r3;
-; SM90-NOFTZ-NEXT: @%p1 bra $L__BB74_1;
-; SM90-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NOFTZ-NEXT: shr.u32 %r13, %r3, %r1;
-; SM90-NOFTZ-NEXT: fence.acquire.cta;
-; SM90-NOFTZ-NEXT: st.param.b16 [func_retval0], %r13;
-; SM90-NOFTZ-NEXT: ret;
+; SM90-NOFTZ-DISALLOW-LABEL: fmin_acq_rel_half_global_cta(
+; SM90-NOFTZ-DISALLOW: {
+; SM90-NOFTZ-DISALLOW-NEXT: .reg .pred %p<2>;
+; SM90-NOFTZ-DISALLOW-NEXT: .reg .b16 %rs<4>;
+; SM90-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<15>;
+; SM90-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
+; SM90-NOFTZ-DISALLOW-EMPTY:
+; SM90-NOFTZ-DISALLOW-NEXT: // %bb.0:
+; SM90-NOFTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fmin_acq_rel_half_global_cta_param_1];
+; SM90-NOFTZ-DISALLOW-NEXT: ld.param.b64 %rd2, [fmin_acq_rel_half_global_cta_param_0];
+; SM90-NOFTZ-DISALLOW-NEXT: fence.release.cta;
+; SM90-NOFTZ-DISALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM90-NOFTZ-DISALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM90-NOFTZ-DISALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM90-NOFTZ-DISALLOW-NEXT: mov.b32 %r6, 65535;
+; SM90-NOFTZ-DISALLOW-NEXT: shl.b32 %r7, %r6, %r1;
+; SM90-NOFTZ-DISALLOW-NEXT: not.b32 %r2, %r7;
+; SM90-NOFTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM90-NOFTZ-DISALLOW-NEXT: $L__BB74_1: // %atomicrmw.start
+; SM90-NOFTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NOFTZ-DISALLOW-NEXT: shr.u32 %r8, %r14, %r1;
+; SM90-NOFTZ-DISALLOW-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM90-NOFTZ-DISALLOW-NEXT: min.f16 %rs3, %rs2, %rs1;
+; SM90-NOFTZ-DISALLOW-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM90-NOFTZ-DISALLOW-NEXT: shl.b32 %r10, %r9, %r1;
+; SM90-NOFTZ-DISALLOW-NEXT: and.b32 %r11, %r14, %r2;
+; SM90-NOFTZ-DISALLOW-NEXT: or.b32 %r12, %r11, %r10;
+; SM90-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM90-NOFTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM90-NOFTZ-DISALLOW-NEXT: mov.b32 %r14, %r3;
+; SM90-NOFTZ-DISALLOW-NEXT: @%p1 bra $L__BB74_1;
+; SM90-NOFTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NOFTZ-DISALLOW-NEXT: shr.u32 %r13, %r3, %r1;
+; SM90-NOFTZ-DISALLOW-NEXT: fence.acquire.cta;
+; SM90-NOFTZ-DISALLOW-NEXT: st.param.b16 [func_retval0], %r13;
+; SM90-NOFTZ-DISALLOW-NEXT: ret;
+;
+; SM90-NOFTZ-ALLOW-LABEL: fmin_acq_rel_half_global_cta(
+; SM90-NOFTZ-ALLOW: {
+; SM90-NOFTZ-ALLOW-NEXT: .reg .pred %p<2>;
+; SM90-NOFTZ-ALLOW-NEXT: .reg .b16 %rs<4>;
+; SM90-NOFTZ-ALLOW-NEXT: .reg .b32 %r<15>;
+; SM90-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
+; SM90-NOFTZ-ALLOW-EMPTY:
+; SM90-NOFTZ-ALLOW-NEXT: // %bb.0:
+; SM90-NOFTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fmin_acq_rel_half_global_cta_param_1];
+; SM90-NOFTZ-ALLOW-NEXT: ld.param.b64 %rd2, [fmin_acq_rel_half_global_cta_param_0];
+; SM90-NOFTZ-ALLOW-NEXT: fence.release.cta;
+; SM90-NOFTZ-ALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NOFTZ-ALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM90-NOFTZ-ALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM90-NOFTZ-ALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM90-NOFTZ-ALLOW-NEXT: mov.b32 %r6, 65535;
+; SM90-NOFTZ-ALLOW-NEXT: shl.b32 %r7, %r6, %r1;
+; SM90-NOFTZ-ALLOW-NEXT: not.b32 %r2, %r7;
+; SM90-NOFTZ-ALLOW-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM90-NOFTZ-ALLOW-NEXT: $L__BB74_1: // %atomicrmw.start
+; SM90-NOFTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NOFTZ-ALLOW-NEXT: shr.u32 %r8, %r14, %r1;
+; SM90-NOFTZ-ALLOW-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM90-NOFTZ-ALLOW-NEXT: min.f16 %rs3, %rs2, %rs1;
+; SM90-NOFTZ-ALLOW-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM90-NOFTZ-ALLOW-NEXT: shl.b32 %r10, %r9, %r1;
+; SM90-NOFTZ-ALLOW-NEXT: and.b32 %r11, %r14, %r2;
+; SM90-NOFTZ-ALLOW-NEXT: or.b32 %r12, %r11, %r10;
+; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM90-NOFTZ-ALLOW-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM90-NOFTZ-ALLOW-NEXT: mov.b32 %r14, %r3;
+; SM90-NOFTZ-ALLOW-NEXT: @%p1 bra $L__BB74_1;
+; SM90-NOFTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NOFTZ-ALLOW-NEXT: shr.u32 %r13, %r3, %r1;
+; SM90-NOFTZ-ALLOW-NEXT: fence.acquire.cta;
+; SM90-NOFTZ-ALLOW-NEXT: st.param.b16 [func_retval0], %r13;
+; SM90-NOFTZ-ALLOW-NEXT: ret;
;
-; SM90-FTZ-LABEL: fmin_acq_rel_half_global_cta(
-; SM90-FTZ: {
-; SM90-FTZ-NEXT: .reg .pred %p<2>;
-; SM90-FTZ-NEXT: .reg .b16 %rs<4>;
-; SM90-FTZ-NEXT: .reg .b32 %r<15>;
-; SM90-FTZ-NEXT: .reg .b64 %rd<3>;
-; SM90-FTZ-EMPTY:
-; SM90-FTZ-NEXT: // %bb.0:
-; SM90-FTZ-NEXT: ld.param.b16 %rs1, [fmin_acq_rel_half_global_cta_param_1];
-; SM90-FTZ-NEXT: ld.param.b64 %rd2, [fmin_acq_rel_half_global_cta_param_0];
-; SM90-FTZ-NEXT: fence.release.cta;
-; SM90-FTZ-NEXT: and.b64 %rd1, %rd2, -4;
-; SM90-FTZ-NEXT: cvt.u32.u64 %r4, %rd2;
-; SM90-FTZ-NEXT: and.b32 %r5, %r4, 3;
-; SM90-FTZ-NEXT: shl.b32 %r1, %r5, 3;
-; SM90-FTZ-NEXT: mov.b32 %r6, 65535;
-; SM90-FTZ-NEXT: shl.b32 %r7, %r6, %r1;
-; SM90-FTZ-NEXT: not.b32 %r2, %r7;
-; SM90-FTZ-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
-; SM90-FTZ-NEXT: $L__BB74_1: // %atomicrmw.start
-; SM90-FTZ-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-FTZ-NEXT: shr.u32 %r8, %r14, %r1;
-; SM90-FTZ-NEXT: cvt.u16.u32 %rs2, %r8;
-; SM90-FTZ-NEXT: min.ftz.f16 %rs3, %rs2, %rs1;
-; SM90-FTZ-NEXT: cvt.u32.u16 %r9, %rs3;
-; SM90-FTZ-NEXT: shl.b32 %r10, %r9, %r1;
-; SM90-FTZ-NEXT: and.b32 %r11, %r14, %r2;
-; SM90-FTZ-NEXT: or.b32 %r12, %r11, %r10;
-; SM90-FTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
-; SM90-FTZ-NEXT: setp.ne.b32 %p1, %r3, %r14;
-; SM90-FTZ-NEXT: mov.b32 %r14, %r3;
-; SM90-FTZ-NEXT: @%p1 bra $L__BB74_1;
-; SM90-FTZ-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-FTZ-NEXT: shr.u32 %r13, %r3, %r1;
-; SM90-FTZ-NEXT: fence.acquire.cta;
-; SM90-FTZ-NEXT: st.param.b16 [func_retval0], %r13;
-; SM90-FTZ-NEXT: ret;
+; SM90-FTZ-DISALLOW-LABEL: fmin_acq_rel_half_global_cta(
+; SM90-FTZ-DISALLOW: {
+; SM90-FTZ-DISALLOW-NEXT: .reg .pred %p<2>;
+; SM90-FTZ-DISALLOW-NEXT: .reg .b16 %rs<4>;
+; SM90-FTZ-DISALLOW-NEXT: .reg .b32 %r<15>;
+; SM90-FTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
+; SM90-FTZ-DISALLOW-EMPTY:
+; SM90-FTZ-DISALLOW-NEXT: // %bb.0:
+; SM90-FTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fmin_acq_rel_half_global_cta_param_1];
+; SM90-FTZ-DISALLOW-NEXT: ld.param.b64 %rd2, [fmin_acq_rel_half_global_cta_param_0];
+; SM90-FTZ-DISALLOW-NEXT: fence.release.cta;
+; SM90-FTZ-DISALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-FTZ-DISALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM90-FTZ-DISALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM90-FTZ-DISALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM90-FTZ-DISALLOW-NEXT: mov.b32 %r6, 65535;
+; SM90-FTZ-DISALLOW-NEXT: shl.b32 %r7, %r6, %r1;
+; SM90-FTZ-DISALLOW-NEXT: not.b32 %r2, %r7;
+; SM90-FTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM90-FTZ-DISALLOW-NEXT: $L__BB74_1: // %atomicrmw.start
+; SM90-FTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-FTZ-DISALLOW-NEXT: shr.u32 %r8, %r14, %r1;
+; SM90-FTZ-DISALLOW-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM90-FTZ-DISALLOW-NEXT: min.ftz.f16 %rs3, %rs2, %rs1;
+; SM90-FTZ-DISALLOW-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM90-FTZ-DISALLOW-NEXT: shl.b32 %r10, %r9, %r1;
+; SM90-FTZ-DISALLOW-NEXT: and.b32 %r11, %r14, %r2;
+; SM90-FTZ-DISALLOW-NEXT: or.b32 %r12, %r11, %r10;
+; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM90-FTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM90-FTZ-DISALLOW-NEXT: mov.b32 %r14, %r3;
+; SM90-FTZ-DISALLOW-NEXT: @%p1 bra $L__BB74_1;
+; SM90-FTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-FTZ-DISALLOW-NEXT: shr.u32 %r13, %r3, %r1;
+; SM90-FTZ-DISALLOW-NEXT: fence.acquire.cta;
+; SM90-FTZ-DISALLOW-NEXT: st.param.b16 [func_retval0], %r13;
+; SM90-FTZ-DISALLOW-NEXT: ret;
+;
+; SM90-FTZ-ALLOW-LABEL: fmin_acq_rel_half_global_cta(
+; SM90-FTZ-ALLOW: {
+; SM90-FTZ-ALLOW-NEXT: .reg .pred %p<2>;
+; SM90-FTZ-ALLOW-NEXT: .reg .b16 %rs<4>;
+; SM90-FTZ-ALLOW-NEXT: .reg .b32 %r<15>;
+; SM90-FTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
+; SM90-FTZ-ALLOW-EMPTY:
+; SM90-FTZ-ALLOW-NEXT: // %bb.0:
+; SM90-FTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fmin_acq_rel_half_global_cta_param_1];
+; SM90-FTZ-ALLOW-NEXT: ld.param.b64 %rd2, [fmin_acq_rel_half_global_cta_param_0];
+; SM90-FTZ-ALLOW-NEXT: fence.release.cta;
+; SM90-FTZ-ALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-FTZ-ALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM90-FTZ-ALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM90-FTZ-ALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM90-FTZ-ALLOW-NEXT: mov.b32 %r6, 65535;
+; SM90-FTZ-ALLOW-NEXT: shl.b32 %r7, %r6, %r1;
+; SM90-FTZ-ALLOW-NEXT: not.b32 %r2, %r7;
+; SM90-FTZ-ALLOW-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM90-FTZ-ALLOW-NEXT: $L__BB74_1: // %atomicrmw.start
+; SM90-FTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-FTZ-ALLOW-NEXT: shr.u32 %r8, %r14, %r1;
+; SM90-FTZ-ALLOW-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM90-FTZ-ALLOW-NEXT: min.ftz.f16 %rs3, %rs2, %rs1;
+; SM90-FTZ-ALLOW-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM90-FTZ-ALLOW-NEXT: shl.b32 %r10, %r9, %r1;
+; SM90-FTZ-ALLOW-NEXT: and.b32 %r11, %r14, %r2;
+; SM90-FTZ-ALLOW-NEXT: or.b32 %r12, %r11, %r10;
+; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM90-FTZ-ALLOW-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM90-FTZ-ALLOW-NEXT: mov.b32 %r14, %r3;
+; SM90-FTZ-ALLOW-NEXT: @%p1 bra $L__BB74_1;
+; SM90-FTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-FTZ-ALLOW-NEXT: shr.u32 %r13, %r3, %r1;
+; SM90-FTZ-ALLOW-NEXT: fence.acquire.cta;
+; SM90-FTZ-ALLOW-NEXT: st.param.b16 [func_retval0], %r13;
+; SM90-FTZ-ALLOW-NEXT: ret;
%retval = atomicrmw fmin ptr addrspace(1) %addr, half %val syncscope("block") acq_rel
ret half %retval
}
define half @fmax_acq_rel_half_global_cta(ptr addrspace(1) %addr, half %val) {
-; SM90-NOFTZ-LABEL: fmax_acq_rel_half_global_cta(
-; SM90-NOFTZ: {
-; SM90-NOFTZ-NEXT: .reg .pred %p<2>;
-; SM90-NOFTZ-NEXT: .reg .b16 %rs<4>;
-; SM90-NOFTZ-NEXT: .reg .b32 %r<15>;
-; SM90-NOFTZ-NEXT: .reg .b64 %rd<3>;
-; SM90-NOFTZ-EMPTY:
-; SM90-NOFTZ-NEXT: // %bb.0:
-; SM90-NOFTZ-NEXT: ld.param.b16 %rs1, [fmax_acq_rel_half_global_cta_param_1];
-; SM90-NOFTZ-NEXT: ld.param.b64 %rd2, [fmax_acq_rel_half_global_cta_param_0];
-; SM90-NOFTZ-NEXT: fence.release.cta;
-; SM90-NOFTZ-NEXT: and.b64 %rd1, %rd2, -4;
-; SM90-NOFTZ-NEXT: cvt.u32.u64 %r4, %rd2;
-; SM90-NOFTZ-NEXT: and.b32 %r5, %r4, 3;
-; SM90-NOFTZ-NEXT: shl.b32 %r1, %r5, 3;
-; SM90-NOFTZ-NEXT: mov.b32 %r6, 65535;
-; SM90-NOFTZ-NEXT: shl.b32 %r7, %r6, %r1;
-; SM90-NOFTZ-NEXT: not.b32 %r2, %r7;
-; SM90-NOFTZ-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
-; SM90-NOFTZ-NEXT: $L__BB75_1: // %atomicrmw.start
-; SM90-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NOFTZ-NEXT: shr.u32 %r8, %r14, %r1;
-; SM90-NOFTZ-NEXT: cvt.u16.u32 %rs2, %r8;
-; SM90-NOFTZ-NEXT: max.f16 %rs3, %rs2, %rs1;
-; SM90-NOFTZ-NEXT: cvt.u32.u16 %r9, %rs3;
-; SM90-NOFTZ-NEXT: shl.b32 %r10, %r9, %r1;
-; SM90-NOFTZ-NEXT: and.b32 %r11, %r14, %r2;
-; SM90-NOFTZ-NEXT: or.b32 %r12, %r11, %r10;
-; SM90-NOFTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
-; SM90-NOFTZ-NEXT: setp.ne.b32 %p1, %r3, %r14;
-; SM90-NOFTZ-NEXT: mov.b32 %r14, %r3;
-; SM90-NOFTZ-NEXT: @%p1 bra $L__BB75_1;
-; SM90-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NOFTZ-NEXT: shr.u32 %r13, %r3, %r1;
-; SM90-NOFTZ-NEXT: fence.acquire.cta;
-; SM90-NOFTZ-NEXT: st.param.b16 [func_retval0], %r13;
-; SM90-NOFTZ-NEXT: ret;
+; SM90-NOFTZ-DISALLOW-LABEL: fmax_acq_rel_half_global_cta(
+; SM90-NOFTZ-DISALLOW: {
+; SM90-NOFTZ-DISALLOW-NEXT: .reg .pred %p<2>;
+; SM90-NOFTZ-DISALLOW-NEXT: .reg .b16 %rs<4>;
+; SM90-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<15>;
+; SM90-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
+; SM90-NOFTZ-DISALLOW-EMPTY:
+; SM90-NOFTZ-DISALLOW-NEXT: // %bb.0:
+; SM90-NOFTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fmax_acq_rel_half_global_cta_param_1];
+; SM90-NOFTZ-DISALLOW-NEXT: ld.param.b64 %rd2, [fmax_acq_rel_half_global_cta_param_0];
+; SM90-NOFTZ-DISALLOW-NEXT: fence.release.cta;
+; SM90-NOFTZ-DISALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM90-NOFTZ-DISALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM90-NOFTZ-DISALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM90-NOFTZ-DISALLOW-NEXT: mov.b32 %r6, 65535;
+; SM90-NOFTZ-DISALLOW-NEXT: shl.b32 %r7, %r6, %r1;
+; SM90-NOFTZ-DISALLOW-NEXT: not.b32 %r2, %r7;
+; SM90-NOFTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM90-NOFTZ-DISALLOW-NEXT: $L__BB75_1: // %atomicrmw.start
+; SM90-NOFTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NOFTZ-DISALLOW-NEXT: shr.u32 %r8, %r14, %r1;
+; SM90-NOFTZ-DISALLOW-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM90-NOFTZ-DISALLOW-NEXT: max.f16 %rs3, %rs2, %rs1;
+; SM90-NOFTZ-DISALLOW-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM90-NOFTZ-DISALLOW-NEXT: shl.b32 %r10, %r9, %r1;
+; SM90-NOFTZ-DISALLOW-NEXT: and.b32 %r11, %r14, %r2;
+; SM90-NOFTZ-DISALLOW-NEXT: or.b32 %r12, %r11, %r10;
+; SM90-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM90-NOFTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM90-NOFTZ-DISALLOW-NEXT: mov.b32 %r14, %r3;
+; SM90-NOFTZ-DISALLOW-NEXT: @%p1 bra $L__BB75_1;
+; SM90-NOFTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NOFTZ-DISALLOW-NEXT: shr.u32 %r13, %r3, %r1;
+; SM90-NOFTZ-DISALLOW-NEXT: fence.acquire.cta;
+; SM90-NOFTZ-DISALLOW-NEXT: st.param.b16 [func_retval0], %r13;
+; SM90-NOFTZ-DISALLOW-NEXT: ret;
+;
+; SM90-NOFTZ-ALLOW-LABEL: fmax_acq_rel_half_global_cta(
+; SM90-NOFTZ-ALLOW: {
+; SM90-NOFTZ-ALLOW-NEXT: .reg .pred %p<2>;
+; SM90-NOFTZ-ALLOW-NEXT: .reg .b16 %rs<4>;
+; SM90-NOFTZ-ALLOW-NEXT: .reg .b32 %r<15>;
+; SM90-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
+; SM90-NOFTZ-ALLOW-EMPTY:
+; SM90-NOFTZ-ALLOW-NEXT: // %bb.0:
+; SM90-NOFTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fmax_acq_rel_half_global_cta_param_1];
+; SM90-NOFTZ-ALLOW-NEXT: ld.param.b64 %rd2, [fmax_acq_rel_half_global_cta_param_0];
+; SM90-NOFTZ-ALLOW-NEXT: fence.release.cta;
+; SM90-NOFTZ-ALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NOFTZ-ALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM90-NOFTZ-ALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM90-NOFTZ-ALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM90-NOFTZ-ALLOW-NEXT: mov.b32 %r6, 65535;
+; SM90-NOFTZ-ALLOW-NEXT: shl.b32 %r7, %r6, %r1;
+; SM90-NOFTZ-ALLOW-NEXT: not.b32 %r2, %r7;
+; SM90-NOFTZ-ALLOW-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM90-NOFTZ-ALLOW-NEXT: $L__BB75_1: // %atomicrmw.start
+; SM90-NOFTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NOFTZ-ALLOW-NEXT: shr.u32 %r8, %r14, %r1;
+; SM90-NOFTZ-ALLOW-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM90-NOFTZ-ALLOW-NEXT: max.f16 %rs3, %rs2, %rs1;
+; SM90-NOFTZ-ALLOW-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM90-NOFTZ-ALLOW-NEXT: shl.b32 %r10, %r9, %r1;
+; SM90-NOFTZ-ALLOW-NEXT: and.b32 %r11, %r14, %r2;
+; SM90-NOFTZ-ALLOW-NEXT: or.b32 %r12, %r11, %r10;
+; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM90-NOFTZ-ALLOW-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM90-NOFTZ-ALLOW-NEXT: mov.b32 %r14, %r3;
+; SM90-NOFTZ-ALLOW-NEXT: @%p1 bra $L__BB75_1;
+; SM90-NOFTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NOFTZ-ALLOW-NEXT: shr.u32 %r13, %r3, %r1;
+; SM90-NOFTZ-ALLOW-NEXT: fence.acquire.cta;
+; SM90-NOFTZ-ALLOW-NEXT: st.param.b16 [func_retval0], %r13;
+; SM90-NOFTZ-ALLOW-NEXT: ret;
+;
+; SM90-FTZ-DISALLOW-LABEL: fmax_acq_rel_half_global_cta(
+; SM90-FTZ-DISALLOW: {
+; SM90-FTZ-DISALLOW-NEXT: .reg .pred %p<2>;
+; SM90-FTZ-DISALLOW-NEXT: .reg .b16 %rs<4>;
+; SM90-FTZ-DISALLOW-NEXT: .reg .b32 %r<15>;
+; SM90-FTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
+; SM90-FTZ-DISALLOW-EMPTY:
+; SM90-FTZ-DISALLOW-NEXT: // %bb.0:
+; SM90-FTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fmax_acq_rel_half_global_cta_param_1];
+; SM90-FTZ-DISALLOW-NEXT: ld.param.b64 %rd2, [fmax_acq_rel_half_global_cta_param_0];
+; SM90-FTZ-DISALLOW-NEXT: fence.release.cta;
+; SM90-FTZ-DISALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-FTZ-DISALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM90-FTZ-DISALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM90-FTZ-DISALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM90-FTZ-DISALLOW-NEXT: mov.b32 %r6, 65535;
+; SM90-FTZ-DISALLOW-NEXT: shl.b32 %r7, %r6, %r1;
+; SM90-FTZ-DISALLOW-NEXT: not.b32 %r2, %r7;
+; SM90-FTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM90-FTZ-DISALLOW-NEXT: $L__BB75_1: // %atomicrmw.start
+; SM90-FTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-FTZ-DISALLOW-NEXT: shr.u32 %r8, %r14, %r1;
+; SM90-FTZ-DISALLOW-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM90-FTZ-DISALLOW-NEXT: max.ftz.f16 %rs3, %rs2, %rs1;
+; SM90-FTZ-DISALLOW-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM90-FTZ-DISALLOW-NEXT: shl.b32 %r10, %r9, %r1;
+; SM90-FTZ-DISALLOW-NEXT: and.b32 %r11, %r14, %r2;
+; SM90-FTZ-DISALLOW-NEXT: or.b32 %r12, %r11, %r10;
+; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM90-FTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM90-FTZ-DISALLOW-NEXT: mov.b32 %r14, %r3;
+; SM90-FTZ-DISALLOW-NEXT: @%p1 bra $L__BB75_1;
+; SM90-FTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-FTZ-DISALLOW-NEXT: shr.u32 %r13, %r3, %r1;
+; SM90-FTZ-DISALLOW-NEXT: fence.acquire.cta;
+; SM90-FTZ-DISALLOW-NEXT: st.param.b16 [func_retval0], %r13;
+; SM90-FTZ-DISALLOW-NEXT: ret;
;
-; SM90-FTZ-LABEL: fmax_acq_rel_half_global_cta(
-; SM90-FTZ: {
-; SM90-FTZ-NEXT: .reg .pred %p<2>;
-; SM90-FTZ-NEXT: .reg .b16 %rs<4>;
-; SM90-FTZ-NEXT: .reg .b32 %r<15>;
-; SM90-FTZ-NEXT: .reg .b64 %rd<3>;
-; SM90-FTZ-EMPTY:
-; SM90-FTZ-NEXT: // %bb.0:
-; SM90-FTZ-NEXT: ld.param.b16 %rs1, [fmax_acq_rel_half_global_cta_param_1];
-; SM90-FTZ-NEXT: ld.param.b64 %rd2, [fmax_acq_rel_half_global_cta_param_0];
-; SM90-FTZ-NEXT: fence.release.cta;
-; SM90-FTZ-NEXT: and.b64 %rd1, %rd2, -4;
-; SM90-FTZ-NEXT: cvt.u32.u64 %r4, %rd2;
-; SM90-FTZ-NEXT: and.b32 %r5, %r4, 3;
-; SM90-FTZ-NEXT: shl.b32 %r1, %r5, 3;
-; SM90-FTZ-NEXT: mov.b32 %r6, 65535;
-; SM90-FTZ-NEXT: shl.b32 %r7, %r6, %r1;
-; SM90-FTZ-NEXT: not.b32 %r2, %r7;
-; SM90-FTZ-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
-; SM90-FTZ-NEXT: $L__BB75_1: // %atomicrmw.start
-; SM90-FTZ-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-FTZ-NEXT: shr.u32 %r8, %r14, %r1;
-; SM90-FTZ-NEXT: cvt.u16.u32 %rs2, %r8;
-; SM90-FTZ-NEXT: max.ftz.f16 %rs3, %rs2, %rs1;
-; SM90-FTZ-NEXT: cvt.u32.u16 %r9, %rs3;
-; SM90-FTZ-NEXT: shl.b32 %r10, %r9, %r1;
-; SM90-FTZ-NEXT: and.b32 %r11, %r14, %r2;
-; SM90-FTZ-NEXT: or.b32 %r12, %r11, %r10;
-; SM90-FTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
-; SM90-FTZ-NEXT: setp.ne.b32 %p1, %r3, %r14;
-; SM90-FTZ-NEXT: mov.b32 %r14, %r3;
-; SM90-FTZ-NEXT: @%p1 bra $L__BB75_1;
-; SM90-FTZ-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-FTZ-NEXT: shr.u32 %r13, %r3, %r1;
-; SM90-FTZ-NEXT: fence.acquire.cta;
-; SM90-FTZ-NEXT: st.param.b16 [func_retval0], %r13;
-; SM90-FTZ-NEXT: ret;
+; SM90-FTZ-ALLOW-LABEL: fmax_acq_rel_half_global_cta(
+; SM90-FTZ-ALLOW: {
+; SM90-FTZ-ALLOW-NEXT: .reg .pred %p<2>;
+; SM90-FTZ-ALLOW-NEXT: .reg .b16 %rs<4>;
+; SM90-FTZ-ALLOW-NEXT: .reg .b32 %r<15>;
+; SM90-FTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
+; SM90-FTZ-ALLOW-EMPTY:
+; SM90-FTZ-ALLOW-NEXT: // %bb.0:
+; SM90-FTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fmax_acq_rel_half_global_cta_param_1];
+; SM90-FTZ-ALLOW-NEXT: ld.param.b64 %rd2, [fmax_acq_rel_half_global_cta_param_0];
+; SM90-FTZ-ALLOW-NEXT: fence.release.cta;
+; SM90-FTZ-ALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-FTZ-ALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM90-FTZ-ALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM90-FTZ-ALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM90-FTZ-ALLOW-NEXT: mov.b32 %r6, 65535;
+; SM90-FTZ-ALLOW-NEXT: shl.b32 %r7, %r6, %r1;
+; SM90-FTZ-ALLOW-NEXT: not.b32 %r2, %r7;
+; SM90-FTZ-ALLOW-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM90-FTZ-ALLOW-NEXT: $L__BB75_1: // %atomicrmw.start
+; SM90-FTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-FTZ-ALLOW-NEXT: shr.u32 %r8, %r14, %r1;
+; SM90-FTZ-ALLOW-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM90-FTZ-ALLOW-NEXT: max.ftz.f16 %rs3, %rs2, %rs1;
+; SM90-FTZ-ALLOW-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM90-FTZ-ALLOW-NEXT: shl.b32 %r10, %r9, %r1;
+; SM90-FTZ-ALLOW-NEXT: and.b32 %r11, %r14, %r2;
+; SM90-FTZ-ALLOW-NEXT: or.b32 %r12, %r11, %r10;
+; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM90-FTZ-ALLOW-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM90-FTZ-ALLOW-NEXT: mov.b32 %r14, %r3;
+; SM90-FTZ-ALLOW-NEXT: @%p1 bra $L__BB75_1;
+; SM90-FTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-FTZ-ALLOW-NEXT: shr.u32 %r13, %r3, %r1;
+; SM90-FTZ-ALLOW-NEXT: fence.acquire.cta;
+; SM90-FTZ-ALLOW-NEXT: st.param.b16 [func_retval0], %r13;
+; SM90-FTZ-ALLOW-NEXT: ret;
%retval = atomicrmw fmax ptr addrspace(1) %addr, half %val syncscope("block") acq_rel
ret half %retval
}
define half @fminimum_acq_rel_half_global_cta(ptr addrspace(1) %addr, half %val) {
-; SM90-NOFTZ-LABEL: fminimum_acq_rel_half_global_cta(
-; SM90-NOFTZ: {
-; SM90-NOFTZ-NEXT: .reg .pred %p<2>;
-; SM90-NOFTZ-NEXT: .reg .b16 %rs<4>;
-; SM90-NOFTZ-NEXT: .reg .b32 %r<15>;
-; SM90-NOFTZ-NEXT: .reg .b64 %rd<3>;
-; SM90-NOFTZ-EMPTY:
-; SM90-NOFTZ-NEXT: // %bb.0:
-; SM90-NOFTZ-NEXT: ld.param.b16 %rs1, [fminimum_acq_rel_half_global_cta_param_1];
-; SM90-NOFTZ-NEXT: ld.param.b64 %rd2, [fminimum_acq_rel_half_global_cta_param_0];
-; SM90-NOFTZ-NEXT: fence.release.cta;
-; SM90-NOFTZ-NEXT: and.b64 %rd1, %rd2, -4;
-; SM90-NOFTZ-NEXT: cvt.u32.u64 %r4, %rd2;
-; SM90-NOFTZ-NEXT: and.b32 %r5, %r4, 3;
-; SM90-NOFTZ-NEXT: shl.b32 %r1, %r5, 3;
-; SM90-NOFTZ-NEXT: mov.b32 %r6, 65535;
-; SM90-NOFTZ-NEXT: shl.b32 %r7, %r6, %r1;
-; SM90-NOFTZ-NEXT: not.b32 %r2, %r7;
-; SM90-NOFTZ-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
-; SM90-NOFTZ-NEXT: $L__BB76_1: // %atomicrmw.start
-; SM90-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NOFTZ-NEXT: shr.u32 %r8, %r14, %r1;
-; SM90-NOFTZ-NEXT: cvt.u16.u32 %rs2, %r8;
-; SM90-NOFTZ-NEXT: min.NaN.f16 %rs3, %rs2, %rs1;
-; SM90-NOFTZ-NEXT: cvt.u32.u16 %r9, %rs3;
-; SM90-NOFTZ-NEXT: shl.b32 %r10, %r9, %r1;
-; SM90-NOFTZ-NEXT: and.b32 %r11, %r14, %r2;
-; SM90-NOFTZ-NEXT: or.b32 %r12, %r11, %r10;
-; SM90-NOFTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
-; SM90-NOFTZ-NEXT: setp.ne.b32 %p1, %r3, %r14;
-; SM90-NOFTZ-NEXT: mov.b32 %r14, %r3;
-; SM90-NOFTZ-NEXT: @%p1 bra $L__BB76_1;
-; SM90-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NOFTZ-NEXT: shr.u32 %r13, %r3, %r1;
-; SM90-NOFTZ-NEXT: fence.acquire.cta;
-; SM90-NOFTZ-NEXT: st.param.b16 [func_retval0], %r13;
-; SM90-NOFTZ-NEXT: ret;
+; SM90-NOFTZ-DISALLOW-LABEL: fminimum_acq_rel_half_global_cta(
+; SM90-NOFTZ-DISALLOW: {
+; SM90-NOFTZ-DISALLOW-NEXT: .reg .pred %p<2>;
+; SM90-NOFTZ-DISALLOW-NEXT: .reg .b16 %rs<4>;
+; SM90-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<15>;
+; SM90-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
+; SM90-NOFTZ-DISALLOW-EMPTY:
+; SM90-NOFTZ-DISALLOW-NEXT: // %bb.0:
+; SM90-NOFTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fminimum_acq_rel_half_global_cta_param_1];
+; SM90-NOFTZ-DISALLOW-NEXT: ld.param.b64 %rd2, [fminimum_acq_rel_half_global_cta_param_0];
+; SM90-NOFTZ-DISALLOW-NEXT: fence.release.cta;
+; SM90-NOFTZ-DISALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM90-NOFTZ-DISALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM90-NOFTZ-DISALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM90-NOFTZ-DISALLOW-NEXT: mov.b32 %r6, 65535;
+; SM90-NOFTZ-DISALLOW-NEXT: shl.b32 %r7, %r6, %r1;
+; SM90-NOFTZ-DISALLOW-NEXT: not.b32 %r2, %r7;
+; SM90-NOFTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM90-NOFTZ-DISALLOW-NEXT: $L__BB76_1: // %atomicrmw.start
+; SM90-NOFTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NOFTZ-DISALLOW-NEXT: shr.u32 %r8, %r14, %r1;
+; SM90-NOFTZ-DISALLOW-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM90-NOFTZ-DISALLOW-NEXT: min.NaN.f16 %rs3, %rs2, %rs1;
+; SM90-NOFTZ-DISALLOW-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM90-NOFTZ-DISALLOW-NEXT: shl.b32 %r10, %r9, %r1;
+; SM90-NOFTZ-DISALLOW-NEXT: and.b32 %r11, %r14, %r2;
+; SM90-NOFTZ-DISALLOW-NEXT: or.b32 %r12, %r11, %r10;
+; SM90-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM90-NOFTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM90-NOFTZ-DISALLOW-NEXT: mov.b32 %r14, %r3;
+; SM90-NOFTZ-DISALLOW-NEXT: @%p1 bra $L__BB76_1;
+; SM90-NOFTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NOFTZ-DISALLOW-NEXT: shr.u32 %r13, %r3, %r1;
+; SM90-NOFTZ-DISALLOW-NEXT: fence.acquire.cta;
+; SM90-NOFTZ-DISALLOW-NEXT: st.param.b16 [func_retval0], %r13;
+; SM90-NOFTZ-DISALLOW-NEXT: ret;
+;
+; SM90-NOFTZ-ALLOW-LABEL: fminimum_acq_rel_half_global_cta(
+; SM90-NOFTZ-ALLOW: {
+; SM90-NOFTZ-ALLOW-NEXT: .reg .pred %p<2>;
+; SM90-NOFTZ-ALLOW-NEXT: .reg .b16 %rs<4>;
+; SM90-NOFTZ-ALLOW-NEXT: .reg .b32 %r<15>;
+; SM90-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
+; SM90-NOFTZ-ALLOW-EMPTY:
+; SM90-NOFTZ-ALLOW-NEXT: // %bb.0:
+; SM90-NOFTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fminimum_acq_rel_half_global_cta_param_1];
+; SM90-NOFTZ-ALLOW-NEXT: ld.param.b64 %rd2, [fminimum_acq_rel_half_global_cta_param_0];
+; SM90-NOFTZ-ALLOW-NEXT: fence.release.cta;
+; SM90-NOFTZ-ALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NOFTZ-ALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM90-NOFTZ-ALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM90-NOFTZ-ALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM90-NOFTZ-ALLOW-NEXT: mov.b32 %r6, 65535;
+; SM90-NOFTZ-ALLOW-NEXT: shl.b32 %r7, %r6, %r1;
+; SM90-NOFTZ-ALLOW-NEXT: not.b32 %r2, %r7;
+; SM90-NOFTZ-ALLOW-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM90-NOFTZ-ALLOW-NEXT: $L__BB76_1: // %atomicrmw.start
+; SM90-NOFTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NOFTZ-ALLOW-NEXT: shr.u32 %r8, %r14, %r1;
+; SM90-NOFTZ-ALLOW-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM90-NOFTZ-ALLOW-NEXT: min.NaN.f16 %rs3, %rs2, %rs1;
+; SM90-NOFTZ-ALLOW-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM90-NOFTZ-ALLOW-NEXT: shl.b32 %r10, %r9, %r1;
+; SM90-NOFTZ-ALLOW-NEXT: and.b32 %r11, %r14, %r2;
+; SM90-NOFTZ-ALLOW-NEXT: or.b32 %r12, %r11, %r10;
+; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM90-NOFTZ-ALLOW-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM90-NOFTZ-ALLOW-NEXT: mov.b32 %r14, %r3;
+; SM90-NOFTZ-ALLOW-NEXT: @%p1 bra $L__BB76_1;
+; SM90-NOFTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NOFTZ-ALLOW-NEXT: shr.u32 %r13, %r3, %r1;
+; SM90-NOFTZ-ALLOW-NEXT: fence.acquire.cta;
+; SM90-NOFTZ-ALLOW-NEXT: st.param.b16 [func_retval0], %r13;
+; SM90-NOFTZ-ALLOW-NEXT: ret;
+;
+; SM90-FTZ-DISALLOW-LABEL: fminimum_acq_rel_half_global_cta(
+; SM90-FTZ-DISALLOW: {
+; SM90-FTZ-DISALLOW-NEXT: .reg .pred %p<2>;
+; SM90-FTZ-DISALLOW-NEXT: .reg .b16 %rs<4>;
+; SM90-FTZ-DISALLOW-NEXT: .reg .b32 %r<15>;
+; SM90-FTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
+; SM90-FTZ-DISALLOW-EMPTY:
+; SM90-FTZ-DISALLOW-NEXT: // %bb.0:
+; SM90-FTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fminimum_acq_rel_half_global_cta_param_1];
+; SM90-FTZ-DISALLOW-NEXT: ld.param.b64 %rd2, [fminimum_acq_rel_half_global_cta_param_0];
+; SM90-FTZ-DISALLOW-NEXT: fence.release.cta;
+; SM90-FTZ-DISALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-FTZ-DISALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM90-FTZ-DISALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM90-FTZ-DISALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM90-FTZ-DISALLOW-NEXT: mov.b32 %r6, 65535;
+; SM90-FTZ-DISALLOW-NEXT: shl.b32 %r7, %r6, %r1;
+; SM90-FTZ-DISALLOW-NEXT: not.b32 %r2, %r7;
+; SM90-FTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM90-FTZ-DISALLOW-NEXT: $L__BB76_1: // %atomicrmw.start
+; SM90-FTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-FTZ-DISALLOW-NEXT: shr.u32 %r8, %r14, %r1;
+; SM90-FTZ-DISALLOW-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM90-FTZ-DISALLOW-NEXT: min.ftz.NaN.f16 %rs3, %rs2, %rs1;
+; SM90-FTZ-DISALLOW-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM90-FTZ-DISALLOW-NEXT: shl.b32 %r10, %r9, %r1;
+; SM90-FTZ-DISALLOW-NEXT: and.b32 %r11, %r14, %r2;
+; SM90-FTZ-DISALLOW-NEXT: or.b32 %r12, %r11, %r10;
+; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM90-FTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM90-FTZ-DISALLOW-NEXT: mov.b32 %r14, %r3;
+; SM90-FTZ-DISALLOW-NEXT: @%p1 bra $L__BB76_1;
+; SM90-FTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-FTZ-DISALLOW-NEXT: shr.u32 %r13, %r3, %r1;
+; SM90-FTZ-DISALLOW-NEXT: fence.acquire.cta;
+; SM90-FTZ-DISALLOW-NEXT: st.param.b16 [func_retval0], %r13;
+; SM90-FTZ-DISALLOW-NEXT: ret;
;
-; SM90-FTZ-LABEL: fminimum_acq_rel_half_global_cta(
-; SM90-FTZ: {
-; SM90-FTZ-NEXT: .reg .pred %p<2>;
-; SM90-FTZ-NEXT: .reg .b16 %rs<4>;
-; SM90-FTZ-NEXT: .reg .b32 %r<15>;
-; SM90-FTZ-NEXT: .reg .b64 %rd<3>;
-; SM90-FTZ-EMPTY:
-; SM90-FTZ-NEXT: // %bb.0:
-; SM90-FTZ-NEXT: ld.param.b16 %rs1, [fminimum_acq_rel_half_global_cta_param_1];
-; SM90-FTZ-NEXT: ld.param.b64 %rd2, [fminimum_acq_rel_half_global_cta_param_0];
-; SM90-FTZ-NEXT: fence.release.cta;
-; SM90-FTZ-NEXT: and.b64 %rd1, %rd2, -4;
-; SM90-FTZ-NEXT: cvt.u32.u64 %r4, %rd2;
-; SM90-FTZ-NEXT: and.b32 %r5, %r4, 3;
-; SM90-FTZ-NEXT: shl.b32 %r1, %r5, 3;
-; SM90-FTZ-NEXT: mov.b32 %r6, 65535;
-; SM90-FTZ-NEXT: shl.b32 %r7, %r6, %r1;
-; SM90-FTZ-NEXT: not.b32 %r2, %r7;
-; SM90-FTZ-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
-; SM90-FTZ-NEXT: $L__BB76_1: // %atomicrmw.start
-; SM90-FTZ-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-FTZ-NEXT: shr.u32 %r8, %r14, %r1;
-; SM90-FTZ-NEXT: cvt.u16.u32 %rs2, %r8;
-; SM90-FTZ-NEXT: min.ftz.NaN.f16 %rs3, %rs2, %rs1;
-; SM90-FTZ-NEXT: cvt.u32.u16 %r9, %rs3;
-; SM90-FTZ-NEXT: shl.b32 %r10, %r9, %r1;
-; SM90-FTZ-NEXT: and.b32 %r11, %r14, %r2;
-; SM90-FTZ-NEXT: or.b32 %r12, %r11, %r10;
-; SM90-FTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
-; SM90-FTZ-NEXT: setp.ne.b32 %p1, %r3, %r14;
-; SM90-FTZ-NEXT: mov.b32 %r14, %r3;
-; SM90-FTZ-NEXT: @%p1 bra $L__BB76_1;
-; SM90-FTZ-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-FTZ-NEXT: shr.u32 %r13, %r3, %r1;
-; SM90-FTZ-NEXT: fence.acquire.cta;
-; SM90-FTZ-NEXT: st.param.b16 [func_retval0], %r13;
-; SM90-FTZ-NEXT: ret;
+; SM90-FTZ-ALLOW-LABEL: fminimum_acq_rel_half_global_cta(
+; SM90-FTZ-ALLOW: {
+; SM90-FTZ-ALLOW-NEXT: .reg .pred %p<2>;
+; SM90-FTZ-ALLOW-NEXT: .reg .b16 %rs<4>;
+; SM90-FTZ-ALLOW-NEXT: .reg .b32 %r<15>;
+; SM90-FTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
+; SM90-FTZ-ALLOW-EMPTY:
+; SM90-FTZ-ALLOW-NEXT: // %bb.0:
+; SM90-FTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fminimum_acq_rel_half_global_cta_param_1];
+; SM90-FTZ-ALLOW-NEXT: ld.param.b64 %rd2, [fminimum_acq_rel_half_global_cta_param_0];
+; SM90-FTZ-ALLOW-NEXT: fence.release.cta;
+; SM90-FTZ-ALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-FTZ-ALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM90-FTZ-ALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM90-FTZ-ALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM90-FTZ-ALLOW-NEXT: mov.b32 %r6, 65535;
+; SM90-FTZ-ALLOW-NEXT: shl.b32 %r7, %r6, %r1;
+; SM90-FTZ-ALLOW-NEXT: not.b32 %r2, %r7;
+; SM90-FTZ-ALLOW-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM90-FTZ-ALLOW-NEXT: $L__BB76_1: // %atomicrmw.start
+; SM90-FTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-FTZ-ALLOW-NEXT: shr.u32 %r8, %r14, %r1;
+; SM90-FTZ-ALLOW-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM90-FTZ-ALLOW-NEXT: min.ftz.NaN.f16 %rs3, %rs2, %rs1;
+; SM90-FTZ-ALLOW-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM90-FTZ-ALLOW-NEXT: shl.b32 %r10, %r9, %r1;
+; SM90-FTZ-ALLOW-NEXT: and.b32 %r11, %r14, %r2;
+; SM90-FTZ-ALLOW-NEXT: or.b32 %r12, %r11, %r10;
+; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM90-FTZ-ALLOW-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM90-FTZ-ALLOW-NEXT: mov.b32 %r14, %r3;
+; SM90-FTZ-ALLOW-NEXT: @%p1 bra $L__BB76_1;
+; SM90-FTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-FTZ-ALLOW-NEXT: shr.u32 %r13, %r3, %r1;
+; SM90-FTZ-ALLOW-NEXT: fence.acquire.cta;
+; SM90-FTZ-ALLOW-NEXT: st.param.b16 [func_retval0], %r13;
+; SM90-FTZ-ALLOW-NEXT: ret;
%retval = atomicrmw fminimum ptr addrspace(1) %addr, half %val syncscope("block") acq_rel
ret half %retval
}
define half @fmaximum_acq_rel_half_global_cta(ptr addrspace(1) %addr, half %val) {
-; SM90-NOFTZ-LABEL: fmaximum_acq_rel_half_global_cta(
-; SM90-NOFTZ: {
-; SM90-NOFTZ-NEXT: .reg .pred %p<2>;
-; SM90-NOFTZ-NEXT: .reg .b16 %rs<4>;
-; SM90-NOFTZ-NEXT: .reg .b32 %r<15>;
-; SM90-NOFTZ-NEXT: .reg .b64 %rd<3>;
-; SM90-NOFTZ-EMPTY:
-; SM90-NOFTZ-NEXT: // %bb.0:
-; SM90-NOFTZ-NEXT: ld.param.b16 %rs1, [fmaximum_acq_rel_half_global_cta_param_1];
-; SM90-NOFTZ-NEXT: ld.param.b64 %rd2, [fmaximum_acq_rel_half_global_cta_param_0];
-; SM90-NOFTZ-NEXT: fence.release.cta;
-; SM90-NOFTZ-NEXT: and.b64 %rd1, %rd2, -4;
-; SM90-NOFTZ-NEXT: cvt.u32.u64 %r4, %rd2;
-; SM90-NOFTZ-NEXT: and.b32 %r5, %r4, 3;
-; SM90-NOFTZ-NEXT: shl.b32 %r1, %r5, 3;
-; SM90-NOFTZ-NEXT: mov.b32 %r6, 65535;
-; SM90-NOFTZ-NEXT: shl.b32 %r7, %r6, %r1;
-; SM90-NOFTZ-NEXT: not.b32 %r2, %r7;
-; SM90-NOFTZ-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
-; SM90-NOFTZ-NEXT: $L__BB77_1: // %atomicrmw.start
-; SM90-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NOFTZ-NEXT: shr.u32 %r8, %r14, %r1;
-; SM90-NOFTZ-NEXT: cvt.u16.u32 %rs2, %r8;
-; SM90-NOFTZ-NEXT: max.NaN.f16 %rs3, %rs2, %rs1;
-; SM90-NOFTZ-NEXT: cvt.u32.u16 %r9, %rs3;
-; SM90-NOFTZ-NEXT: shl.b32 %r10, %r9, %r1;
-; SM90-NOFTZ-NEXT: and.b32 %r11, %r14, %r2;
-; SM90-NOFTZ-NEXT: or.b32 %r12, %r11, %r10;
-; SM90-NOFTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
-; SM90-NOFTZ-NEXT: setp.ne.b32 %p1, %r3, %r14;
-; SM90-NOFTZ-NEXT: mov.b32 %r14, %r3;
-; SM90-NOFTZ-NEXT: @%p1 bra $L__BB77_1;
-; SM90-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NOFTZ-NEXT: shr.u32 %r13, %r3, %r1;
-; SM90-NOFTZ-NEXT: fence.acquire.cta;
-; SM90-NOFTZ-NEXT: st.param.b16 [func_retval0], %r13;
-; SM90-NOFTZ-NEXT: ret;
+; SM90-NOFTZ-DISALLOW-LABEL: fmaximum_acq_rel_half_global_cta(
+; SM90-NOFTZ-DISALLOW: {
+; SM90-NOFTZ-DISALLOW-NEXT: .reg .pred %p<2>;
+; SM90-NOFTZ-DISALLOW-NEXT: .reg .b16 %rs<4>;
+; SM90-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<15>;
+; SM90-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
+; SM90-NOFTZ-DISALLOW-EMPTY:
+; SM90-NOFTZ-DISALLOW-NEXT: // %bb.0:
+; SM90-NOFTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fmaximum_acq_rel_half_global_cta_param_1];
+; SM90-NOFTZ-DISALLOW-NEXT: ld.param.b64 %rd2, [fmaximum_acq_rel_half_global_cta_param_0];
+; SM90-NOFTZ-DISALLOW-NEXT: fence.release.cta;
+; SM90-NOFTZ-DISALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NOFTZ-DISALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM90-NOFTZ-DISALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM90-NOFTZ-DISALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM90-NOFTZ-DISALLOW-NEXT: mov.b32 %r6, 65535;
+; SM90-NOFTZ-DISALLOW-NEXT: shl.b32 %r7, %r6, %r1;
+; SM90-NOFTZ-DISALLOW-NEXT: not.b32 %r2, %r7;
+; SM90-NOFTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM90-NOFTZ-DISALLOW-NEXT: $L__BB77_1: // %atomicrmw.start
+; SM90-NOFTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NOFTZ-DISALLOW-NEXT: shr.u32 %r8, %r14, %r1;
+; SM90-NOFTZ-DISALLOW-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM90-NOFTZ-DISALLOW-NEXT: max.NaN.f16 %rs3, %rs2, %rs1;
+; SM90-NOFTZ-DISALLOW-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM90-NOFTZ-DISALLOW-NEXT: shl.b32 %r10, %r9, %r1;
+; SM90-NOFTZ-DISALLOW-NEXT: and.b32 %r11, %r14, %r2;
+; SM90-NOFTZ-DISALLOW-NEXT: or.b32 %r12, %r11, %r10;
+; SM90-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM90-NOFTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM90-NOFTZ-DISALLOW-NEXT: mov.b32 %r14, %r3;
+; SM90-NOFTZ-DISALLOW-NEXT: @%p1 bra $L__BB77_1;
+; SM90-NOFTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NOFTZ-DISALLOW-NEXT: shr.u32 %r13, %r3, %r1;
+; SM90-NOFTZ-DISALLOW-NEXT: fence.acquire.cta;
+; SM90-NOFTZ-DISALLOW-NEXT: st.param.b16 [func_retval0], %r13;
+; SM90-NOFTZ-DISALLOW-NEXT: ret;
;
-; SM90-FTZ-LABEL: fmaximum_acq_rel_half_global_cta(
-; SM90-FTZ: {
-; SM90-FTZ-NEXT: .reg .pred %p<2>;
-; SM90-FTZ-NEXT: .reg .b16 %rs<4>;
-; SM90-FTZ-NEXT: .reg .b32 %r<15>;
-; SM90-FTZ-NEXT: .reg .b64 %rd<3>;
-; SM90-FTZ-EMPTY:
-; SM90-FTZ-NEXT: // %bb.0:
-; SM90-FTZ-NEXT: ld.param.b16 %rs1, [fmaximum_acq_rel_half_global_cta_param_1];
-; SM90-FTZ-NEXT: ld.param.b64 %rd2, [fmaximum_acq_rel_half_global_cta_param_0];
-; SM90-FTZ-NEXT: fence.release.cta;
-; SM90-FTZ-NEXT: and.b64 %rd1, %rd2, -4;
-; SM90-FTZ-NEXT: cvt.u32.u64 %r4, %rd2;
-; SM90-FTZ-NEXT: and.b32 %r5, %r4, 3;
-; SM90-FTZ-NEXT: shl.b32 %r1, %r5, 3;
-; SM90-FTZ-NEXT: mov.b32 %r6, 65535;
-; SM90-FTZ-NEXT: shl.b32 %r7, %r6, %r1;
-; SM90-FTZ-NEXT: not.b32 %r2, %r7;
-; SM90-FTZ-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
-; SM90-FTZ-NEXT: $L__BB77_1: // %atomicrmw.start
-; SM90-FTZ-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-FTZ-NEXT: shr.u32 %r8, %r14, %r1;
-; SM90-FTZ-NEXT: cvt.u16.u32 %rs2, %r8;
-; SM90-FTZ-NEXT: max.ftz.NaN.f16 %rs3, %rs2, %rs1;
-; SM90-FTZ-NEXT: cvt.u32.u16 %r9, %rs3;
-; SM90-FTZ-NEXT: shl.b32 %r10, %r9, %r1;
-; SM90-FTZ-NEXT: and.b32 %r11, %r14, %r2;
-; SM90-FTZ-NEXT: or.b32 %r12, %r11, %r10;
-; SM90-FTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
-; SM90-FTZ-NEXT: setp.ne.b32 %p1, %r3, %r14;
-; SM90-FTZ-NEXT: mov.b32 %r14, %r3;
-; SM90-FTZ-NEXT: @%p1 bra $L__BB77_1;
-; SM90-FTZ-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-FTZ-NEXT: shr.u32 %r13, %r3, %r1;
-; SM90-FTZ-NEXT: fence.acquire.cta;
-; SM90-FTZ-NEXT: st.param.b16 [func_retval0], %r13;
-; SM90-FTZ-NEXT: ret;
+; SM90-NOFTZ-ALLOW-LABEL: fmaximum_acq_rel_half_global_cta(
+; SM90-NOFTZ-ALLOW: {
+; SM90-NOFTZ-ALLOW-NEXT: .reg .pred %p<2>;
+; SM90-NOFTZ-ALLOW-NEXT: .reg .b16 %rs<4>;
+; SM90-NOFTZ-ALLOW-NEXT: .reg .b32 %r<15>;
+; SM90-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
+; SM90-NOFTZ-ALLOW-EMPTY:
+; SM90-NOFTZ-ALLOW-NEXT: // %bb.0:
+; SM90-NOFTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fmaximum_acq_rel_half_global_cta_param_1];
+; SM90-NOFTZ-ALLOW-NEXT: ld.param.b64 %rd2, [fmaximum_acq_rel_half_global_cta_param_0];
+; SM90-NOFTZ-ALLOW-NEXT: fence.release.cta;
+; SM90-NOFTZ-ALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-NOFTZ-ALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM90-NOFTZ-ALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM90-NOFTZ-ALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM90-NOFTZ-ALLOW-NEXT: mov.b32 %r6, 65535;
+; SM90-NOFTZ-ALLOW-NEXT: shl.b32 %r7, %r6, %r1;
+; SM90-NOFTZ-ALLOW-NEXT: not.b32 %r2, %r7;
+; SM90-NOFTZ-ALLOW-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM90-NOFTZ-ALLOW-NEXT: $L__BB77_1: // %atomicrmw.start
+; SM90-NOFTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NOFTZ-ALLOW-NEXT: shr.u32 %r8, %r14, %r1;
+; SM90-NOFTZ-ALLOW-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM90-NOFTZ-ALLOW-NEXT: max.NaN.f16 %rs3, %rs2, %rs1;
+; SM90-NOFTZ-ALLOW-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM90-NOFTZ-ALLOW-NEXT: shl.b32 %r10, %r9, %r1;
+; SM90-NOFTZ-ALLOW-NEXT: and.b32 %r11, %r14, %r2;
+; SM90-NOFTZ-ALLOW-NEXT: or.b32 %r12, %r11, %r10;
+; SM90-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM90-NOFTZ-ALLOW-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM90-NOFTZ-ALLOW-NEXT: mov.b32 %r14, %r3;
+; SM90-NOFTZ-ALLOW-NEXT: @%p1 bra $L__BB77_1;
+; SM90-NOFTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NOFTZ-ALLOW-NEXT: shr.u32 %r13, %r3, %r1;
+; SM90-NOFTZ-ALLOW-NEXT: fence.acquire.cta;
+; SM90-NOFTZ-ALLOW-NEXT: st.param.b16 [func_retval0], %r13;
+; SM90-NOFTZ-ALLOW-NEXT: ret;
+;
+; SM90-FTZ-DISALLOW-LABEL: fmaximum_acq_rel_half_global_cta(
+; SM90-FTZ-DISALLOW: {
+; SM90-FTZ-DISALLOW-NEXT: .reg .pred %p<2>;
+; SM90-FTZ-DISALLOW-NEXT: .reg .b16 %rs<4>;
+; SM90-FTZ-DISALLOW-NEXT: .reg .b32 %r<15>;
+; SM90-FTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
+; SM90-FTZ-DISALLOW-EMPTY:
+; SM90-FTZ-DISALLOW-NEXT: // %bb.0:
+; SM90-FTZ-DISALLOW-NEXT: ld.param.b16 %rs1, [fmaximum_acq_rel_half_global_cta_param_1];
+; SM90-FTZ-DISALLOW-NEXT: ld.param.b64 %rd2, [fmaximum_acq_rel_half_global_cta_param_0];
+; SM90-FTZ-DISALLOW-NEXT: fence.release.cta;
+; SM90-FTZ-DISALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-FTZ-DISALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM90-FTZ-DISALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM90-FTZ-DISALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM90-FTZ-DISALLOW-NEXT: mov.b32 %r6, 65535;
+; SM90-FTZ-DISALLOW-NEXT: shl.b32 %r7, %r6, %r1;
+; SM90-FTZ-DISALLOW-NEXT: not.b32 %r2, %r7;
+; SM90-FTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM90-FTZ-DISALLOW-NEXT: $L__BB77_1: // %atomicrmw.start
+; SM90-FTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-FTZ-DISALLOW-NEXT: shr.u32 %r8, %r14, %r1;
+; SM90-FTZ-DISALLOW-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM90-FTZ-DISALLOW-NEXT: max.ftz.NaN.f16 %rs3, %rs2, %rs1;
+; SM90-FTZ-DISALLOW-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM90-FTZ-DISALLOW-NEXT: shl.b32 %r10, %r9, %r1;
+; SM90-FTZ-DISALLOW-NEXT: and.b32 %r11, %r14, %r2;
+; SM90-FTZ-DISALLOW-NEXT: or.b32 %r12, %r11, %r10;
+; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM90-FTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM90-FTZ-DISALLOW-NEXT: mov.b32 %r14, %r3;
+; SM90-FTZ-DISALLOW-NEXT: @%p1 bra $L__BB77_1;
+; SM90-FTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-FTZ-DISALLOW-NEXT: shr.u32 %r13, %r3, %r1;
+; SM90-FTZ-DISALLOW-NEXT: fence.acquire.cta;
+; SM90-FTZ-DISALLOW-NEXT: st.param.b16 [func_retval0], %r13;
+; SM90-FTZ-DISALLOW-NEXT: ret;
+;
+; SM90-FTZ-ALLOW-LABEL: fmaximum_acq_rel_half_global_cta(
+; SM90-FTZ-ALLOW: {
+; SM90-FTZ-ALLOW-NEXT: .reg .pred %p<2>;
+; SM90-FTZ-ALLOW-NEXT: .reg .b16 %rs<4>;
+; SM90-FTZ-ALLOW-NEXT: .reg .b32 %r<15>;
+; SM90-FTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
+; SM90-FTZ-ALLOW-EMPTY:
+; SM90-FTZ-ALLOW-NEXT: // %bb.0:
+; SM90-FTZ-ALLOW-NEXT: ld.param.b16 %rs1, [fmaximum_acq_rel_half_global_cta_param_1];
+; SM90-FTZ-ALLOW-NEXT: ld.param.b64 %rd2, [fmaximum_acq_rel_half_global_cta_param_0];
+; SM90-FTZ-ALLOW-NEXT: fence.release.cta;
+; SM90-FTZ-ALLOW-NEXT: and.b64 %rd1, %rd2, -4;
+; SM90-FTZ-ALLOW-NEXT: cvt.u32.u64 %r4, %rd2;
+; SM90-FTZ-ALLOW-NEXT: and.b32 %r5, %r4, 3;
+; SM90-FTZ-ALLOW-NEXT: shl.b32 %r1, %r5, 3;
+; SM90-FTZ-ALLOW-NEXT: mov.b32 %r6, 65535;
+; SM90-FTZ-ALLOW-NEXT: shl.b32 %r7, %r6, %r1;
+; SM90-FTZ-ALLOW-NEXT: not.b32 %r2, %r7;
+; SM90-FTZ-ALLOW-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1];
+; SM90-FTZ-ALLOW-NEXT: $L__BB77_1: // %atomicrmw.start
+; SM90-FTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-FTZ-ALLOW-NEXT: shr.u32 %r8, %r14, %r1;
+; SM90-FTZ-ALLOW-NEXT: cvt.u16.u32 %rs2, %r8;
+; SM90-FTZ-ALLOW-NEXT: max.ftz.NaN.f16 %rs3, %rs2, %rs1;
+; SM90-FTZ-ALLOW-NEXT: cvt.u32.u16 %r9, %rs3;
+; SM90-FTZ-ALLOW-NEXT: shl.b32 %r10, %r9, %r1;
+; SM90-FTZ-ALLOW-NEXT: and.b32 %r11, %r14, %r2;
+; SM90-FTZ-ALLOW-NEXT: or.b32 %r12, %r11, %r10;
+; SM90-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
+; SM90-FTZ-ALLOW-NEXT: setp.ne.b32 %p1, %r3, %r14;
+; SM90-FTZ-ALLOW-NEXT: mov.b32 %r14, %r3;
+; SM90-FTZ-ALLOW-NEXT: @%p1 bra $L__BB77_1;
+; SM90-FTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-FTZ-ALLOW-NEXT: shr.u32 %r13, %r3, %r1;
+; SM90-FTZ-ALLOW-NEXT: fence.acquire.cta;
+; SM90-FTZ-ALLOW-NEXT: st.param.b16 [func_retval0], %r13;
+; SM90-FTZ-ALLOW-NEXT: ret;
%retval = atomicrmw fmaximum ptr addrspace(1) %addr, half %val syncscope("block") acq_rel
ret half %retval
}
@@ -3464,40 +4130,64 @@ define i64 @nand_acq_rel_i64_global_sys(ptr addrspace(1) %addr, i64 %val) {
}
define float @fadd_acq_rel_float_global_sys(ptr addrspace(1) %addr, float %val) {
-; SM90-NOFTZ-LABEL: fadd_acq_rel_float_global_sys(
-; SM90-NOFTZ: {
-; SM90-NOFTZ-NEXT: .reg .pred %p<2>;
-; SM90-NOFTZ-NEXT: .reg .b32 %r<5>;
-; SM90-NOFTZ-NEXT: .reg .b64 %rd<2>;
-; SM90-NOFTZ-EMPTY:
-; SM90-NOFTZ-NEXT: // %bb.0:
-; SM90-NOFTZ-NEXT: ld.param.b32 %r2, [fadd_acq_rel_float_global_sys_param_1];
-; SM90-NOFTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_sys_param_0];
-; SM90-NOFTZ-NEXT: fence.release.sys;
-; SM90-NOFTZ-NEXT: ld.relaxed.sys.global.b32 %r4, [%rd1];
-; SM90-NOFTZ-NEXT: $L__BB106_1: // %atomicrmw.start
-; SM90-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NOFTZ-NEXT: add.rn.f32 %r3, %r4, %r2;
-; SM90-NOFTZ-NEXT: atom.relaxed.sys.global.cas.b32 %r1, [%rd1], %r4, %r3;
-; SM90-NOFTZ-NEXT: setp.ne.b32 %p1, %r1, %r4;
-; SM90-NOFTZ-NEXT: mov.b32 %r4, %r1;
-; SM90-NOFTZ-NEXT: @%p1 bra $L__BB106_1;
-; SM90-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NOFTZ-NEXT: fence.acquire.sys;
-; SM90-NOFTZ-NEXT: st.param.b32 [func_retval0], %r1;
-; SM90-NOFTZ-NEXT: ret;
+; SM90-NOFTZ-DISALLOW-LABEL: fadd_acq_rel_float_global_sys(
+; SM90-NOFTZ-DISALLOW: {
+; SM90-NOFTZ-DISALLOW-NEXT: .reg .pred %p<2>;
+; SM90-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<5>;
+; SM90-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
+; SM90-NOFTZ-DISALLOW-EMPTY:
+; SM90-NOFTZ-DISALLOW-NEXT: // %bb.0:
+; SM90-NOFTZ-DISALLOW-NEXT: ld.param.b32 %r2, [fadd_acq_rel_float_global_sys_param_1];
+; SM90-NOFTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_sys_param_0];
+; SM90-NOFTZ-DISALLOW-NEXT: fence.release.sys;
+; SM90-NOFTZ-DISALLOW-NEXT: ld.relaxed.sys.global.b32 %r4, [%rd1];
+; SM90-NOFTZ-DISALLOW-NEXT: $L__BB106_1: // %atomicrmw.start
+; SM90-NOFTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NOFTZ-DISALLOW-NEXT: add.rn.f32 %r3, %r4, %r2;
+; SM90-NOFTZ-DISALLOW-NEXT: atom.relaxed.sys.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM90-NOFTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM90-NOFTZ-DISALLOW-NEXT: mov.b32 %r4, %r1;
+; SM90-NOFTZ-DISALLOW-NEXT: @%p1 bra $L__BB106_1;
+; SM90-NOFTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NOFTZ-DISALLOW-NEXT: fence.acquire.sys;
+; SM90-NOFTZ-DISALLOW-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NOFTZ-DISALLOW-NEXT: ret;
+;
+; SM90-NOFTZ-ALLOW-LABEL: fadd_acq_rel_float_global_sys(
+; SM90-NOFTZ-ALLOW: {
+; SM90-NOFTZ-ALLOW-NEXT: .reg .b32 %r<3>;
+; SM90-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
+; SM90-NOFTZ-ALLOW-EMPTY:
+; SM90-NOFTZ-ALLOW-NEXT: // %bb.0:
+; SM90-NOFTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_sys_param_0];
+; SM90-NOFTZ-ALLOW-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_global_sys_param_1];
+; SM90-NOFTZ-ALLOW-NEXT: atom.acq_rel.sys.global.add.f32 %r2, [%rd1], %r1;
+; SM90-NOFTZ-ALLOW-NEXT: st.param.b32 [func_retval0], %r2;
+; SM90-NOFTZ-ALLOW-NEXT: ret;
+;
+; SM90-FTZ-DISALLOW-LABEL: fadd_acq_rel_float_global_sys(
+; SM90-FTZ-DISALLOW: {
+; SM90-FTZ-DISALLOW-NEXT: .reg .b32 %r<3>;
+; SM90-FTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
+; SM90-FTZ-DISALLOW-EMPTY:
+; SM90-FTZ-DISALLOW-NEXT: // %bb.0:
+; SM90-FTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_sys_param_0];
+; SM90-FTZ-DISALLOW-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_global_sys_param_1];
+; SM90-FTZ-DISALLOW-NEXT: atom.acq_rel.sys.global.add.f32 %r2, [%rd1], %r1;
+; SM90-FTZ-DISALLOW-NEXT: st.param.b32 [func_retval0], %r2;
+; SM90-FTZ-DISALLOW-NEXT: ret;
;
-; SM90-FTZ-LABEL: fadd_acq_rel_float_global_sys(
-; SM90-FTZ: {
-; SM90-FTZ-NEXT: .reg .b32 %r<3>;
-; SM90-FTZ-NEXT: .reg .b64 %rd<2>;
-; SM90-FTZ-EMPTY:
-; SM90-FTZ-NEXT: // %bb.0:
-; SM90-FTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_sys_param_0];
-; SM90-FTZ-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_global_sys_param_1];
-; SM90-FTZ-NEXT: atom.acq_rel.sys.global.add.f32 %r2, [%rd1], %r1;
-; SM90-FTZ-NEXT: st.param.b32 [func_retval0], %r2;
-; SM90-FTZ-NEXT: ret;
+; SM90-FTZ-ALLOW-LABEL: fadd_acq_rel_float_global_sys(
+; SM90-FTZ-ALLOW: {
+; SM90-FTZ-ALLOW-NEXT: .reg .b32 %r<3>;
+; SM90-FTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
+; SM90-FTZ-ALLOW-EMPTY:
+; SM90-FTZ-ALLOW-NEXT: // %bb.0:
+; SM90-FTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_sys_param_0];
+; SM90-FTZ-ALLOW-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_global_sys_param_1];
+; SM90-FTZ-ALLOW-NEXT: atom.acq_rel.sys.global.add.f32 %r2, [%rd1], %r1;
+; SM90-FTZ-ALLOW-NEXT: st.param.b32 [func_retval0], %r2;
+; SM90-FTZ-ALLOW-NEXT: ret;
%retval = atomicrmw fadd ptr addrspace(1) %addr, float %val syncscope("") acq_rel
ret float %retval
}
@@ -3636,40 +4326,64 @@ define i64 @nand_acq_rel_i64_global_cluster(ptr addrspace(1) %addr, i64 %val) {
}
define float @fadd_acq_rel_float_global_cluster(ptr addrspace(1) %addr, float %val) {
-; SM90-NOFTZ-LABEL: fadd_acq_rel_float_global_cluster(
-; SM90-NOFTZ: {
-; SM90-NOFTZ-NEXT: .reg .pred %p<2>;
-; SM90-NOFTZ-NEXT: .reg .b32 %r<5>;
-; SM90-NOFTZ-NEXT: .reg .b64 %rd<2>;
-; SM90-NOFTZ-EMPTY:
-; SM90-NOFTZ-NEXT: // %bb.0:
-; SM90-NOFTZ-NEXT: ld.param.b32 %r2, [fadd_acq_rel_float_global_cluster_param_1];
-; SM90-NOFTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_cluster_param_0];
-; SM90-NOFTZ-NEXT: fence.release.cluster;
-; SM90-NOFTZ-NEXT: ld.relaxed.cluster.global.b32 %r4, [%rd1];
-; SM90-NOFTZ-NEXT: $L__BB114_1: // %atomicrmw.start
-; SM90-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NOFTZ-NEXT: add.rn.f32 %r3, %r4, %r2;
-; SM90-NOFTZ-NEXT: atom.relaxed.cluster.global.cas.b32 %r1, [%rd1], %r4, %r3;
-; SM90-NOFTZ-NEXT: setp.ne.b32 %p1, %r1, %r4;
-; SM90-NOFTZ-NEXT: mov.b32 %r4, %r1;
-; SM90-NOFTZ-NEXT: @%p1 bra $L__BB114_1;
-; SM90-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NOFTZ-NEXT: fence.acquire.cluster;
-; SM90-NOFTZ-NEXT: st.param.b32 [func_retval0], %r1;
-; SM90-NOFTZ-NEXT: ret;
+; SM90-NOFTZ-DISALLOW-LABEL: fadd_acq_rel_float_global_cluster(
+; SM90-NOFTZ-DISALLOW: {
+; SM90-NOFTZ-DISALLOW-NEXT: .reg .pred %p<2>;
+; SM90-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<5>;
+; SM90-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
+; SM90-NOFTZ-DISALLOW-EMPTY:
+; SM90-NOFTZ-DISALLOW-NEXT: // %bb.0:
+; SM90-NOFTZ-DISALLOW-NEXT: ld.param.b32 %r2, [fadd_acq_rel_float_global_cluster_param_1];
+; SM90-NOFTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_cluster_param_0];
+; SM90-NOFTZ-DISALLOW-NEXT: fence.release.cluster;
+; SM90-NOFTZ-DISALLOW-NEXT: ld.relaxed.cluster.global.b32 %r4, [%rd1];
+; SM90-NOFTZ-DISALLOW-NEXT: $L__BB114_1: // %atomicrmw.start
+; SM90-NOFTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NOFTZ-DISALLOW-NEXT: add.rn.f32 %r3, %r4, %r2;
+; SM90-NOFTZ-DISALLOW-NEXT: atom.relaxed.cluster.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM90-NOFTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM90-NOFTZ-DISALLOW-NEXT: mov.b32 %r4, %r1;
+; SM90-NOFTZ-DISALLOW-NEXT: @%p1 bra $L__BB114_1;
+; SM90-NOFTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NOFTZ-DISALLOW-NEXT: fence.acquire.cluster;
+; SM90-NOFTZ-DISALLOW-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NOFTZ-DISALLOW-NEXT: ret;
;
-; SM90-FTZ-LABEL: fadd_acq_rel_float_global_cluster(
-; SM90-FTZ: {
-; SM90-FTZ-NEXT: .reg .b32 %r<3>;
-; SM90-FTZ-NEXT: .reg .b64 %rd<2>;
-; SM90-FTZ-EMPTY:
-; SM90-FTZ-NEXT: // %bb.0:
-; SM90-FTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_cluster_param_0];
-; SM90-FTZ-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_global_cluster_param_1];
-; SM90-FTZ-NEXT: atom.acq_rel.cluster.global.add.f32 %r2, [%rd1], %r1;
-; SM90-FTZ-NEXT: st.param.b32 [func_retval0], %r2;
-; SM90-FTZ-NEXT: ret;
+; SM90-NOFTZ-ALLOW-LABEL: fadd_acq_rel_float_global_cluster(
+; SM90-NOFTZ-ALLOW: {
+; SM90-NOFTZ-ALLOW-NEXT: .reg .b32 %r<3>;
+; SM90-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
+; SM90-NOFTZ-ALLOW-EMPTY:
+; SM90-NOFTZ-ALLOW-NEXT: // %bb.0:
+; SM90-NOFTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_cluster_param_0];
+; SM90-NOFTZ-ALLOW-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_global_cluster_param_1];
+; SM90-NOFTZ-ALLOW-NEXT: atom.acq_rel.cluster.global.add.f32 %r2, [%rd1], %r1;
+; SM90-NOFTZ-ALLOW-NEXT: st.param.b32 [func_retval0], %r2;
+; SM90-NOFTZ-ALLOW-NEXT: ret;
+;
+; SM90-FTZ-DISALLOW-LABEL: fadd_acq_rel_float_global_cluster(
+; SM90-FTZ-DISALLOW: {
+; SM90-FTZ-DISALLOW-NEXT: .reg .b32 %r<3>;
+; SM90-FTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
+; SM90-FTZ-DISALLOW-EMPTY:
+; SM90-FTZ-DISALLOW-NEXT: // %bb.0:
+; SM90-FTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_cluster_param_0];
+; SM90-FTZ-DISALLOW-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_global_cluster_param_1];
+; SM90-FTZ-DISALLOW-NEXT: atom.acq_rel.cluster.global.add.f32 %r2, [%rd1], %r1;
+; SM90-FTZ-DISALLOW-NEXT: st.param.b32 [func_retval0], %r2;
+; SM90-FTZ-DISALLOW-NEXT: ret;
+;
+; SM90-FTZ-ALLOW-LABEL: fadd_acq_rel_float_global_cluster(
+; SM90-FTZ-ALLOW: {
+; SM90-FTZ-ALLOW-NEXT: .reg .b32 %r<3>;
+; SM90-FTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
+; SM90-FTZ-ALLOW-EMPTY:
+; SM90-FTZ-ALLOW-NEXT: // %bb.0:
+; SM90-FTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_cluster_param_0];
+; SM90-FTZ-ALLOW-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_global_cluster_param_1];
+; SM90-FTZ-ALLOW-NEXT: atom.acq_rel.cluster.global.add.f32 %r2, [%rd1], %r1;
+; SM90-FTZ-ALLOW-NEXT: st.param.b32 [func_retval0], %r2;
+; SM90-FTZ-ALLOW-NEXT: ret;
%retval = atomicrmw fadd ptr addrspace(1) %addr, float %val syncscope("cluster") acq_rel
ret float %retval
}
@@ -3808,40 +4522,64 @@ define i64 @nand_acq_rel_i64_global_gpu(ptr addrspace(1) %addr, i64 %val) {
}
define float @fadd_acq_rel_float_global_gpu(ptr addrspace(1) %addr, float %val) {
-; SM90-NOFTZ-LABEL: fadd_acq_rel_float_global_gpu(
-; SM90-NOFTZ: {
-; SM90-NOFTZ-NEXT: .reg .pred %p<2>;
-; SM90-NOFTZ-NEXT: .reg .b32 %r<5>;
-; SM90-NOFTZ-NEXT: .reg .b64 %rd<2>;
-; SM90-NOFTZ-EMPTY:
-; SM90-NOFTZ-NEXT: // %bb.0:
-; SM90-NOFTZ-NEXT: ld.param.b32 %r2, [fadd_acq_rel_float_global_gpu_param_1];
-; SM90-NOFTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_gpu_param_0];
-; SM90-NOFTZ-NEXT: fence.release.gpu;
-; SM90-NOFTZ-NEXT: ld.relaxed.gpu.global.b32 %r4, [%rd1];
-; SM90-NOFTZ-NEXT: $L__BB122_1: // %atomicrmw.start
-; SM90-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NOFTZ-NEXT: add.rn.f32 %r3, %r4, %r2;
-; SM90-NOFTZ-NEXT: atom.relaxed.gpu.global.cas.b32 %r1, [%rd1], %r4, %r3;
-; SM90-NOFTZ-NEXT: setp.ne.b32 %p1, %r1, %r4;
-; SM90-NOFTZ-NEXT: mov.b32 %r4, %r1;
-; SM90-NOFTZ-NEXT: @%p1 bra $L__BB122_1;
-; SM90-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NOFTZ-NEXT: fence.acquire.gpu;
-; SM90-NOFTZ-NEXT: st.param.b32 [func_retval0], %r1;
-; SM90-NOFTZ-NEXT: ret;
+; SM90-NOFTZ-DISALLOW-LABEL: fadd_acq_rel_float_global_gpu(
+; SM90-NOFTZ-DISALLOW: {
+; SM90-NOFTZ-DISALLOW-NEXT: .reg .pred %p<2>;
+; SM90-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<5>;
+; SM90-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
+; SM90-NOFTZ-DISALLOW-EMPTY:
+; SM90-NOFTZ-DISALLOW-NEXT: // %bb.0:
+; SM90-NOFTZ-DISALLOW-NEXT: ld.param.b32 %r2, [fadd_acq_rel_float_global_gpu_param_1];
+; SM90-NOFTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_gpu_param_0];
+; SM90-NOFTZ-DISALLOW-NEXT: fence.release.gpu;
+; SM90-NOFTZ-DISALLOW-NEXT: ld.relaxed.gpu.global.b32 %r4, [%rd1];
+; SM90-NOFTZ-DISALLOW-NEXT: $L__BB122_1: // %atomicrmw.start
+; SM90-NOFTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NOFTZ-DISALLOW-NEXT: add.rn.f32 %r3, %r4, %r2;
+; SM90-NOFTZ-DISALLOW-NEXT: atom.relaxed.gpu.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM90-NOFTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM90-NOFTZ-DISALLOW-NEXT: mov.b32 %r4, %r1;
+; SM90-NOFTZ-DISALLOW-NEXT: @%p1 bra $L__BB122_1;
+; SM90-NOFTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NOFTZ-DISALLOW-NEXT: fence.acquire.gpu;
+; SM90-NOFTZ-DISALLOW-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NOFTZ-DISALLOW-NEXT: ret;
+;
+; SM90-NOFTZ-ALLOW-LABEL: fadd_acq_rel_float_global_gpu(
+; SM90-NOFTZ-ALLOW: {
+; SM90-NOFTZ-ALLOW-NEXT: .reg .b32 %r<3>;
+; SM90-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
+; SM90-NOFTZ-ALLOW-EMPTY:
+; SM90-NOFTZ-ALLOW-NEXT: // %bb.0:
+; SM90-NOFTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_gpu_param_0];
+; SM90-NOFTZ-ALLOW-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_global_gpu_param_1];
+; SM90-NOFTZ-ALLOW-NEXT: atom.acq_rel.gpu.global.add.f32 %r2, [%rd1], %r1;
+; SM90-NOFTZ-ALLOW-NEXT: st.param.b32 [func_retval0], %r2;
+; SM90-NOFTZ-ALLOW-NEXT: ret;
+;
+; SM90-FTZ-DISALLOW-LABEL: fadd_acq_rel_float_global_gpu(
+; SM90-FTZ-DISALLOW: {
+; SM90-FTZ-DISALLOW-NEXT: .reg .b32 %r<3>;
+; SM90-FTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
+; SM90-FTZ-DISALLOW-EMPTY:
+; SM90-FTZ-DISALLOW-NEXT: // %bb.0:
+; SM90-FTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_gpu_param_0];
+; SM90-FTZ-DISALLOW-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_global_gpu_param_1];
+; SM90-FTZ-DISALLOW-NEXT: atom.acq_rel.gpu.global.add.f32 %r2, [%rd1], %r1;
+; SM90-FTZ-DISALLOW-NEXT: st.param.b32 [func_retval0], %r2;
+; SM90-FTZ-DISALLOW-NEXT: ret;
;
-; SM90-FTZ-LABEL: fadd_acq_rel_float_global_gpu(
-; SM90-FTZ: {
-; SM90-FTZ-NEXT: .reg .b32 %r<3>;
-; SM90-FTZ-NEXT: .reg .b64 %rd<2>;
-; SM90-FTZ-EMPTY:
-; SM90-FTZ-NEXT: // %bb.0:
-; SM90-FTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_gpu_param_0];
-; SM90-FTZ-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_global_gpu_param_1];
-; SM90-FTZ-NEXT: atom.acq_rel.gpu.global.add.f32 %r2, [%rd1], %r1;
-; SM90-FTZ-NEXT: st.param.b32 [func_retval0], %r2;
-; SM90-FTZ-NEXT: ret;
+; SM90-FTZ-ALLOW-LABEL: fadd_acq_rel_float_global_gpu(
+; SM90-FTZ-ALLOW: {
+; SM90-FTZ-ALLOW-NEXT: .reg .b32 %r<3>;
+; SM90-FTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
+; SM90-FTZ-ALLOW-EMPTY:
+; SM90-FTZ-ALLOW-NEXT: // %bb.0:
+; SM90-FTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_gpu_param_0];
+; SM90-FTZ-ALLOW-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_global_gpu_param_1];
+; SM90-FTZ-ALLOW-NEXT: atom.acq_rel.gpu.global.add.f32 %r2, [%rd1], %r1;
+; SM90-FTZ-ALLOW-NEXT: st.param.b32 [func_retval0], %r2;
+; SM90-FTZ-ALLOW-NEXT: ret;
%retval = atomicrmw fadd ptr addrspace(1) %addr, float %val syncscope("device") acq_rel
ret float %retval
}
@@ -3980,51 +4718,75 @@ define i64 @nand_acq_rel_i64_generic_cta(ptr %addr, i64 %val) {
}
define float @fadd_acq_rel_float_generic_cta(ptr %addr, float %val) {
-; SM90-NOFTZ-LABEL: fadd_acq_rel_float_generic_cta(
-; SM90-NOFTZ: {
-; SM90-NOFTZ-NEXT: .reg .pred %p<2>;
-; SM90-NOFTZ-NEXT: .reg .b32 %r<5>;
-; SM90-NOFTZ-NEXT: .reg .b64 %rd<2>;
-; SM90-NOFTZ-EMPTY:
-; SM90-NOFTZ-NEXT: // %bb.0:
-; SM90-NOFTZ-NEXT: ld.param.b32 %r2, [fadd_acq_rel_float_generic_cta_param_1];
-; SM90-NOFTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_generic_cta_param_0];
-; SM90-NOFTZ-NEXT: fence.release.cta;
-; SM90-NOFTZ-NEXT: ld.relaxed.cta.b32 %r4, [%rd1];
-; SM90-NOFTZ-NEXT: $L__BB130_1: // %atomicrmw.start
-; SM90-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-NOFTZ-NEXT: add.rn.f32 %r3, %r4, %r2;
-; SM90-NOFTZ-NEXT: atom.relaxed.cta.cas.b32 %r1, [%rd1], %r4, %r3;
-; SM90-NOFTZ-NEXT: setp.ne.b32 %p1, %r1, %r4;
-; SM90-NOFTZ-NEXT: mov.b32 %r4, %r1;
-; SM90-NOFTZ-NEXT: @%p1 bra $L__BB130_1;
-; SM90-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-NOFTZ-NEXT: fence.acquire.cta;
-; SM90-NOFTZ-NEXT: st.param.b32 [func_retval0], %r1;
-; SM90-NOFTZ-NEXT: ret;
+; SM90-NOFTZ-DISALLOW-LABEL: fadd_acq_rel_float_generic_cta(
+; SM90-NOFTZ-DISALLOW: {
+; SM90-NOFTZ-DISALLOW-NEXT: .reg .pred %p<2>;
+; SM90-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<5>;
+; SM90-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
+; SM90-NOFTZ-DISALLOW-EMPTY:
+; SM90-NOFTZ-DISALLOW-NEXT: // %bb.0:
+; SM90-NOFTZ-DISALLOW-NEXT: ld.param.b32 %r2, [fadd_acq_rel_float_generic_cta_param_1];
+; SM90-NOFTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_generic_cta_param_0];
+; SM90-NOFTZ-DISALLOW-NEXT: fence.release.cta;
+; SM90-NOFTZ-DISALLOW-NEXT: ld.relaxed.cta.b32 %r4, [%rd1];
+; SM90-NOFTZ-DISALLOW-NEXT: $L__BB130_1: // %atomicrmw.start
+; SM90-NOFTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-NOFTZ-DISALLOW-NEXT: add.rn.f32 %r3, %r4, %r2;
+; SM90-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM90-NOFTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM90-NOFTZ-DISALLOW-NEXT: mov.b32 %r4, %r1;
+; SM90-NOFTZ-DISALLOW-NEXT: @%p1 bra $L__BB130_1;
+; SM90-NOFTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-NOFTZ-DISALLOW-NEXT: fence.acquire.cta;
+; SM90-NOFTZ-DISALLOW-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-NOFTZ-DISALLOW-NEXT: ret;
;
-; SM90-FTZ-LABEL: fadd_acq_rel_float_generic_cta(
-; SM90-FTZ: {
-; SM90-FTZ-NEXT: .reg .pred %p<2>;
-; SM90-FTZ-NEXT: .reg .b32 %r<5>;
-; SM90-FTZ-NEXT: .reg .b64 %rd<2>;
-; SM90-FTZ-EMPTY:
-; SM90-FTZ-NEXT: // %bb.0:
-; SM90-FTZ-NEXT: ld.param.b32 %r2, [fadd_acq_rel_float_generic_cta_param_1];
-; SM90-FTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_generic_cta_param_0];
-; SM90-FTZ-NEXT: fence.release.cta;
-; SM90-FTZ-NEXT: ld.relaxed.cta.b32 %r4, [%rd1];
-; SM90-FTZ-NEXT: $L__BB130_1: // %atomicrmw.start
-; SM90-FTZ-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-FTZ-NEXT: add.rn.ftz.f32 %r3, %r4, %r2;
-; SM90-FTZ-NEXT: atom.relaxed.cta.cas.b32 %r1, [%rd1], %r4, %r3;
-; SM90-FTZ-NEXT: setp.ne.b32 %p1, %r1, %r4;
-; SM90-FTZ-NEXT: mov.b32 %r4, %r1;
-; SM90-FTZ-NEXT: @%p1 bra $L__BB130_1;
-; SM90-FTZ-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-FTZ-NEXT: fence.acquire.cta;
-; SM90-FTZ-NEXT: st.param.b32 [func_retval0], %r1;
-; SM90-FTZ-NEXT: ret;
+; SM90-NOFTZ-ALLOW-LABEL: fadd_acq_rel_float_generic_cta(
+; SM90-NOFTZ-ALLOW: {
+; SM90-NOFTZ-ALLOW-NEXT: .reg .b32 %r<3>;
+; SM90-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
+; SM90-NOFTZ-ALLOW-EMPTY:
+; SM90-NOFTZ-ALLOW-NEXT: // %bb.0:
+; SM90-NOFTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_generic_cta_param_0];
+; SM90-NOFTZ-ALLOW-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_generic_cta_param_1];
+; SM90-NOFTZ-ALLOW-NEXT: atom.acq_rel.cta.add.f32 %r2, [%rd1], %r1;
+; SM90-NOFTZ-ALLOW-NEXT: st.param.b32 [func_retval0], %r2;
+; SM90-NOFTZ-ALLOW-NEXT: ret;
+;
+; SM90-FTZ-DISALLOW-LABEL: fadd_acq_rel_float_generic_cta(
+; SM90-FTZ-DISALLOW: {
+; SM90-FTZ-DISALLOW-NEXT: .reg .pred %p<2>;
+; SM90-FTZ-DISALLOW-NEXT: .reg .b32 %r<5>;
+; SM90-FTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
+; SM90-FTZ-DISALLOW-EMPTY:
+; SM90-FTZ-DISALLOW-NEXT: // %bb.0:
+; SM90-FTZ-DISALLOW-NEXT: ld.param.b32 %r2, [fadd_acq_rel_float_generic_cta_param_1];
+; SM90-FTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_generic_cta_param_0];
+; SM90-FTZ-DISALLOW-NEXT: fence.release.cta;
+; SM90-FTZ-DISALLOW-NEXT: ld.relaxed.cta.b32 %r4, [%rd1];
+; SM90-FTZ-DISALLOW-NEXT: $L__BB130_1: // %atomicrmw.start
+; SM90-FTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-FTZ-DISALLOW-NEXT: add.rn.ftz.f32 %r3, %r4, %r2;
+; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM90-FTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM90-FTZ-DISALLOW-NEXT: mov.b32 %r4, %r1;
+; SM90-FTZ-DISALLOW-NEXT: @%p1 bra $L__BB130_1;
+; SM90-FTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-FTZ-DISALLOW-NEXT: fence.acquire.cta;
+; SM90-FTZ-DISALLOW-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-FTZ-DISALLOW-NEXT: ret;
+;
+; SM90-FTZ-ALLOW-LABEL: fadd_acq_rel_float_generic_cta(
+; SM90-FTZ-ALLOW: {
+; SM90-FTZ-ALLOW-NEXT: .reg .b32 %r<3>;
+; SM90-FTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
+; SM90-FTZ-ALLOW-EMPTY:
+; SM90-FTZ-ALLOW-NEXT: // %bb.0:
+; SM90-FTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_generic_cta_param_0];
+; SM90-FTZ-ALLOW-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_generic_cta_param_1];
+; SM90-FTZ-ALLOW-NEXT: atom.acq_rel.cta.add.f32 %r2, [%rd1], %r1;
+; SM90-FTZ-ALLOW-NEXT: st.param.b32 [func_retval0], %r2;
+; SM90-FTZ-ALLOW-NEXT: ret;
%retval = atomicrmw fadd ptr %addr, float %val syncscope("block") acq_rel
ret float %retval
}
@@ -4163,40 +4925,64 @@ define i64 @nand_acq_rel_i64_shared_cta(ptr addrspace(3) %addr, i64 %val) {
}
define float @fadd_acq_rel_float_shared_cta(ptr addrspace(3) %addr, float %val) {
-; SM90-NOFTZ-LABEL: fadd_acq_rel_float_shared_cta(
-; SM90-NOFTZ: {
-; SM90-NOFTZ-NEXT: .reg .b32 %r<3>;
-; SM90-NOFTZ-NEXT: .reg .b64 %rd<2>;
-; SM90-NOFTZ-EMPTY:
-; SM90-NOFTZ-NEXT: // %bb.0:
-; SM90-NOFTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_shared_cta_param_0];
-; SM90-NOFTZ-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_shared_cta_param_1];
-; SM90-NOFTZ-NEXT: atom.acq_rel.cta.shared.add.f32 %r2, [%rd1], %r1;
-; SM90-NOFTZ-NEXT: st.param.b32 [func_retval0], %r2;
-; SM90-NOFTZ-NEXT: ret;
+; SM90-NOFTZ-DISALLOW-LABEL: fadd_acq_rel_float_shared_cta(
+; SM90-NOFTZ-DISALLOW: {
+; SM90-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<3>;
+; SM90-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
+; SM90-NOFTZ-DISALLOW-EMPTY:
+; SM90-NOFTZ-DISALLOW-NEXT: // %bb.0:
+; SM90-NOFTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_shared_cta_param_0];
+; SM90-NOFTZ-DISALLOW-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_shared_cta_param_1];
+; SM90-NOFTZ-DISALLOW-NEXT: atom.acq_rel.cta.shared.add.f32 %r2, [%rd1], %r1;
+; SM90-NOFTZ-DISALLOW-NEXT: st.param.b32 [func_retval0], %r2;
+; SM90-NOFTZ-DISALLOW-NEXT: ret;
+;
+; SM90-NOFTZ-ALLOW-LABEL: fadd_acq_rel_float_shared_cta(
+; SM90-NOFTZ-ALLOW: {
+; SM90-NOFTZ-ALLOW-NEXT: .reg .b32 %r<3>;
+; SM90-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
+; SM90-NOFTZ-ALLOW-EMPTY:
+; SM90-NOFTZ-ALLOW-NEXT: // %bb.0:
+; SM90-NOFTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_shared_cta_param_0];
+; SM90-NOFTZ-ALLOW-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_shared_cta_param_1];
+; SM90-NOFTZ-ALLOW-NEXT: atom.acq_rel.cta.shared.add.f32 %r2, [%rd1], %r1;
+; SM90-NOFTZ-ALLOW-NEXT: st.param.b32 [func_retval0], %r2;
+; SM90-NOFTZ-ALLOW-NEXT: ret;
+;
+; SM90-FTZ-DISALLOW-LABEL: fadd_acq_rel_float_shared_cta(
+; SM90-FTZ-DISALLOW: {
+; SM90-FTZ-DISALLOW-NEXT: .reg .pred %p<2>;
+; SM90-FTZ-DISALLOW-NEXT: .reg .b32 %r<5>;
+; SM90-FTZ-DISALLOW-NEXT: .reg .b64 %rd<2>;
+; SM90-FTZ-DISALLOW-EMPTY:
+; SM90-FTZ-DISALLOW-NEXT: // %bb.0:
+; SM90-FTZ-DISALLOW-NEXT: ld.param.b32 %r2, [fadd_acq_rel_float_shared_cta_param_1];
+; SM90-FTZ-DISALLOW-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_shared_cta_param_0];
+; SM90-FTZ-DISALLOW-NEXT: fence.release.cta;
+; SM90-FTZ-DISALLOW-NEXT: ld.relaxed.cta.shared.b32 %r4, [%rd1];
+; SM90-FTZ-DISALLOW-NEXT: $L__BB138_1: // %atomicrmw.start
+; SM90-FTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; SM90-FTZ-DISALLOW-NEXT: add.rn.ftz.f32 %r3, %r4, %r2;
+; SM90-FTZ-DISALLOW-NEXT: atom.relaxed.cta.shared.cas.b32 %r1, [%rd1], %r4, %r3;
+; SM90-FTZ-DISALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; SM90-FTZ-DISALLOW-NEXT: mov.b32 %r4, %r1;
+; SM90-FTZ-DISALLOW-NEXT: @%p1 bra $L__BB138_1;
+; SM90-FTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; SM90-FTZ-DISALLOW-NEXT: fence.acquire.cta;
+; SM90-FTZ-DISALLOW-NEXT: st.param.b32 [func_retval0], %r1;
+; SM90-FTZ-DISALLOW-NEXT: ret;
;
-; SM90-FTZ-LABEL: fadd_acq_rel_float_shared_cta(
-; SM90-FTZ: {
-; SM90-FTZ-NEXT: .reg .pred %p<2>;
-; SM90-FTZ-NEXT: .reg .b32 %r<5>;
-; SM90-FTZ-NEXT: .reg .b64 %rd<2>;
-; SM90-FTZ-EMPTY:
-; SM90-FTZ-NEXT: // %bb.0:
-; SM90-FTZ-NEXT: ld.param.b32 %r2, [fadd_acq_rel_float_shared_cta_param_1];
-; SM90-FTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_shared_cta_param_0];
-; SM90-FTZ-NEXT: fence.release.cta;
-; SM90-FTZ-NEXT: ld.relaxed.cta.shared.b32 %r4, [%rd1];
-; SM90-FTZ-NEXT: $L__BB138_1: // %atomicrmw.start
-; SM90-FTZ-NEXT: // =>This Inner Loop Header: Depth=1
-; SM90-FTZ-NEXT: add.rn.ftz.f32 %r3, %r4, %r2;
-; SM90-FTZ-NEXT: atom.relaxed.cta.shared.cas.b32 %r1, [%rd1], %r4, %r3;
-; SM90-FTZ-NEXT: setp.ne.b32 %p1, %r1, %r4;
-; SM90-FTZ-NEXT: mov.b32 %r4, %r1;
-; SM90-FTZ-NEXT: @%p1 bra $L__BB138_1;
-; SM90-FTZ-NEXT: // %bb.2: // %atomicrmw.end
-; SM90-FTZ-NEXT: fence.acquire.cta;
-; SM90-FTZ-NEXT: st.param.b32 [func_retval0], %r1;
-; SM90-FTZ-NEXT: ret;
+; SM90-FTZ-ALLOW-LABEL: fadd_acq_rel_float_shared_cta(
+; SM90-FTZ-ALLOW: {
+; SM90-FTZ-ALLOW-NEXT: .reg .b32 %r<3>;
+; SM90-FTZ-ALLOW-NEXT: .reg .b64 %rd<2>;
+; SM90-FTZ-ALLOW-EMPTY:
+; SM90-FTZ-ALLOW-NEXT: // %bb.0:
+; SM90-FTZ-ALLOW-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_shared_cta_param_0];
+; SM90-FTZ-ALLOW-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_shared_cta_param_1];
+; SM90-FTZ-ALLOW-NEXT: atom.acq_rel.cta.shared.add.f32 %r2, [%rd1], %r1;
+; SM90-FTZ-ALLOW-NEXT: st.param.b32 [func_retval0], %r2;
+; SM90-FTZ-ALLOW-NEXT: ret;
%retval = atomicrmw fadd ptr addrspace(3) %addr, float %val syncscope("block") acq_rel
ret float %retval
}
diff --git a/llvm/test/CodeGen/NVPTX/atomicrmw.py b/llvm/test/CodeGen/NVPTX/atomicrmw.py
index 966d62bf1b03a..cad1573550a6c 100644
--- a/llvm/test/CodeGen/NVPTX/atomicrmw.py
+++ b/llvm/test/CodeGen/NVPTX/atomicrmw.py
@@ -60,9 +60,17 @@
# SM${sm}-FTZ. (-nvptx-allow-ftz-atomics is covered separately in
# atomicrmw-allow-ftz-atomics.ll.)
run_statement = Template(
- """; RUN: llc < %s -march=nvptx64 -mcpu=sm_${sm} -mattr=+ptx${ptx} | FileCheck %s --check-prefixes=SM${sm},SM${sm}-NOFTZ
-; RUN: llc < %s -march=nvptx64 -mcpu=sm_${sm} -mattr=+ptx${ptx} -denormal-fp-math-f32=preserve-sign | FileCheck %s --check-prefixes=SM${sm},SM${sm}-FTZ
+ """; RUN: llc < %s -march=nvptx64 -mcpu=sm_${sm} -mattr=+ptx${ptx} -nvptx-allow-ftz-atomics=false | FileCheck %s --check-prefixes=SM${sm},SM${sm}-NOFTZ-DISALLOW
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_${sm} -mattr=+ptx${ptx} -nvptx-allow-ftz-atomics=true | FileCheck %s --check-prefixes=SM${sm},SM${sm}-NOFTZ-ALLOW
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_${sm} -mattr=+ptx${ptx} | FileCheck %s --check-prefixes=SM${sm},SM${sm}-NOFTZ-ALLOW
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_${sm} -mattr=+ptx${ptx} -denormal-fp-math-f32=preserve-sign -nvptx-allow-ftz-atomics=false | FileCheck %s --check-prefixes=SM${sm},SM${sm}-FTZ-DISALLOW
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_${sm} -mattr=+ptx${ptx} -denormal-fp-math-f32=preserve-sign -nvptx-allow-ftz-atomics=true | FileCheck %s --check-prefixes=SM${sm},SM${sm}-FTZ-ALLOW
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_${sm} -mattr=+ptx${ptx} -denormal-fp-math-f32=preserve-sign | FileCheck %s --check-prefixes=SM${sm},SM${sm}-FTZ-ALLOW
+; RUN: %if ptxas-sm_${sm} && ptxas-isa-${ptxfp} %{ llc < %s -march=nvptx64 -mcpu=sm_${sm} -mattr=+ptx${ptx} -nvptx-allow-ftz-atomics=false | %ptxas-verify -arch=sm_${sm} %}
+; RUN: %if ptxas-sm_${sm} && ptxas-isa-${ptxfp} %{ llc < %s -march=nvptx64 -mcpu=sm_${sm} -mattr=+ptx${ptx} -nvptx-allow-ftz-atomics=true | %ptxas-verify -arch=sm_${sm} %}
; RUN: %if ptxas-sm_${sm} && ptxas-isa-${ptxfp} %{ llc < %s -march=nvptx64 -mcpu=sm_${sm} -mattr=+ptx${ptx} | %ptxas-verify -arch=sm_${sm} %}
+; RUN: %if ptxas-sm_${sm} && ptxas-isa-${ptxfp} %{ llc < %s -march=nvptx64 -mcpu=sm_${sm} -mattr=+ptx${ptx} -denormal-fp-math-f32=preserve-sign -nvptx-allow-ftz-atomics=false | %ptxas-verify -arch=sm_${sm} %}
+; RUN: %if ptxas-sm_${sm} && ptxas-isa-${ptxfp} %{ llc < %s -march=nvptx64 -mcpu=sm_${sm} -mattr=+ptx${ptx} -denormal-fp-math-f32=preserve-sign -nvptx-allow-ftz-atomics=true | %ptxas-verify -arch=sm_${sm} %}
; RUN: %if ptxas-sm_${sm} && ptxas-isa-${ptxfp} %{ llc < %s -march=nvptx64 -mcpu=sm_${sm} -mattr=+ptx${ptx} -denormal-fp-math-f32=preserve-sign | %ptxas-verify -arch=sm_${sm} %}
"""
)
diff --git a/llvm/test/CodeGen/NVPTX/atomics.ll b/llvm/test/CodeGen/NVPTX/atomics.ll
index ab05bf1b7ea3f..e1853f1b00cf3 100644
--- a/llvm/test/CodeGen/NVPTX/atomics.ll
+++ b/llvm/test/CodeGen/NVPTX/atomics.ll
@@ -1,31 +1,48 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
-; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_32 | FileCheck %s
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_32 -nvptx-allow-ftz-atomics=false | FileCheck %s --check-prefixes=CHECK,DISALLOW
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_32 -nvptx-allow-ftz-atomics=true | FileCheck %s --check-prefixes=CHECK,ALLOW
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_32 | FileCheck %s --check-prefixes=CHECK,ALLOW
+; RUN: %if ptxas %{ llc < %s -mtriple=nvptx64 -mcpu=sm_32 -nvptx-allow-ftz-atomics=false | %ptxas-verify %}
+; RUN: %if ptxas %{ llc < %s -mtriple=nvptx64 -mcpu=sm_32 -nvptx-allow-ftz-atomics=true | %ptxas-verify %}
; RUN: %if ptxas %{ llc < %s -mtriple=nvptx64 -mcpu=sm_32 | %ptxas-verify %}
declare float @llvm.nvvm.atomic.load.add.f32.p0(ptr %addr, float %val)
; CHECK-LABEL: atomic_add_f32_generic
define float @atomic_add_f32_generic(ptr %addr, float %val) {
-; CHECK-LABEL: atomic_add_f32_generic(
-; CHECK: {
-; CHECK-NEXT: .reg .pred %p<2>;
-; CHECK-NEXT: .reg .b32 %r<5>;
-; CHECK-NEXT: .reg .b64 %rd<2>;
-; CHECK-EMPTY:
-; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: ld.param.b32 %r2, [atomic_add_f32_generic_param_1];
-; CHECK-NEXT: ld.param.b64 %rd1, [atomic_add_f32_generic_param_0];
-; CHECK-NEXT: ld.volatile.b32 %r4, [%rd1];
-; CHECK-NEXT: $L__BB0_1: // %atomicrmw.start
-; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: add.rn.f32 %r3, %r4, %r2;
-; CHECK-NEXT: atom.cas.b32 %r1, [%rd1], %r4, %r3;
-; CHECK-NEXT: setp.ne.b32 %p1, %r1, %r4;
-; CHECK-NEXT: mov.b32 %r4, %r1;
-; CHECK-NEXT: @%p1 bra $L__BB0_1;
-; CHECK-NEXT: // %bb.2: // %atomicrmw.end
-; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
-; CHECK-NEXT: ret;
+;
+; DISALLOW-LABEL: atomic_add_f32_generic(
+; DISALLOW: {
+; DISALLOW-NEXT: .reg .pred %p<2>;
+; DISALLOW-NEXT: .reg .b32 %r<5>;
+; DISALLOW-NEXT: .reg .b64 %rd<2>;
+; DISALLOW-EMPTY:
+; DISALLOW-NEXT: // %bb.0:
+; DISALLOW-NEXT: ld.param.b32 %r2, [atomic_add_f32_generic_param_1];
+; DISALLOW-NEXT: ld.param.b64 %rd1, [atomic_add_f32_generic_param_0];
+; DISALLOW-NEXT: ld.volatile.b32 %r4, [%rd1];
+; DISALLOW-NEXT: $L__BB0_1: // %atomicrmw.start
+; DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; DISALLOW-NEXT: add.rn.f32 %r3, %r4, %r2;
+; DISALLOW-NEXT: atom.cas.b32 %r1, [%rd1], %r4, %r3;
+; DISALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; DISALLOW-NEXT: mov.b32 %r4, %r1;
+; DISALLOW-NEXT: @%p1 bra $L__BB0_1;
+; DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; DISALLOW-NEXT: st.param.b32 [func_retval0], %r1;
+; DISALLOW-NEXT: ret;
+;
+; ALLOW-LABEL: atomic_add_f32_generic(
+; ALLOW: {
+; ALLOW-NEXT: .reg .b32 %r<3>;
+; ALLOW-NEXT: .reg .b64 %rd<2>;
+; ALLOW-EMPTY:
+; ALLOW-NEXT: // %bb.0:
+; ALLOW-NEXT: ld.param.b64 %rd1, [atomic_add_f32_generic_param_0];
+; ALLOW-NEXT: ld.param.b32 %r1, [atomic_add_f32_generic_param_1];
+; ALLOW-NEXT: atom.add.f32 %r2, [%rd1], %r1;
+; ALLOW-NEXT: st.param.b32 [func_retval0], %r2;
+; ALLOW-NEXT: ret;
%ret = call float @llvm.nvvm.atomic.load.add.f32.p0(ptr %addr, float %val)
ret float %ret
}
@@ -34,26 +51,39 @@ declare float @llvm.nvvm.atomic.load.add.f32.p1(ptr addrspace(1) %addr, float %v
; CHECK-LABEL: atomic_add_f32_addrspace1
define float @atomic_add_f32_addrspace1(ptr addrspace(1) %addr, float %val) {
-; CHECK-LABEL: atomic_add_f32_addrspace1(
-; CHECK: {
-; CHECK-NEXT: .reg .pred %p<2>;
-; CHECK-NEXT: .reg .b32 %r<5>;
-; CHECK-NEXT: .reg .b64 %rd<2>;
-; CHECK-EMPTY:
-; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: ld.param.b32 %r2, [atomic_add_f32_addrspace1_param_1];
-; CHECK-NEXT: ld.param.b64 %rd1, [atomic_add_f32_addrspace1_param_0];
-; CHECK-NEXT: ld.volatile.global.b32 %r4, [%rd1];
-; CHECK-NEXT: $L__BB1_1: // %atomicrmw.start
-; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: add.rn.f32 %r3, %r4, %r2;
-; CHECK-NEXT: atom.global.cas.b32 %r1, [%rd1], %r4, %r3;
-; CHECK-NEXT: setp.ne.b32 %p1, %r1, %r4;
-; CHECK-NEXT: mov.b32 %r4, %r1;
-; CHECK-NEXT: @%p1 bra $L__BB1_1;
-; CHECK-NEXT: // %bb.2: // %atomicrmw.end
-; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
-; CHECK-NEXT: ret;
+;
+; DISALLOW-LABEL: atomic_add_f32_addrspace1(
+; DISALLOW: {
+; DISALLOW-NEXT: .reg .pred %p<2>;
+; DISALLOW-NEXT: .reg .b32 %r<5>;
+; DISALLOW-NEXT: .reg .b64 %rd<2>;
+; DISALLOW-EMPTY:
+; DISALLOW-NEXT: // %bb.0:
+; DISALLOW-NEXT: ld.param.b32 %r2, [atomic_add_f32_addrspace1_param_1];
+; DISALLOW-NEXT: ld.param.b64 %rd1, [atomic_add_f32_addrspace1_param_0];
+; DISALLOW-NEXT: ld.volatile.global.b32 %r4, [%rd1];
+; DISALLOW-NEXT: $L__BB1_1: // %atomicrmw.start
+; DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
+; DISALLOW-NEXT: add.rn.f32 %r3, %r4, %r2;
+; DISALLOW-NEXT: atom.global.cas.b32 %r1, [%rd1], %r4, %r3;
+; DISALLOW-NEXT: setp.ne.b32 %p1, %r1, %r4;
+; DISALLOW-NEXT: mov.b32 %r4, %r1;
+; DISALLOW-NEXT: @%p1 bra $L__BB1_1;
+; DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
+; DISALLOW-NEXT: st.param.b32 [func_retval0], %r1;
+; DISALLOW-NEXT: ret;
+;
+; ALLOW-LABEL: atomic_add_f32_addrspace1(
+; ALLOW: {
+; ALLOW-NEXT: .reg .b32 %r<3>;
+; ALLOW-NEXT: .reg .b64 %rd<2>;
+; ALLOW-EMPTY:
+; ALLOW-NEXT: // %bb.0:
+; ALLOW-NEXT: ld.param.b64 %rd1, [atomic_add_f32_addrspace1_param_0];
+; ALLOW-NEXT: ld.param.b32 %r1, [atomic_add_f32_addrspace1_param_1];
+; ALLOW-NEXT: atom.global.add.f32 %r2, [%rd1], %r1;
+; ALLOW-NEXT: st.param.b32 [func_retval0], %r2;
+; ALLOW-NEXT: ret;
%ret = call float @llvm.nvvm.atomic.load.add.f32.p1(ptr addrspace(1) %addr, float %val)
ret float %ret
}
More information about the llvm-branch-commits
mailing list