[llvm] [NVPTX] Fix BFE selection for i64 values (PR #228685)
via llvm-commits
llvm-commits at lists.llvm.org
Sat Oct 3 23:52:21 PDT 2026
https://github.com/wenhu1024 updated https://github.com/llvm/llvm-project/pull/228685
>From 563c988f3a142baf823f0c541152a7eb063a5d8b Mon Sep 17 00:00:00 2001
From: wenhu1024 <1340773297 at qq.com>
Date: Sat, 3 Oct 2026 17:41:41 +0800
Subject: [PATCH] [NVPTX] Restrict BFE selection to native targets
BFE is only a native SASS instruction for 32-bit values before sm_70. Keep the combine for those targets, where it can fold patterns that ptxas does not recognize, but leave i64 and sm_70+ operations unchanged for ptxas to optimize.
---
llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp | 31 +-
.../CodeGen/NVPTX/arbitrary-fp-to-float.ll | 1192 +++++++++--------
llvm/test/CodeGen/NVPTX/atomicrmw-sm70.ll | 560 ++++----
llvm/test/CodeGen/NVPTX/bf16-instructions.ll | 404 +++---
llvm/test/CodeGen/NVPTX/bfe.ll | 239 +++-
llvm/test/CodeGen/NVPTX/convert-fp-i128.ll | 282 ++--
llvm/test/CodeGen/NVPTX/fexp2.ll | 53 +-
.../CodeGen/NVPTX/float-to-arbitrary-fp.ll | 416 +++---
llvm/test/CodeGen/NVPTX/fma-relu-contract.ll | 441 +++---
.../CodeGen/NVPTX/fma-relu-fma-intrinsic.ll | 346 ++---
.../NVPTX/fma-relu-instruction-flag.ll | 716 +++++-----
llvm/test/CodeGen/NVPTX/mulwide.ll | 12 +-
llvm/test/CodeGen/NVPTX/tanhf.ll | 53 +-
13 files changed, 2554 insertions(+), 2191 deletions(-)
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp b/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp
index 186bfa185e135..b952caf3a7775 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp
@@ -1773,6 +1773,11 @@ bool NVPTXDAGToDAGISel::tryStoreVector(SDNode *N) {
/// SelectBFE - Look for instruction sequences that can be made more efficient
/// by using the 'bfe' (bit-field extract) PTX instruction
bool NVPTXDAGToDAGISel::tryBFE(SDNode *N) {
+ // BFE is not a native instruction starting with sm_70. Let ptxas see the
+ // original operations so it can optimize them for the target architecture.
+ if (Subtarget->hasFeature(NVPTX::SM70))
+ return false;
+
SDLoc DL(N);
SDValue LHS = N->getOperand(0);
SDValue RHS = N->getOperand(1);
@@ -1816,7 +1821,7 @@ bool NVPTXDAGToDAGISel::tryBFE(SDNode *N) {
uint64_t StartVal = StartConst->getZExtValue();
// How many "good" bits do we have left? "good" is defined here as bits
// that exist in the original value, not shifted in.
- int64_t GoodBits = Start.getValueSizeInBits() - StartVal;
+ int64_t GoodBits = Val.getValueSizeInBits() - StartVal;
if (NumBits > GoodBits) {
// Do not handle the case where bits have been shifted in. In theory
// we could handle this, but the cost is likely higher than just
@@ -1953,27 +1958,11 @@ bool NVPTXDAGToDAGISel::tryBFE(SDNode *N) {
// No can do...
return false;
}
-
-
- unsigned Opc;
- // For the BFE operations we form here from "and" and "srl", always use the
- // unsigned variants.
- if (Val.getValueType() == MVT::i32) {
- if (IsSigned) {
- Opc = NVPTX::BFE_S32rii;
- } else {
- Opc = NVPTX::BFE_U32rii;
- }
- } else if (Val.getValueType() == MVT::i64) {
- if (IsSigned) {
- Opc = NVPTX::BFE_S64rii;
- } else {
- Opc = NVPTX::BFE_U64rii;
- }
- } else {
- // We cannot handle this type
+ // Only 32-bit BFE has a native SASS implementation.
+ if (Val.getValueType() != MVT::i32)
return false;
- }
+
+ unsigned Opc = IsSigned ? NVPTX::BFE_S32rii : NVPTX::BFE_U32rii;
SDValue Ops[] = {
Val, Start, Len
diff --git a/llvm/test/CodeGen/NVPTX/arbitrary-fp-to-float.ll b/llvm/test/CodeGen/NVPTX/arbitrary-fp-to-float.ll
index 78be7a35129c1..ebad0a960847c 100644
--- a/llvm/test/CodeGen/NVPTX/arbitrary-fp-to-float.ll
+++ b/llvm/test/CodeGen/NVPTX/arbitrary-fp-to-float.ll
@@ -122,7 +122,7 @@ define float @from_f8e5m2_dynamic(i8 %x) {
; CHECK-LABEL: from_f8e5m2_dynamic(
; CHECK: {
; CHECK-NEXT: .reg .pred %p<5>;
-; CHECK-NEXT: .reg .b32 %r<31>;
+; CHECK-NEXT: .reg .b32 %r<32>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
; CHECK-NEXT: ld.param.b8 %r1, [from_f8e5m2_dynamic_param_0];
@@ -140,26 +140,27 @@ define float @from_f8e5m2_dynamic(i8 %x) {
; CHECK-NEXT: add.s32 %r13, %r5, -8;
; CHECK-NEXT: shl.b32 %r14, %r12, %r13;
; CHECK-NEXT: or.b32 %r15, %r8, %r14;
-; CHECK-NEXT: bfe.u32 %r16, %r1, 2, 5;
-; CHECK-NEXT: shl.b32 %r17, %r16, 23;
-; CHECK-NEXT: or.b32 %r18, %r17, %r3;
-; CHECK-NEXT: shl.b32 %r19, %r4, 21;
-; CHECK-NEXT: or.b32 %r20, %r18, %r19;
-; CHECK-NEXT: add.s32 %r21, %r20, 939524096;
+; CHECK-NEXT: shr.u32 %r16, %r1, 2;
+; CHECK-NEXT: and.b32 %r17, %r16, 31;
+; CHECK-NEXT: shl.b32 %r18, %r17, 23;
+; CHECK-NEXT: or.b32 %r19, %r18, %r3;
+; CHECK-NEXT: shl.b32 %r20, %r4, 21;
+; CHECK-NEXT: or.b32 %r21, %r19, %r20;
+; CHECK-NEXT: add.s32 %r22, %r21, 939524096;
; CHECK-NEXT: setp.ne.b32 %p1, %r4, 0;
-; CHECK-NEXT: selp.b32 %r22, %r15, %r21, %p1;
-; CHECK-NEXT: setp.eq.b32 %p2, %r16, 0;
-; CHECK-NEXT: selp.b32 %r23, %r22, %r21, %p2;
-; CHECK-NEXT: or.b32 %r24, %r16, %r4;
-; CHECK-NEXT: setp.eq.b32 %p3, %r24, 0;
-; CHECK-NEXT: selp.b32 %r25, %r3, %r23, %p3;
-; CHECK-NEXT: or.b32 %r26, %r3, 2139095040;
-; CHECK-NEXT: selp.b32 %r27, %r25, %r26, %p1;
-; CHECK-NEXT: setp.eq.b32 %p4, %r16, 31;
-; CHECK-NEXT: selp.b32 %r28, %r27, %r25, %p4;
-; CHECK-NEXT: selp.b32 %r29, 2143289344, %r28, %p1;
-; CHECK-NEXT: selp.b32 %r30, %r29, %r28, %p4;
-; CHECK-NEXT: st.param.b32 [func_retval0], %r30;
+; CHECK-NEXT: selp.b32 %r23, %r15, %r22, %p1;
+; CHECK-NEXT: setp.eq.b32 %p2, %r17, 0;
+; CHECK-NEXT: selp.b32 %r24, %r23, %r22, %p2;
+; CHECK-NEXT: or.b32 %r25, %r17, %r4;
+; CHECK-NEXT: setp.eq.b32 %p3, %r25, 0;
+; CHECK-NEXT: selp.b32 %r26, %r3, %r24, %p3;
+; CHECK-NEXT: or.b32 %r27, %r3, 2139095040;
+; CHECK-NEXT: selp.b32 %r28, %r26, %r27, %p1;
+; CHECK-NEXT: setp.eq.b32 %p4, %r17, 31;
+; CHECK-NEXT: selp.b32 %r29, %r28, %r26, %p4;
+; CHECK-NEXT: selp.b32 %r30, 2143289344, %r29, %p1;
+; CHECK-NEXT: selp.b32 %r31, %r30, %r29, %p4;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r31;
; CHECK-NEXT: ret;
%r = call float @llvm.convert.from.arbitrary.fp.f32.i8(i8 %x, metadata !"Float8E5M2")
ret float %r
@@ -248,7 +249,7 @@ define float @from_f8e4m3fn_dynamic(i8 %x) {
; CHECK-LABEL: from_f8e4m3fn_dynamic(
; CHECK: {
; CHECK-NEXT: .reg .pred %p<6>;
-; CHECK-NEXT: .reg .b32 %r<28>;
+; CHECK-NEXT: .reg .b32 %r<29>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
; CHECK-NEXT: ld.param.b8 %r1, [from_f8e4m3fn_dynamic_param_0];
@@ -266,24 +267,25 @@ define float @from_f8e4m3fn_dynamic(i8 %x) {
; CHECK-NEXT: add.s32 %r13, %r5, -8;
; CHECK-NEXT: shl.b32 %r14, %r12, %r13;
; CHECK-NEXT: or.b32 %r15, %r8, %r14;
-; CHECK-NEXT: bfe.u32 %r16, %r1, 3, 4;
-; CHECK-NEXT: shl.b32 %r17, %r16, 23;
-; CHECK-NEXT: or.b32 %r18, %r17, %r3;
-; CHECK-NEXT: shl.b32 %r19, %r4, 20;
-; CHECK-NEXT: or.b32 %r20, %r18, %r19;
-; CHECK-NEXT: add.s32 %r21, %r20, 1006632960;
+; CHECK-NEXT: shr.u32 %r16, %r1, 3;
+; CHECK-NEXT: and.b32 %r17, %r16, 15;
+; CHECK-NEXT: shl.b32 %r18, %r17, 23;
+; CHECK-NEXT: or.b32 %r19, %r18, %r3;
+; CHECK-NEXT: shl.b32 %r20, %r4, 20;
+; CHECK-NEXT: or.b32 %r21, %r19, %r20;
+; CHECK-NEXT: add.s32 %r22, %r21, 1006632960;
; CHECK-NEXT: setp.ne.b32 %p1, %r4, 0;
-; CHECK-NEXT: selp.b32 %r22, %r15, %r21, %p1;
-; CHECK-NEXT: setp.eq.b32 %p2, %r16, 0;
-; CHECK-NEXT: selp.b32 %r23, %r22, %r21, %p2;
-; CHECK-NEXT: or.b32 %r24, %r16, %r4;
-; CHECK-NEXT: setp.eq.b32 %p3, %r24, 0;
-; CHECK-NEXT: selp.b32 %r25, %r3, %r23, %p3;
+; CHECK-NEXT: selp.b32 %r23, %r15, %r22, %p1;
+; CHECK-NEXT: setp.eq.b32 %p2, %r17, 0;
+; CHECK-NEXT: selp.b32 %r24, %r23, %r22, %p2;
+; CHECK-NEXT: or.b32 %r25, %r17, %r4;
+; CHECK-NEXT: setp.eq.b32 %p3, %r25, 0;
+; CHECK-NEXT: selp.b32 %r26, %r3, %r24, %p3;
; CHECK-NEXT: setp.eq.b32 %p4, %r4, 7;
-; CHECK-NEXT: selp.b32 %r26, 2143289344, %r25, %p4;
-; CHECK-NEXT: setp.eq.b32 %p5, %r16, 15;
-; CHECK-NEXT: selp.b32 %r27, %r26, %r25, %p5;
-; CHECK-NEXT: st.param.b32 [func_retval0], %r27;
+; CHECK-NEXT: selp.b32 %r27, 2143289344, %r26, %p4;
+; CHECK-NEXT: setp.eq.b32 %p5, %r17, 15;
+; CHECK-NEXT: selp.b32 %r28, %r27, %r26, %p5;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r28;
; CHECK-NEXT: ret;
%r = call float @llvm.convert.from.arbitrary.fp.f32.i8(i8 %x, metadata !"Float8E4M3FN")
ret float %r
@@ -359,7 +361,7 @@ define float @from_f6e3m2fn_dynamic(i6 %x) {
; CHECK: {
; CHECK-NEXT: .reg .pred %p<4>;
; CHECK-NEXT: .reg .b16 %rs<5>;
-; CHECK-NEXT: .reg .b32 %r<26>;
+; CHECK-NEXT: .reg .b32 %r<27>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
; CHECK-NEXT: ld.param.b8 %rs1, [from_f6e3m2fn_dynamic_param_0+1];
@@ -381,20 +383,21 @@ define float @from_f6e3m2fn_dynamic(i6 %x) {
; CHECK-NEXT: add.s32 %r13, %r5, -8;
; CHECK-NEXT: shl.b32 %r14, %r12, %r13;
; CHECK-NEXT: or.b32 %r15, %r8, %r14;
-; CHECK-NEXT: bfe.u32 %r16, %r1, 2, 3;
-; CHECK-NEXT: shl.b32 %r17, %r16, 23;
-; CHECK-NEXT: or.b32 %r18, %r17, %r3;
-; CHECK-NEXT: shl.b32 %r19, %r4, 21;
-; CHECK-NEXT: or.b32 %r20, %r18, %r19;
-; CHECK-NEXT: add.s32 %r21, %r20, 1040187392;
+; CHECK-NEXT: shr.u32 %r16, %r1, 2;
+; CHECK-NEXT: and.b32 %r17, %r16, 7;
+; CHECK-NEXT: shl.b32 %r18, %r17, 23;
+; CHECK-NEXT: or.b32 %r19, %r18, %r3;
+; CHECK-NEXT: shl.b32 %r20, %r4, 21;
+; CHECK-NEXT: or.b32 %r21, %r19, %r20;
+; CHECK-NEXT: add.s32 %r22, %r21, 1040187392;
; CHECK-NEXT: setp.ne.b32 %p1, %r4, 0;
-; CHECK-NEXT: selp.b32 %r22, %r15, %r21, %p1;
-; CHECK-NEXT: setp.eq.b32 %p2, %r16, 0;
-; CHECK-NEXT: selp.b32 %r23, %r22, %r21, %p2;
-; CHECK-NEXT: or.b32 %r24, %r16, %r4;
-; CHECK-NEXT: setp.eq.b32 %p3, %r24, 0;
-; CHECK-NEXT: selp.b32 %r25, %r3, %r23, %p3;
-; CHECK-NEXT: st.param.b32 [func_retval0], %r25;
+; CHECK-NEXT: selp.b32 %r23, %r15, %r22, %p1;
+; CHECK-NEXT: setp.eq.b32 %p2, %r17, 0;
+; CHECK-NEXT: selp.b32 %r24, %r23, %r22, %p2;
+; CHECK-NEXT: or.b32 %r25, %r17, %r4;
+; CHECK-NEXT: setp.eq.b32 %p3, %r25, 0;
+; CHECK-NEXT: selp.b32 %r26, %r3, %r24, %p3;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r26;
; CHECK-NEXT: ret;
%r = call float @llvm.convert.from.arbitrary.fp.f32.i6(i6 %x, metadata !"Float6E3M2FN")
ret float %r
@@ -458,7 +461,7 @@ define float @from_f6e2m3fn_dynamic(i6 %x) {
; CHECK: {
; CHECK-NEXT: .reg .pred %p<4>;
; CHECK-NEXT: .reg .b16 %rs<5>;
-; CHECK-NEXT: .reg .b32 %r<26>;
+; CHECK-NEXT: .reg .b32 %r<27>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
; CHECK-NEXT: ld.param.b8 %rs1, [from_f6e2m3fn_dynamic_param_0+1];
@@ -480,20 +483,21 @@ define float @from_f6e2m3fn_dynamic(i6 %x) {
; CHECK-NEXT: add.s32 %r13, %r5, -8;
; CHECK-NEXT: shl.b32 %r14, %r12, %r13;
; CHECK-NEXT: or.b32 %r15, %r8, %r14;
-; CHECK-NEXT: bfe.u32 %r16, %r1, 3, 2;
-; CHECK-NEXT: shl.b32 %r17, %r16, 23;
-; CHECK-NEXT: or.b32 %r18, %r17, %r3;
-; CHECK-NEXT: shl.b32 %r19, %r4, 20;
-; CHECK-NEXT: or.b32 %r20, %r18, %r19;
-; CHECK-NEXT: add.s32 %r21, %r20, 1056964608;
+; CHECK-NEXT: shr.u32 %r16, %r1, 3;
+; CHECK-NEXT: and.b32 %r17, %r16, 3;
+; CHECK-NEXT: shl.b32 %r18, %r17, 23;
+; CHECK-NEXT: or.b32 %r19, %r18, %r3;
+; CHECK-NEXT: shl.b32 %r20, %r4, 20;
+; CHECK-NEXT: or.b32 %r21, %r19, %r20;
+; CHECK-NEXT: add.s32 %r22, %r21, 1056964608;
; CHECK-NEXT: setp.ne.b32 %p1, %r4, 0;
-; CHECK-NEXT: selp.b32 %r22, %r15, %r21, %p1;
-; CHECK-NEXT: setp.eq.b32 %p2, %r16, 0;
-; CHECK-NEXT: selp.b32 %r23, %r22, %r21, %p2;
-; CHECK-NEXT: or.b32 %r24, %r16, %r4;
-; CHECK-NEXT: setp.eq.b32 %p3, %r24, 0;
-; CHECK-NEXT: selp.b32 %r25, %r3, %r23, %p3;
-; CHECK-NEXT: st.param.b32 [func_retval0], %r25;
+; CHECK-NEXT: selp.b32 %r23, %r15, %r22, %p1;
+; CHECK-NEXT: setp.eq.b32 %p2, %r17, 0;
+; CHECK-NEXT: selp.b32 %r24, %r23, %r22, %p2;
+; CHECK-NEXT: or.b32 %r25, %r17, %r4;
+; CHECK-NEXT: setp.eq.b32 %p3, %r25, 0;
+; CHECK-NEXT: selp.b32 %r26, %r3, %r24, %p3;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r26;
; CHECK-NEXT: ret;
%r = call float @llvm.convert.from.arbitrary.fp.f32.i6(i6 %x, metadata !"Float6E2M3FN")
ret float %r
@@ -545,7 +549,7 @@ define float @from_f4e2m1fn_dynamic(i4 %x) {
; CHECK: {
; CHECK-NEXT: .reg .pred %p<4>;
; CHECK-NEXT: .reg .b16 %rs<6>;
-; CHECK-NEXT: .reg .b32 %r<26>;
+; CHECK-NEXT: .reg .b32 %r<27>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
; CHECK-NEXT: ld.param.b8 %rs1, [from_f4e2m1fn_dynamic_param_0+1];
@@ -567,21 +571,22 @@ define float @from_f4e2m1fn_dynamic(i4 %x) {
; CHECK-NEXT: add.s32 %r13, %r5, -8;
; CHECK-NEXT: shl.b32 %r14, %r12, %r13;
; CHECK-NEXT: or.b32 %r15, %r8, %r14;
-; CHECK-NEXT: bfe.u32 %r16, %r1, 1, 2;
-; CHECK-NEXT: shl.b32 %r17, %r16, 23;
-; CHECK-NEXT: or.b32 %r18, %r17, %r3;
-; CHECK-NEXT: shl.b32 %r19, %r4, 22;
-; CHECK-NEXT: or.b32 %r20, %r18, %r19;
-; CHECK-NEXT: add.s32 %r21, %r20, 1056964608;
+; CHECK-NEXT: shr.u32 %r16, %r1, 1;
+; CHECK-NEXT: and.b32 %r17, %r16, 3;
+; CHECK-NEXT: shl.b32 %r18, %r17, 23;
+; CHECK-NEXT: or.b32 %r19, %r18, %r3;
+; CHECK-NEXT: shl.b32 %r20, %r4, 22;
+; CHECK-NEXT: or.b32 %r21, %r19, %r20;
+; CHECK-NEXT: add.s32 %r22, %r21, 1056964608;
; CHECK-NEXT: and.b16 %rs5, %rs3, 1;
; CHECK-NEXT: setp.ne.b16 %p1, %rs5, 0;
-; CHECK-NEXT: selp.b32 %r22, %r15, %r21, %p1;
-; CHECK-NEXT: setp.eq.b32 %p2, %r16, 0;
-; CHECK-NEXT: selp.b32 %r23, %r22, %r21, %p2;
-; CHECK-NEXT: or.b32 %r24, %r16, %r4;
-; CHECK-NEXT: setp.eq.b32 %p3, %r24, 0;
-; CHECK-NEXT: selp.b32 %r25, %r3, %r23, %p3;
-; CHECK-NEXT: st.param.b32 [func_retval0], %r25;
+; CHECK-NEXT: selp.b32 %r23, %r15, %r22, %p1;
+; CHECK-NEXT: setp.eq.b32 %p2, %r17, 0;
+; CHECK-NEXT: selp.b32 %r24, %r23, %r22, %p2;
+; CHECK-NEXT: or.b32 %r25, %r17, %r4;
+; CHECK-NEXT: setp.eq.b32 %p3, %r25, 0;
+; CHECK-NEXT: selp.b32 %r26, %r3, %r24, %p3;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r26;
; CHECK-NEXT: ret;
%r = call float @llvm.convert.from.arbitrary.fp.f32.i4(i4 %x, metadata !"Float4E2M1FN")
ret float %r
@@ -616,7 +621,7 @@ define <4 x float> @fp4_to_f32_vec(<4 x i4> %x) {
; CHECK-LABEL: fp4_to_f32_vec(
; CHECK: {
; CHECK-NEXT: .reg .pred %p<13>;
-; CHECK-NEXT: .reg .b32 %r<101>;
+; CHECK-NEXT: .reg .b32 %r<105>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
; CHECK-NEXT: ld.param.b16 %r1, [fp4_to_f32_vec_param_0+2];
@@ -637,101 +642,105 @@ define <4 x float> @fp4_to_f32_vec(<4 x i4> %x) {
; CHECK-NEXT: add.s32 %r16, %r8, -8;
; CHECK-NEXT: shl.b32 %r17, %r15, %r16;
; CHECK-NEXT: or.b32 %r18, %r11, %r17;
-; CHECK-NEXT: bfe.u32 %r19, %r4, 1, 2;
-; CHECK-NEXT: shl.b32 %r20, %r19, 23;
-; CHECK-NEXT: or.b32 %r21, %r20, %r6;
-; CHECK-NEXT: shl.b32 %r22, %r7, 22;
-; CHECK-NEXT: or.b32 %r23, %r21, %r22;
-; CHECK-NEXT: add.s32 %r24, %r23, 1056964608;
+; CHECK-NEXT: shr.u32 %r19, %r4, 1;
+; CHECK-NEXT: and.b32 %r20, %r19, 3;
+; CHECK-NEXT: shl.b32 %r21, %r20, 23;
+; CHECK-NEXT: or.b32 %r22, %r21, %r6;
+; CHECK-NEXT: shl.b32 %r23, %r7, 22;
+; CHECK-NEXT: or.b32 %r24, %r22, %r23;
+; CHECK-NEXT: add.s32 %r25, %r24, 1056964608;
; CHECK-NEXT: setp.ne.b32 %p1, %r7, 0;
-; CHECK-NEXT: selp.b32 %r25, %r18, %r24, %p1;
-; CHECK-NEXT: setp.eq.b32 %p2, %r19, 0;
-; CHECK-NEXT: selp.b32 %r26, %r25, %r24, %p2;
-; CHECK-NEXT: or.b32 %r27, %r19, %r7;
-; CHECK-NEXT: setp.eq.b32 %p3, %r27, 0;
-; CHECK-NEXT: selp.b32 %r28, %r6, %r26, %p3;
-; CHECK-NEXT: prmt.b32 %r29, %r3, 0, 0x7770U;
-; CHECK-NEXT: shl.b32 %r30, %r29, 28;
-; CHECK-NEXT: and.b32 %r31, %r30, -2147483648;
-; CHECK-NEXT: and.b32 %r32, %r29, 1;
-; CHECK-NEXT: clz.b32 %r33, %r32;
-; CHECK-NEXT: sub.s32 %r34, 157, %r33;
-; CHECK-NEXT: shl.b32 %r35, %r34, 23;
-; CHECK-NEXT: or.b32 %r36, %r31, %r35;
-; CHECK-NEXT: sub.s32 %r37, 31, %r33;
-; CHECK-NEXT: shl.b32 %r38, %r13, %r37;
-; CHECK-NEXT: xor.b32 %r39, %r32, %r38;
-; CHECK-NEXT: add.s32 %r40, %r33, -8;
-; CHECK-NEXT: shl.b32 %r41, %r39, %r40;
-; CHECK-NEXT: or.b32 %r42, %r36, %r41;
-; CHECK-NEXT: bfe.u32 %r43, %r29, 1, 2;
-; CHECK-NEXT: shl.b32 %r44, %r43, 23;
-; CHECK-NEXT: or.b32 %r45, %r44, %r31;
-; CHECK-NEXT: shl.b32 %r46, %r32, 22;
-; CHECK-NEXT: or.b32 %r47, %r45, %r46;
-; CHECK-NEXT: add.s32 %r48, %r47, 1056964608;
-; CHECK-NEXT: setp.ne.b32 %p4, %r32, 0;
-; CHECK-NEXT: selp.b32 %r49, %r42, %r48, %p4;
-; CHECK-NEXT: setp.eq.b32 %p5, %r43, 0;
-; CHECK-NEXT: selp.b32 %r50, %r49, %r48, %p5;
-; CHECK-NEXT: or.b32 %r51, %r43, %r32;
-; CHECK-NEXT: setp.eq.b32 %p6, %r51, 0;
-; CHECK-NEXT: selp.b32 %r52, %r31, %r50, %p6;
-; CHECK-NEXT: prmt.b32 %r53, %r2, 0, 0x7773U;
-; CHECK-NEXT: shl.b32 %r54, %r53, 28;
-; CHECK-NEXT: and.b32 %r55, %r54, -2147483648;
-; CHECK-NEXT: and.b32 %r56, %r53, 1;
-; CHECK-NEXT: clz.b32 %r57, %r56;
-; CHECK-NEXT: sub.s32 %r58, 157, %r57;
-; CHECK-NEXT: shl.b32 %r59, %r58, 23;
-; CHECK-NEXT: or.b32 %r60, %r55, %r59;
-; CHECK-NEXT: sub.s32 %r61, 31, %r57;
-; CHECK-NEXT: shl.b32 %r62, %r13, %r61;
-; CHECK-NEXT: xor.b32 %r63, %r56, %r62;
-; CHECK-NEXT: add.s32 %r64, %r57, -8;
-; CHECK-NEXT: shl.b32 %r65, %r63, %r64;
-; CHECK-NEXT: or.b32 %r66, %r60, %r65;
-; CHECK-NEXT: bfe.u32 %r67, %r53, 1, 2;
-; CHECK-NEXT: shl.b32 %r68, %r67, 23;
-; CHECK-NEXT: or.b32 %r69, %r68, %r55;
-; CHECK-NEXT: shl.b32 %r70, %r56, 22;
-; CHECK-NEXT: or.b32 %r71, %r69, %r70;
-; CHECK-NEXT: add.s32 %r72, %r71, 1056964608;
-; CHECK-NEXT: setp.ne.b32 %p7, %r56, 0;
-; CHECK-NEXT: selp.b32 %r73, %r66, %r72, %p7;
-; CHECK-NEXT: setp.eq.b32 %p8, %r67, 0;
-; CHECK-NEXT: selp.b32 %r74, %r73, %r72, %p8;
-; CHECK-NEXT: or.b32 %r75, %r67, %r56;
-; CHECK-NEXT: setp.eq.b32 %p9, %r75, 0;
-; CHECK-NEXT: selp.b32 %r76, %r55, %r74, %p9;
-; CHECK-NEXT: prmt.b32 %r77, %r2, 0, 0x7772U;
-; CHECK-NEXT: shl.b32 %r78, %r77, 28;
-; CHECK-NEXT: and.b32 %r79, %r78, -2147483648;
-; CHECK-NEXT: and.b32 %r80, %r77, 1;
-; CHECK-NEXT: clz.b32 %r81, %r80;
-; CHECK-NEXT: sub.s32 %r82, 157, %r81;
-; CHECK-NEXT: shl.b32 %r83, %r82, 23;
-; CHECK-NEXT: or.b32 %r84, %r79, %r83;
-; CHECK-NEXT: sub.s32 %r85, 31, %r81;
-; CHECK-NEXT: shl.b32 %r86, %r13, %r85;
-; CHECK-NEXT: xor.b32 %r87, %r80, %r86;
-; CHECK-NEXT: add.s32 %r88, %r81, -8;
-; CHECK-NEXT: shl.b32 %r89, %r87, %r88;
-; CHECK-NEXT: or.b32 %r90, %r84, %r89;
-; CHECK-NEXT: bfe.u32 %r91, %r77, 1, 2;
-; CHECK-NEXT: shl.b32 %r92, %r91, 23;
-; CHECK-NEXT: or.b32 %r93, %r92, %r79;
-; CHECK-NEXT: shl.b32 %r94, %r80, 22;
-; CHECK-NEXT: or.b32 %r95, %r93, %r94;
-; CHECK-NEXT: add.s32 %r96, %r95, 1056964608;
-; CHECK-NEXT: setp.ne.b32 %p10, %r80, 0;
-; CHECK-NEXT: selp.b32 %r97, %r90, %r96, %p10;
-; CHECK-NEXT: setp.eq.b32 %p11, %r91, 0;
-; CHECK-NEXT: selp.b32 %r98, %r97, %r96, %p11;
-; CHECK-NEXT: or.b32 %r99, %r91, %r80;
-; CHECK-NEXT: setp.eq.b32 %p12, %r99, 0;
-; CHECK-NEXT: selp.b32 %r100, %r79, %r98, %p12;
-; CHECK-NEXT: st.param.v4.b32 [func_retval0], {%r52, %r28, %r100, %r76};
+; CHECK-NEXT: selp.b32 %r26, %r18, %r25, %p1;
+; CHECK-NEXT: setp.eq.b32 %p2, %r20, 0;
+; CHECK-NEXT: selp.b32 %r27, %r26, %r25, %p2;
+; CHECK-NEXT: or.b32 %r28, %r20, %r7;
+; CHECK-NEXT: setp.eq.b32 %p3, %r28, 0;
+; CHECK-NEXT: selp.b32 %r29, %r6, %r27, %p3;
+; CHECK-NEXT: prmt.b32 %r30, %r3, 0, 0x7770U;
+; CHECK-NEXT: shl.b32 %r31, %r30, 28;
+; CHECK-NEXT: and.b32 %r32, %r31, -2147483648;
+; CHECK-NEXT: and.b32 %r33, %r30, 1;
+; CHECK-NEXT: clz.b32 %r34, %r33;
+; CHECK-NEXT: sub.s32 %r35, 157, %r34;
+; CHECK-NEXT: shl.b32 %r36, %r35, 23;
+; CHECK-NEXT: or.b32 %r37, %r32, %r36;
+; CHECK-NEXT: sub.s32 %r38, 31, %r34;
+; CHECK-NEXT: shl.b32 %r39, %r13, %r38;
+; CHECK-NEXT: xor.b32 %r40, %r33, %r39;
+; CHECK-NEXT: add.s32 %r41, %r34, -8;
+; CHECK-NEXT: shl.b32 %r42, %r40, %r41;
+; CHECK-NEXT: or.b32 %r43, %r37, %r42;
+; CHECK-NEXT: shr.u32 %r44, %r30, 1;
+; CHECK-NEXT: and.b32 %r45, %r44, 3;
+; CHECK-NEXT: shl.b32 %r46, %r45, 23;
+; CHECK-NEXT: or.b32 %r47, %r46, %r32;
+; CHECK-NEXT: shl.b32 %r48, %r33, 22;
+; CHECK-NEXT: or.b32 %r49, %r47, %r48;
+; CHECK-NEXT: add.s32 %r50, %r49, 1056964608;
+; CHECK-NEXT: setp.ne.b32 %p4, %r33, 0;
+; CHECK-NEXT: selp.b32 %r51, %r43, %r50, %p4;
+; CHECK-NEXT: setp.eq.b32 %p5, %r45, 0;
+; CHECK-NEXT: selp.b32 %r52, %r51, %r50, %p5;
+; CHECK-NEXT: or.b32 %r53, %r45, %r33;
+; CHECK-NEXT: setp.eq.b32 %p6, %r53, 0;
+; CHECK-NEXT: selp.b32 %r54, %r32, %r52, %p6;
+; CHECK-NEXT: prmt.b32 %r55, %r2, 0, 0x7773U;
+; CHECK-NEXT: shl.b32 %r56, %r55, 28;
+; CHECK-NEXT: and.b32 %r57, %r56, -2147483648;
+; CHECK-NEXT: and.b32 %r58, %r55, 1;
+; CHECK-NEXT: clz.b32 %r59, %r58;
+; CHECK-NEXT: sub.s32 %r60, 157, %r59;
+; CHECK-NEXT: shl.b32 %r61, %r60, 23;
+; CHECK-NEXT: or.b32 %r62, %r57, %r61;
+; CHECK-NEXT: sub.s32 %r63, 31, %r59;
+; CHECK-NEXT: shl.b32 %r64, %r13, %r63;
+; CHECK-NEXT: xor.b32 %r65, %r58, %r64;
+; CHECK-NEXT: add.s32 %r66, %r59, -8;
+; CHECK-NEXT: shl.b32 %r67, %r65, %r66;
+; CHECK-NEXT: or.b32 %r68, %r62, %r67;
+; CHECK-NEXT: shr.u32 %r69, %r55, 1;
+; CHECK-NEXT: and.b32 %r70, %r69, 3;
+; CHECK-NEXT: shl.b32 %r71, %r70, 23;
+; CHECK-NEXT: or.b32 %r72, %r71, %r57;
+; CHECK-NEXT: shl.b32 %r73, %r58, 22;
+; CHECK-NEXT: or.b32 %r74, %r72, %r73;
+; CHECK-NEXT: add.s32 %r75, %r74, 1056964608;
+; CHECK-NEXT: setp.ne.b32 %p7, %r58, 0;
+; CHECK-NEXT: selp.b32 %r76, %r68, %r75, %p7;
+; CHECK-NEXT: setp.eq.b32 %p8, %r70, 0;
+; CHECK-NEXT: selp.b32 %r77, %r76, %r75, %p8;
+; CHECK-NEXT: or.b32 %r78, %r70, %r58;
+; CHECK-NEXT: setp.eq.b32 %p9, %r78, 0;
+; CHECK-NEXT: selp.b32 %r79, %r57, %r77, %p9;
+; CHECK-NEXT: prmt.b32 %r80, %r2, 0, 0x7772U;
+; CHECK-NEXT: shl.b32 %r81, %r80, 28;
+; CHECK-NEXT: and.b32 %r82, %r81, -2147483648;
+; CHECK-NEXT: and.b32 %r83, %r80, 1;
+; CHECK-NEXT: clz.b32 %r84, %r83;
+; CHECK-NEXT: sub.s32 %r85, 157, %r84;
+; CHECK-NEXT: shl.b32 %r86, %r85, 23;
+; CHECK-NEXT: or.b32 %r87, %r82, %r86;
+; CHECK-NEXT: sub.s32 %r88, 31, %r84;
+; CHECK-NEXT: shl.b32 %r89, %r13, %r88;
+; CHECK-NEXT: xor.b32 %r90, %r83, %r89;
+; CHECK-NEXT: add.s32 %r91, %r84, -8;
+; CHECK-NEXT: shl.b32 %r92, %r90, %r91;
+; CHECK-NEXT: or.b32 %r93, %r87, %r92;
+; CHECK-NEXT: shr.u32 %r94, %r80, 1;
+; CHECK-NEXT: and.b32 %r95, %r94, 3;
+; CHECK-NEXT: shl.b32 %r96, %r95, 23;
+; CHECK-NEXT: or.b32 %r97, %r96, %r82;
+; CHECK-NEXT: shl.b32 %r98, %r83, 22;
+; CHECK-NEXT: or.b32 %r99, %r97, %r98;
+; CHECK-NEXT: add.s32 %r100, %r99, 1056964608;
+; CHECK-NEXT: setp.ne.b32 %p10, %r83, 0;
+; CHECK-NEXT: selp.b32 %r101, %r93, %r100, %p10;
+; CHECK-NEXT: setp.eq.b32 %p11, %r95, 0;
+; CHECK-NEXT: selp.b32 %r102, %r101, %r100, %p11;
+; CHECK-NEXT: or.b32 %r103, %r95, %r83;
+; CHECK-NEXT: setp.eq.b32 %p12, %r103, 0;
+; CHECK-NEXT: selp.b32 %r104, %r82, %r102, %p12;
+; CHECK-NEXT: st.param.v4.b32 [func_retval0], {%r54, %r29, %r104, %r79};
; CHECK-NEXT: ret;
%r = call <4 x float> @llvm.convert.from.arbitrary.fp.v4f32.v4i4(<4 x i4> %x, metadata !"Float4E2M1FN")
ret <4 x float> %r
@@ -743,7 +752,7 @@ define <2 x float> @from_f8e4m3fn_v2f32(<2 x i8> %x) {
; CHECK: {
; CHECK-NEXT: .reg .pred %p<11>;
; CHECK-NEXT: .reg .b16 %rs<3>;
-; CHECK-NEXT: .reg .b32 %r<54>;
+; CHECK-NEXT: .reg .b32 %r<56>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
; CHECK-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [from_f8e4m3fn_v2f32_param_0];
@@ -762,55 +771,57 @@ define <2 x float> @from_f8e4m3fn_v2f32(<2 x i8> %x) {
; CHECK-NEXT: add.s32 %r13, %r5, -8;
; CHECK-NEXT: shl.b32 %r14, %r12, %r13;
; CHECK-NEXT: or.b32 %r15, %r8, %r14;
-; CHECK-NEXT: bfe.u32 %r16, %r1, 3, 4;
-; CHECK-NEXT: shl.b32 %r17, %r16, 23;
-; CHECK-NEXT: or.b32 %r18, %r17, %r3;
-; CHECK-NEXT: shl.b32 %r19, %r4, 20;
-; CHECK-NEXT: or.b32 %r20, %r18, %r19;
-; CHECK-NEXT: add.s32 %r21, %r20, 1006632960;
+; CHECK-NEXT: shr.u32 %r16, %r1, 3;
+; CHECK-NEXT: and.b32 %r17, %r16, 15;
+; CHECK-NEXT: shl.b32 %r18, %r17, 23;
+; CHECK-NEXT: or.b32 %r19, %r18, %r3;
+; CHECK-NEXT: shl.b32 %r20, %r4, 20;
+; CHECK-NEXT: or.b32 %r21, %r19, %r20;
+; CHECK-NEXT: add.s32 %r22, %r21, 1006632960;
; CHECK-NEXT: setp.ne.b32 %p1, %r4, 0;
-; CHECK-NEXT: selp.b32 %r22, %r15, %r21, %p1;
-; CHECK-NEXT: setp.eq.b32 %p2, %r16, 0;
-; CHECK-NEXT: selp.b32 %r23, %r22, %r21, %p2;
-; CHECK-NEXT: or.b32 %r24, %r16, %r4;
-; CHECK-NEXT: setp.eq.b32 %p3, %r24, 0;
-; CHECK-NEXT: selp.b32 %r25, %r3, %r23, %p3;
+; CHECK-NEXT: selp.b32 %r23, %r15, %r22, %p1;
+; CHECK-NEXT: setp.eq.b32 %p2, %r17, 0;
+; CHECK-NEXT: selp.b32 %r24, %r23, %r22, %p2;
+; CHECK-NEXT: or.b32 %r25, %r17, %r4;
+; CHECK-NEXT: setp.eq.b32 %p3, %r25, 0;
+; CHECK-NEXT: selp.b32 %r26, %r3, %r24, %p3;
; CHECK-NEXT: setp.eq.b32 %p4, %r4, 7;
-; CHECK-NEXT: selp.b32 %r26, 2143289344, %r25, %p4;
-; CHECK-NEXT: setp.eq.b32 %p5, %r16, 15;
-; CHECK-NEXT: selp.b32 %r27, %r26, %r25, %p5;
-; CHECK-NEXT: cvt.u32.u16 %r28, %rs1;
-; CHECK-NEXT: shl.b32 %r29, %r28, 24;
-; CHECK-NEXT: and.b32 %r30, %r29, -2147483648;
-; CHECK-NEXT: and.b32 %r31, %r28, 7;
-; CHECK-NEXT: clz.b32 %r32, %r31;
-; CHECK-NEXT: sub.s32 %r33, 149, %r32;
-; CHECK-NEXT: shl.b32 %r34, %r33, 23;
-; CHECK-NEXT: or.b32 %r35, %r30, %r34;
-; CHECK-NEXT: sub.s32 %r36, 31, %r32;
-; CHECK-NEXT: shl.b32 %r37, %r10, %r36;
-; CHECK-NEXT: xor.b32 %r38, %r31, %r37;
-; CHECK-NEXT: add.s32 %r39, %r32, -8;
-; CHECK-NEXT: shl.b32 %r40, %r38, %r39;
-; CHECK-NEXT: or.b32 %r41, %r35, %r40;
-; CHECK-NEXT: bfe.u32 %r42, %r28, 3, 4;
-; CHECK-NEXT: shl.b32 %r43, %r42, 23;
-; CHECK-NEXT: or.b32 %r44, %r43, %r30;
-; CHECK-NEXT: shl.b32 %r45, %r31, 20;
-; CHECK-NEXT: or.b32 %r46, %r44, %r45;
-; CHECK-NEXT: add.s32 %r47, %r46, 1006632960;
-; CHECK-NEXT: setp.ne.b32 %p6, %r31, 0;
-; CHECK-NEXT: selp.b32 %r48, %r41, %r47, %p6;
-; CHECK-NEXT: setp.eq.b32 %p7, %r42, 0;
-; CHECK-NEXT: selp.b32 %r49, %r48, %r47, %p7;
-; CHECK-NEXT: or.b32 %r50, %r42, %r31;
-; CHECK-NEXT: setp.eq.b32 %p8, %r50, 0;
-; CHECK-NEXT: selp.b32 %r51, %r30, %r49, %p8;
-; CHECK-NEXT: setp.eq.b32 %p9, %r31, 7;
-; CHECK-NEXT: selp.b32 %r52, 2143289344, %r51, %p9;
-; CHECK-NEXT: setp.eq.b32 %p10, %r42, 15;
-; CHECK-NEXT: selp.b32 %r53, %r52, %r51, %p10;
-; CHECK-NEXT: st.param.v2.b32 [func_retval0], {%r53, %r27};
+; CHECK-NEXT: selp.b32 %r27, 2143289344, %r26, %p4;
+; CHECK-NEXT: setp.eq.b32 %p5, %r17, 15;
+; CHECK-NEXT: selp.b32 %r28, %r27, %r26, %p5;
+; CHECK-NEXT: cvt.u32.u16 %r29, %rs1;
+; CHECK-NEXT: shl.b32 %r30, %r29, 24;
+; CHECK-NEXT: and.b32 %r31, %r30, -2147483648;
+; CHECK-NEXT: and.b32 %r32, %r29, 7;
+; CHECK-NEXT: clz.b32 %r33, %r32;
+; CHECK-NEXT: sub.s32 %r34, 149, %r33;
+; CHECK-NEXT: shl.b32 %r35, %r34, 23;
+; CHECK-NEXT: or.b32 %r36, %r31, %r35;
+; CHECK-NEXT: sub.s32 %r37, 31, %r33;
+; CHECK-NEXT: shl.b32 %r38, %r10, %r37;
+; CHECK-NEXT: xor.b32 %r39, %r32, %r38;
+; CHECK-NEXT: add.s32 %r40, %r33, -8;
+; CHECK-NEXT: shl.b32 %r41, %r39, %r40;
+; CHECK-NEXT: or.b32 %r42, %r36, %r41;
+; CHECK-NEXT: shr.u32 %r43, %r29, 3;
+; CHECK-NEXT: and.b32 %r44, %r43, 15;
+; CHECK-NEXT: shl.b32 %r45, %r44, 23;
+; CHECK-NEXT: or.b32 %r46, %r45, %r31;
+; CHECK-NEXT: shl.b32 %r47, %r32, 20;
+; CHECK-NEXT: or.b32 %r48, %r46, %r47;
+; CHECK-NEXT: add.s32 %r49, %r48, 1006632960;
+; CHECK-NEXT: setp.ne.b32 %p6, %r32, 0;
+; CHECK-NEXT: selp.b32 %r50, %r42, %r49, %p6;
+; CHECK-NEXT: setp.eq.b32 %p7, %r44, 0;
+; CHECK-NEXT: selp.b32 %r51, %r50, %r49, %p7;
+; CHECK-NEXT: or.b32 %r52, %r44, %r32;
+; CHECK-NEXT: setp.eq.b32 %p8, %r52, 0;
+; CHECK-NEXT: selp.b32 %r53, %r31, %r51, %p8;
+; CHECK-NEXT: setp.eq.b32 %p9, %r32, 7;
+; CHECK-NEXT: selp.b32 %r54, 2143289344, %r53, %p9;
+; CHECK-NEXT: setp.eq.b32 %p10, %r44, 15;
+; CHECK-NEXT: selp.b32 %r55, %r54, %r53, %p10;
+; CHECK-NEXT: st.param.v2.b32 [func_retval0], {%r55, %r28};
; CHECK-NEXT: ret;
%r = call <2 x float> @llvm.convert.from.arbitrary.fp.v2f32.v2i8(<2 x i8> %x, metadata !"Float8E4M3FN")
ret <2 x float> %r
@@ -822,7 +833,7 @@ define <2 x float> @from_f8e5m2_v2f32(<2 x i8> %x) {
; CHECK: {
; CHECK-NEXT: .reg .pred %p<9>;
; CHECK-NEXT: .reg .b16 %rs<3>;
-; CHECK-NEXT: .reg .b32 %r<60>;
+; CHECK-NEXT: .reg .b32 %r<62>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
; CHECK-NEXT: ld.param.v2.b8 {%rs1, %rs2}, [from_f8e5m2_v2f32_param_0];
@@ -841,59 +852,61 @@ define <2 x float> @from_f8e5m2_v2f32(<2 x i8> %x) {
; CHECK-NEXT: add.s32 %r13, %r5, -8;
; CHECK-NEXT: shl.b32 %r14, %r12, %r13;
; CHECK-NEXT: or.b32 %r15, %r8, %r14;
-; CHECK-NEXT: bfe.u32 %r16, %r1, 2, 5;
-; CHECK-NEXT: shl.b32 %r17, %r16, 23;
-; CHECK-NEXT: or.b32 %r18, %r17, %r3;
-; CHECK-NEXT: shl.b32 %r19, %r4, 21;
-; CHECK-NEXT: or.b32 %r20, %r18, %r19;
-; CHECK-NEXT: add.s32 %r21, %r20, 939524096;
+; CHECK-NEXT: shr.u32 %r16, %r1, 2;
+; CHECK-NEXT: and.b32 %r17, %r16, 31;
+; CHECK-NEXT: shl.b32 %r18, %r17, 23;
+; CHECK-NEXT: or.b32 %r19, %r18, %r3;
+; CHECK-NEXT: shl.b32 %r20, %r4, 21;
+; CHECK-NEXT: or.b32 %r21, %r19, %r20;
+; CHECK-NEXT: add.s32 %r22, %r21, 939524096;
; CHECK-NEXT: setp.ne.b32 %p1, %r4, 0;
-; CHECK-NEXT: selp.b32 %r22, %r15, %r21, %p1;
-; CHECK-NEXT: setp.eq.b32 %p2, %r16, 0;
-; CHECK-NEXT: selp.b32 %r23, %r22, %r21, %p2;
-; CHECK-NEXT: or.b32 %r24, %r16, %r4;
-; CHECK-NEXT: setp.eq.b32 %p3, %r24, 0;
-; CHECK-NEXT: selp.b32 %r25, %r3, %r23, %p3;
-; CHECK-NEXT: or.b32 %r26, %r3, 2139095040;
-; CHECK-NEXT: selp.b32 %r27, %r25, %r26, %p1;
-; CHECK-NEXT: setp.eq.b32 %p4, %r16, 31;
-; CHECK-NEXT: selp.b32 %r28, %r27, %r25, %p4;
-; CHECK-NEXT: selp.b32 %r29, 2143289344, %r28, %p1;
-; CHECK-NEXT: selp.b32 %r30, %r29, %r28, %p4;
-; CHECK-NEXT: cvt.u32.u16 %r31, %rs1;
-; CHECK-NEXT: shl.b32 %r32, %r31, 24;
-; CHECK-NEXT: and.b32 %r33, %r32, -2147483648;
-; CHECK-NEXT: and.b32 %r34, %r31, 3;
-; CHECK-NEXT: clz.b32 %r35, %r34;
-; CHECK-NEXT: sub.s32 %r36, 142, %r35;
-; CHECK-NEXT: shl.b32 %r37, %r36, 23;
-; CHECK-NEXT: or.b32 %r38, %r33, %r37;
-; CHECK-NEXT: sub.s32 %r39, 31, %r35;
-; CHECK-NEXT: shl.b32 %r40, %r10, %r39;
-; CHECK-NEXT: xor.b32 %r41, %r34, %r40;
-; CHECK-NEXT: add.s32 %r42, %r35, -8;
-; CHECK-NEXT: shl.b32 %r43, %r41, %r42;
-; CHECK-NEXT: or.b32 %r44, %r38, %r43;
-; CHECK-NEXT: bfe.u32 %r45, %r31, 2, 5;
-; CHECK-NEXT: shl.b32 %r46, %r45, 23;
-; CHECK-NEXT: or.b32 %r47, %r46, %r33;
-; CHECK-NEXT: shl.b32 %r48, %r34, 21;
-; CHECK-NEXT: or.b32 %r49, %r47, %r48;
-; CHECK-NEXT: add.s32 %r50, %r49, 939524096;
-; CHECK-NEXT: setp.ne.b32 %p5, %r34, 0;
-; CHECK-NEXT: selp.b32 %r51, %r44, %r50, %p5;
-; CHECK-NEXT: setp.eq.b32 %p6, %r45, 0;
-; CHECK-NEXT: selp.b32 %r52, %r51, %r50, %p6;
-; CHECK-NEXT: or.b32 %r53, %r45, %r34;
-; CHECK-NEXT: setp.eq.b32 %p7, %r53, 0;
-; CHECK-NEXT: selp.b32 %r54, %r33, %r52, %p7;
-; CHECK-NEXT: or.b32 %r55, %r33, 2139095040;
-; CHECK-NEXT: selp.b32 %r56, %r54, %r55, %p5;
-; CHECK-NEXT: setp.eq.b32 %p8, %r45, 31;
-; CHECK-NEXT: selp.b32 %r57, %r56, %r54, %p8;
-; CHECK-NEXT: selp.b32 %r58, 2143289344, %r57, %p5;
-; CHECK-NEXT: selp.b32 %r59, %r58, %r57, %p8;
-; CHECK-NEXT: st.param.v2.b32 [func_retval0], {%r59, %r30};
+; CHECK-NEXT: selp.b32 %r23, %r15, %r22, %p1;
+; CHECK-NEXT: setp.eq.b32 %p2, %r17, 0;
+; CHECK-NEXT: selp.b32 %r24, %r23, %r22, %p2;
+; CHECK-NEXT: or.b32 %r25, %r17, %r4;
+; CHECK-NEXT: setp.eq.b32 %p3, %r25, 0;
+; CHECK-NEXT: selp.b32 %r26, %r3, %r24, %p3;
+; CHECK-NEXT: or.b32 %r27, %r3, 2139095040;
+; CHECK-NEXT: selp.b32 %r28, %r26, %r27, %p1;
+; CHECK-NEXT: setp.eq.b32 %p4, %r17, 31;
+; CHECK-NEXT: selp.b32 %r29, %r28, %r26, %p4;
+; CHECK-NEXT: selp.b32 %r30, 2143289344, %r29, %p1;
+; CHECK-NEXT: selp.b32 %r31, %r30, %r29, %p4;
+; CHECK-NEXT: cvt.u32.u16 %r32, %rs1;
+; CHECK-NEXT: shl.b32 %r33, %r32, 24;
+; CHECK-NEXT: and.b32 %r34, %r33, -2147483648;
+; CHECK-NEXT: and.b32 %r35, %r32, 3;
+; CHECK-NEXT: clz.b32 %r36, %r35;
+; CHECK-NEXT: sub.s32 %r37, 142, %r36;
+; CHECK-NEXT: shl.b32 %r38, %r37, 23;
+; CHECK-NEXT: or.b32 %r39, %r34, %r38;
+; CHECK-NEXT: sub.s32 %r40, 31, %r36;
+; CHECK-NEXT: shl.b32 %r41, %r10, %r40;
+; CHECK-NEXT: xor.b32 %r42, %r35, %r41;
+; CHECK-NEXT: add.s32 %r43, %r36, -8;
+; CHECK-NEXT: shl.b32 %r44, %r42, %r43;
+; CHECK-NEXT: or.b32 %r45, %r39, %r44;
+; CHECK-NEXT: shr.u32 %r46, %r32, 2;
+; CHECK-NEXT: and.b32 %r47, %r46, 31;
+; CHECK-NEXT: shl.b32 %r48, %r47, 23;
+; CHECK-NEXT: or.b32 %r49, %r48, %r34;
+; CHECK-NEXT: shl.b32 %r50, %r35, 21;
+; CHECK-NEXT: or.b32 %r51, %r49, %r50;
+; CHECK-NEXT: add.s32 %r52, %r51, 939524096;
+; CHECK-NEXT: setp.ne.b32 %p5, %r35, 0;
+; CHECK-NEXT: selp.b32 %r53, %r45, %r52, %p5;
+; CHECK-NEXT: setp.eq.b32 %p6, %r47, 0;
+; CHECK-NEXT: selp.b32 %r54, %r53, %r52, %p6;
+; CHECK-NEXT: or.b32 %r55, %r47, %r35;
+; CHECK-NEXT: setp.eq.b32 %p7, %r55, 0;
+; CHECK-NEXT: selp.b32 %r56, %r34, %r54, %p7;
+; CHECK-NEXT: or.b32 %r57, %r34, 2139095040;
+; CHECK-NEXT: selp.b32 %r58, %r56, %r57, %p5;
+; CHECK-NEXT: setp.eq.b32 %p8, %r47, 31;
+; CHECK-NEXT: selp.b32 %r59, %r58, %r56, %p8;
+; CHECK-NEXT: selp.b32 %r60, 2143289344, %r59, %p5;
+; CHECK-NEXT: selp.b32 %r61, %r60, %r59, %p8;
+; CHECK-NEXT: st.param.v2.b32 [func_retval0], {%r61, %r31};
; CHECK-NEXT: ret;
%r = call <2 x float> @llvm.convert.from.arbitrary.fp.v2f32.v2i8(<2 x i8> %x, metadata !"Float8E5M2")
ret <2 x float> %r
@@ -904,7 +917,7 @@ define <3 x float> @from_f8e4m3fn_v3f32(<3 x i8> %x) {
; CHECK-LABEL: from_f8e4m3fn_v3f32(
; CHECK: {
; CHECK-NEXT: .reg .pred %p<16>;
-; CHECK-NEXT: .reg .b32 %r<81>;
+; CHECK-NEXT: .reg .b32 %r<84>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
; CHECK-NEXT: ld.param.b32 %r1, [from_f8e4m3fn_v3f32_param_0];
@@ -923,87 +936,90 @@ define <3 x float> @from_f8e4m3fn_v3f32(<3 x i8> %x) {
; CHECK-NEXT: add.s32 %r14, %r6, -8;
; CHECK-NEXT: shl.b32 %r15, %r13, %r14;
; CHECK-NEXT: or.b32 %r16, %r9, %r15;
-; CHECK-NEXT: bfe.u32 %r17, %r2, 3, 4;
-; CHECK-NEXT: shl.b32 %r18, %r17, 23;
-; CHECK-NEXT: or.b32 %r19, %r18, %r4;
-; CHECK-NEXT: shl.b32 %r20, %r5, 20;
-; CHECK-NEXT: or.b32 %r21, %r19, %r20;
-; CHECK-NEXT: add.s32 %r22, %r21, 1006632960;
+; CHECK-NEXT: shr.u32 %r17, %r2, 3;
+; CHECK-NEXT: and.b32 %r18, %r17, 15;
+; CHECK-NEXT: shl.b32 %r19, %r18, 23;
+; CHECK-NEXT: or.b32 %r20, %r19, %r4;
+; CHECK-NEXT: shl.b32 %r21, %r5, 20;
+; CHECK-NEXT: or.b32 %r22, %r20, %r21;
+; CHECK-NEXT: add.s32 %r23, %r22, 1006632960;
; CHECK-NEXT: setp.ne.b32 %p1, %r5, 0;
-; CHECK-NEXT: selp.b32 %r23, %r16, %r22, %p1;
-; CHECK-NEXT: setp.eq.b32 %p2, %r17, 0;
-; CHECK-NEXT: selp.b32 %r24, %r23, %r22, %p2;
-; CHECK-NEXT: or.b32 %r25, %r17, %r5;
-; CHECK-NEXT: setp.eq.b32 %p3, %r25, 0;
-; CHECK-NEXT: selp.b32 %r26, %r4, %r24, %p3;
+; CHECK-NEXT: selp.b32 %r24, %r16, %r23, %p1;
+; CHECK-NEXT: setp.eq.b32 %p2, %r18, 0;
+; CHECK-NEXT: selp.b32 %r25, %r24, %r23, %p2;
+; CHECK-NEXT: or.b32 %r26, %r18, %r5;
+; CHECK-NEXT: setp.eq.b32 %p3, %r26, 0;
+; CHECK-NEXT: selp.b32 %r27, %r4, %r25, %p3;
; CHECK-NEXT: setp.eq.b32 %p4, %r5, 7;
-; CHECK-NEXT: selp.b32 %r27, 2143289344, %r26, %p4;
-; CHECK-NEXT: setp.eq.b32 %p5, %r17, 15;
-; CHECK-NEXT: selp.b32 %r28, %r27, %r26, %p5;
-; CHECK-NEXT: prmt.b32 %r29, %r1, 0, 0x7771U;
-; CHECK-NEXT: shl.b32 %r30, %r29, 24;
-; CHECK-NEXT: and.b32 %r31, %r30, -2147483648;
-; CHECK-NEXT: and.b32 %r32, %r29, 7;
-; CHECK-NEXT: clz.b32 %r33, %r32;
-; CHECK-NEXT: sub.s32 %r34, 149, %r33;
-; CHECK-NEXT: shl.b32 %r35, %r34, 23;
-; CHECK-NEXT: or.b32 %r36, %r31, %r35;
-; CHECK-NEXT: sub.s32 %r37, 31, %r33;
-; CHECK-NEXT: shl.b32 %r38, %r11, %r37;
-; CHECK-NEXT: xor.b32 %r39, %r32, %r38;
-; CHECK-NEXT: add.s32 %r40, %r33, -8;
-; CHECK-NEXT: shl.b32 %r41, %r39, %r40;
-; CHECK-NEXT: or.b32 %r42, %r36, %r41;
-; CHECK-NEXT: bfe.u32 %r43, %r29, 3, 4;
-; CHECK-NEXT: shl.b32 %r44, %r43, 23;
-; CHECK-NEXT: or.b32 %r45, %r44, %r31;
-; CHECK-NEXT: shl.b32 %r46, %r32, 20;
-; CHECK-NEXT: or.b32 %r47, %r45, %r46;
-; CHECK-NEXT: add.s32 %r48, %r47, 1006632960;
-; CHECK-NEXT: setp.ne.b32 %p6, %r32, 0;
-; CHECK-NEXT: selp.b32 %r49, %r42, %r48, %p6;
-; CHECK-NEXT: setp.eq.b32 %p7, %r43, 0;
-; CHECK-NEXT: selp.b32 %r50, %r49, %r48, %p7;
-; CHECK-NEXT: or.b32 %r51, %r43, %r32;
-; CHECK-NEXT: setp.eq.b32 %p8, %r51, 0;
-; CHECK-NEXT: selp.b32 %r52, %r31, %r50, %p8;
-; CHECK-NEXT: setp.eq.b32 %p9, %r32, 7;
-; CHECK-NEXT: selp.b32 %r53, 2143289344, %r52, %p9;
-; CHECK-NEXT: setp.eq.b32 %p10, %r43, 15;
-; CHECK-NEXT: selp.b32 %r54, %r53, %r52, %p10;
-; CHECK-NEXT: prmt.b32 %r55, %r1, 0, 0x7770U;
-; CHECK-NEXT: shl.b32 %r56, %r55, 24;
-; CHECK-NEXT: and.b32 %r57, %r56, -2147483648;
-; CHECK-NEXT: and.b32 %r58, %r55, 7;
-; CHECK-NEXT: clz.b32 %r59, %r58;
-; CHECK-NEXT: sub.s32 %r60, 149, %r59;
-; CHECK-NEXT: shl.b32 %r61, %r60, 23;
-; CHECK-NEXT: or.b32 %r62, %r57, %r61;
-; CHECK-NEXT: sub.s32 %r63, 31, %r59;
-; CHECK-NEXT: shl.b32 %r64, %r11, %r63;
-; CHECK-NEXT: xor.b32 %r65, %r58, %r64;
-; CHECK-NEXT: add.s32 %r66, %r59, -8;
-; CHECK-NEXT: shl.b32 %r67, %r65, %r66;
-; CHECK-NEXT: or.b32 %r68, %r62, %r67;
-; CHECK-NEXT: bfe.u32 %r69, %r55, 3, 4;
-; CHECK-NEXT: shl.b32 %r70, %r69, 23;
-; CHECK-NEXT: or.b32 %r71, %r70, %r57;
-; CHECK-NEXT: shl.b32 %r72, %r58, 20;
-; CHECK-NEXT: or.b32 %r73, %r71, %r72;
-; CHECK-NEXT: add.s32 %r74, %r73, 1006632960;
-; CHECK-NEXT: setp.ne.b32 %p11, %r58, 0;
-; CHECK-NEXT: selp.b32 %r75, %r68, %r74, %p11;
-; CHECK-NEXT: setp.eq.b32 %p12, %r69, 0;
-; CHECK-NEXT: selp.b32 %r76, %r75, %r74, %p12;
-; CHECK-NEXT: or.b32 %r77, %r69, %r58;
-; CHECK-NEXT: setp.eq.b32 %p13, %r77, 0;
-; CHECK-NEXT: selp.b32 %r78, %r57, %r76, %p13;
-; CHECK-NEXT: setp.eq.b32 %p14, %r58, 7;
-; CHECK-NEXT: selp.b32 %r79, 2143289344, %r78, %p14;
-; CHECK-NEXT: setp.eq.b32 %p15, %r69, 15;
-; CHECK-NEXT: selp.b32 %r80, %r79, %r78, %p15;
-; CHECK-NEXT: st.param.v2.b32 [func_retval0], {%r80, %r54};
-; CHECK-NEXT: st.param.b32 [func_retval0+8], %r28;
+; CHECK-NEXT: selp.b32 %r28, 2143289344, %r27, %p4;
+; CHECK-NEXT: setp.eq.b32 %p5, %r18, 15;
+; CHECK-NEXT: selp.b32 %r29, %r28, %r27, %p5;
+; CHECK-NEXT: prmt.b32 %r30, %r1, 0, 0x7771U;
+; CHECK-NEXT: shl.b32 %r31, %r30, 24;
+; CHECK-NEXT: and.b32 %r32, %r31, -2147483648;
+; CHECK-NEXT: and.b32 %r33, %r30, 7;
+; CHECK-NEXT: clz.b32 %r34, %r33;
+; CHECK-NEXT: sub.s32 %r35, 149, %r34;
+; CHECK-NEXT: shl.b32 %r36, %r35, 23;
+; CHECK-NEXT: or.b32 %r37, %r32, %r36;
+; CHECK-NEXT: sub.s32 %r38, 31, %r34;
+; CHECK-NEXT: shl.b32 %r39, %r11, %r38;
+; CHECK-NEXT: xor.b32 %r40, %r33, %r39;
+; CHECK-NEXT: add.s32 %r41, %r34, -8;
+; CHECK-NEXT: shl.b32 %r42, %r40, %r41;
+; CHECK-NEXT: or.b32 %r43, %r37, %r42;
+; CHECK-NEXT: shr.u32 %r44, %r30, 3;
+; CHECK-NEXT: and.b32 %r45, %r44, 15;
+; CHECK-NEXT: shl.b32 %r46, %r45, 23;
+; CHECK-NEXT: or.b32 %r47, %r46, %r32;
+; CHECK-NEXT: shl.b32 %r48, %r33, 20;
+; CHECK-NEXT: or.b32 %r49, %r47, %r48;
+; CHECK-NEXT: add.s32 %r50, %r49, 1006632960;
+; CHECK-NEXT: setp.ne.b32 %p6, %r33, 0;
+; CHECK-NEXT: selp.b32 %r51, %r43, %r50, %p6;
+; CHECK-NEXT: setp.eq.b32 %p7, %r45, 0;
+; CHECK-NEXT: selp.b32 %r52, %r51, %r50, %p7;
+; CHECK-NEXT: or.b32 %r53, %r45, %r33;
+; CHECK-NEXT: setp.eq.b32 %p8, %r53, 0;
+; CHECK-NEXT: selp.b32 %r54, %r32, %r52, %p8;
+; CHECK-NEXT: setp.eq.b32 %p9, %r33, 7;
+; CHECK-NEXT: selp.b32 %r55, 2143289344, %r54, %p9;
+; CHECK-NEXT: setp.eq.b32 %p10, %r45, 15;
+; CHECK-NEXT: selp.b32 %r56, %r55, %r54, %p10;
+; CHECK-NEXT: prmt.b32 %r57, %r1, 0, 0x7770U;
+; CHECK-NEXT: shl.b32 %r58, %r57, 24;
+; CHECK-NEXT: and.b32 %r59, %r58, -2147483648;
+; CHECK-NEXT: and.b32 %r60, %r57, 7;
+; CHECK-NEXT: clz.b32 %r61, %r60;
+; CHECK-NEXT: sub.s32 %r62, 149, %r61;
+; CHECK-NEXT: shl.b32 %r63, %r62, 23;
+; CHECK-NEXT: or.b32 %r64, %r59, %r63;
+; CHECK-NEXT: sub.s32 %r65, 31, %r61;
+; CHECK-NEXT: shl.b32 %r66, %r11, %r65;
+; CHECK-NEXT: xor.b32 %r67, %r60, %r66;
+; CHECK-NEXT: add.s32 %r68, %r61, -8;
+; CHECK-NEXT: shl.b32 %r69, %r67, %r68;
+; CHECK-NEXT: or.b32 %r70, %r64, %r69;
+; CHECK-NEXT: shr.u32 %r71, %r57, 3;
+; CHECK-NEXT: and.b32 %r72, %r71, 15;
+; CHECK-NEXT: shl.b32 %r73, %r72, 23;
+; CHECK-NEXT: or.b32 %r74, %r73, %r59;
+; CHECK-NEXT: shl.b32 %r75, %r60, 20;
+; CHECK-NEXT: or.b32 %r76, %r74, %r75;
+; CHECK-NEXT: add.s32 %r77, %r76, 1006632960;
+; CHECK-NEXT: setp.ne.b32 %p11, %r60, 0;
+; CHECK-NEXT: selp.b32 %r78, %r70, %r77, %p11;
+; CHECK-NEXT: setp.eq.b32 %p12, %r72, 0;
+; CHECK-NEXT: selp.b32 %r79, %r78, %r77, %p12;
+; CHECK-NEXT: or.b32 %r80, %r72, %r60;
+; CHECK-NEXT: setp.eq.b32 %p13, %r80, 0;
+; CHECK-NEXT: selp.b32 %r81, %r59, %r79, %p13;
+; CHECK-NEXT: setp.eq.b32 %p14, %r60, 7;
+; CHECK-NEXT: selp.b32 %r82, 2143289344, %r81, %p14;
+; CHECK-NEXT: setp.eq.b32 %p15, %r72, 15;
+; CHECK-NEXT: selp.b32 %r83, %r82, %r81, %p15;
+; CHECK-NEXT: st.param.v2.b32 [func_retval0], {%r83, %r56};
+; CHECK-NEXT: st.param.b32 [func_retval0+8], %r29;
; CHECK-NEXT: ret;
%r = call <3 x float> @llvm.convert.from.arbitrary.fp.v3f32.v3i8(<3 x i8> %x, metadata !"Float8E4M3FN")
ret <3 x float> %r
@@ -1014,7 +1030,7 @@ define <4 x float> @from_f8e4m3fn_v4f32(<4 x i8> %x) {
; CHECK-LABEL: from_f8e4m3fn_v4f32(
; CHECK: {
; CHECK-NEXT: .reg .pred %p<21>;
-; CHECK-NEXT: .reg .b32 %r<107>;
+; CHECK-NEXT: .reg .b32 %r<111>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
; CHECK-NEXT: ld.param.b32 %r1, [from_f8e4m3fn_v4f32_param_0];
@@ -1033,117 +1049,121 @@ define <4 x float> @from_f8e4m3fn_v4f32(<4 x i8> %x) {
; CHECK-NEXT: add.s32 %r14, %r6, -8;
; CHECK-NEXT: shl.b32 %r15, %r13, %r14;
; CHECK-NEXT: or.b32 %r16, %r9, %r15;
-; CHECK-NEXT: bfe.u32 %r17, %r2, 3, 4;
-; CHECK-NEXT: shl.b32 %r18, %r17, 23;
-; CHECK-NEXT: or.b32 %r19, %r18, %r4;
-; CHECK-NEXT: shl.b32 %r20, %r5, 20;
-; CHECK-NEXT: or.b32 %r21, %r19, %r20;
-; CHECK-NEXT: add.s32 %r22, %r21, 1006632960;
+; CHECK-NEXT: shr.u32 %r17, %r2, 3;
+; CHECK-NEXT: and.b32 %r18, %r17, 15;
+; CHECK-NEXT: shl.b32 %r19, %r18, 23;
+; CHECK-NEXT: or.b32 %r20, %r19, %r4;
+; CHECK-NEXT: shl.b32 %r21, %r5, 20;
+; CHECK-NEXT: or.b32 %r22, %r20, %r21;
+; CHECK-NEXT: add.s32 %r23, %r22, 1006632960;
; CHECK-NEXT: setp.ne.b32 %p1, %r5, 0;
-; CHECK-NEXT: selp.b32 %r23, %r16, %r22, %p1;
-; CHECK-NEXT: setp.eq.b32 %p2, %r17, 0;
-; CHECK-NEXT: selp.b32 %r24, %r23, %r22, %p2;
-; CHECK-NEXT: or.b32 %r25, %r17, %r5;
-; CHECK-NEXT: setp.eq.b32 %p3, %r25, 0;
-; CHECK-NEXT: selp.b32 %r26, %r4, %r24, %p3;
+; CHECK-NEXT: selp.b32 %r24, %r16, %r23, %p1;
+; CHECK-NEXT: setp.eq.b32 %p2, %r18, 0;
+; CHECK-NEXT: selp.b32 %r25, %r24, %r23, %p2;
+; CHECK-NEXT: or.b32 %r26, %r18, %r5;
+; CHECK-NEXT: setp.eq.b32 %p3, %r26, 0;
+; CHECK-NEXT: selp.b32 %r27, %r4, %r25, %p3;
; CHECK-NEXT: setp.eq.b32 %p4, %r5, 7;
-; CHECK-NEXT: selp.b32 %r27, 2143289344, %r26, %p4;
-; CHECK-NEXT: setp.eq.b32 %p5, %r17, 15;
-; CHECK-NEXT: selp.b32 %r28, %r27, %r26, %p5;
-; CHECK-NEXT: prmt.b32 %r29, %r1, 0, 0x7772U;
-; CHECK-NEXT: shl.b32 %r30, %r29, 24;
-; CHECK-NEXT: and.b32 %r31, %r30, -2147483648;
-; CHECK-NEXT: and.b32 %r32, %r29, 7;
-; CHECK-NEXT: clz.b32 %r33, %r32;
-; CHECK-NEXT: sub.s32 %r34, 149, %r33;
-; CHECK-NEXT: shl.b32 %r35, %r34, 23;
-; CHECK-NEXT: or.b32 %r36, %r31, %r35;
-; CHECK-NEXT: sub.s32 %r37, 31, %r33;
-; CHECK-NEXT: shl.b32 %r38, %r11, %r37;
-; CHECK-NEXT: xor.b32 %r39, %r32, %r38;
-; CHECK-NEXT: add.s32 %r40, %r33, -8;
-; CHECK-NEXT: shl.b32 %r41, %r39, %r40;
-; CHECK-NEXT: or.b32 %r42, %r36, %r41;
-; CHECK-NEXT: bfe.u32 %r43, %r29, 3, 4;
-; CHECK-NEXT: shl.b32 %r44, %r43, 23;
-; CHECK-NEXT: or.b32 %r45, %r44, %r31;
-; CHECK-NEXT: shl.b32 %r46, %r32, 20;
-; CHECK-NEXT: or.b32 %r47, %r45, %r46;
-; CHECK-NEXT: add.s32 %r48, %r47, 1006632960;
-; CHECK-NEXT: setp.ne.b32 %p6, %r32, 0;
-; CHECK-NEXT: selp.b32 %r49, %r42, %r48, %p6;
-; CHECK-NEXT: setp.eq.b32 %p7, %r43, 0;
-; CHECK-NEXT: selp.b32 %r50, %r49, %r48, %p7;
-; CHECK-NEXT: or.b32 %r51, %r43, %r32;
-; CHECK-NEXT: setp.eq.b32 %p8, %r51, 0;
-; CHECK-NEXT: selp.b32 %r52, %r31, %r50, %p8;
-; CHECK-NEXT: setp.eq.b32 %p9, %r32, 7;
-; CHECK-NEXT: selp.b32 %r53, 2143289344, %r52, %p9;
-; CHECK-NEXT: setp.eq.b32 %p10, %r43, 15;
-; CHECK-NEXT: selp.b32 %r54, %r53, %r52, %p10;
-; CHECK-NEXT: prmt.b32 %r55, %r1, 0, 0x7771U;
-; CHECK-NEXT: shl.b32 %r56, %r55, 24;
-; CHECK-NEXT: and.b32 %r57, %r56, -2147483648;
-; CHECK-NEXT: and.b32 %r58, %r55, 7;
-; CHECK-NEXT: clz.b32 %r59, %r58;
-; CHECK-NEXT: sub.s32 %r60, 149, %r59;
-; CHECK-NEXT: shl.b32 %r61, %r60, 23;
-; CHECK-NEXT: or.b32 %r62, %r57, %r61;
-; CHECK-NEXT: sub.s32 %r63, 31, %r59;
-; CHECK-NEXT: shl.b32 %r64, %r11, %r63;
-; CHECK-NEXT: xor.b32 %r65, %r58, %r64;
-; CHECK-NEXT: add.s32 %r66, %r59, -8;
-; CHECK-NEXT: shl.b32 %r67, %r65, %r66;
-; CHECK-NEXT: or.b32 %r68, %r62, %r67;
-; CHECK-NEXT: bfe.u32 %r69, %r55, 3, 4;
-; CHECK-NEXT: shl.b32 %r70, %r69, 23;
-; CHECK-NEXT: or.b32 %r71, %r70, %r57;
-; CHECK-NEXT: shl.b32 %r72, %r58, 20;
-; CHECK-NEXT: or.b32 %r73, %r71, %r72;
-; CHECK-NEXT: add.s32 %r74, %r73, 1006632960;
-; CHECK-NEXT: setp.ne.b32 %p11, %r58, 0;
-; CHECK-NEXT: selp.b32 %r75, %r68, %r74, %p11;
-; CHECK-NEXT: setp.eq.b32 %p12, %r69, 0;
-; CHECK-NEXT: selp.b32 %r76, %r75, %r74, %p12;
-; CHECK-NEXT: or.b32 %r77, %r69, %r58;
-; CHECK-NEXT: setp.eq.b32 %p13, %r77, 0;
-; CHECK-NEXT: selp.b32 %r78, %r57, %r76, %p13;
-; CHECK-NEXT: setp.eq.b32 %p14, %r58, 7;
-; CHECK-NEXT: selp.b32 %r79, 2143289344, %r78, %p14;
-; CHECK-NEXT: setp.eq.b32 %p15, %r69, 15;
-; CHECK-NEXT: selp.b32 %r80, %r79, %r78, %p15;
-; CHECK-NEXT: prmt.b32 %r81, %r1, 0, 0x7770U;
-; CHECK-NEXT: shl.b32 %r82, %r81, 24;
-; CHECK-NEXT: and.b32 %r83, %r82, -2147483648;
-; CHECK-NEXT: and.b32 %r84, %r81, 7;
-; CHECK-NEXT: clz.b32 %r85, %r84;
-; CHECK-NEXT: sub.s32 %r86, 149, %r85;
-; CHECK-NEXT: shl.b32 %r87, %r86, 23;
-; CHECK-NEXT: or.b32 %r88, %r83, %r87;
-; CHECK-NEXT: sub.s32 %r89, 31, %r85;
-; CHECK-NEXT: shl.b32 %r90, %r11, %r89;
-; CHECK-NEXT: xor.b32 %r91, %r84, %r90;
-; CHECK-NEXT: add.s32 %r92, %r85, -8;
-; CHECK-NEXT: shl.b32 %r93, %r91, %r92;
-; CHECK-NEXT: or.b32 %r94, %r88, %r93;
-; CHECK-NEXT: bfe.u32 %r95, %r81, 3, 4;
-; CHECK-NEXT: shl.b32 %r96, %r95, 23;
-; CHECK-NEXT: or.b32 %r97, %r96, %r83;
-; CHECK-NEXT: shl.b32 %r98, %r84, 20;
-; CHECK-NEXT: or.b32 %r99, %r97, %r98;
-; CHECK-NEXT: add.s32 %r100, %r99, 1006632960;
-; CHECK-NEXT: setp.ne.b32 %p16, %r84, 0;
-; CHECK-NEXT: selp.b32 %r101, %r94, %r100, %p16;
-; CHECK-NEXT: setp.eq.b32 %p17, %r95, 0;
-; CHECK-NEXT: selp.b32 %r102, %r101, %r100, %p17;
-; CHECK-NEXT: or.b32 %r103, %r95, %r84;
-; CHECK-NEXT: setp.eq.b32 %p18, %r103, 0;
-; CHECK-NEXT: selp.b32 %r104, %r83, %r102, %p18;
-; CHECK-NEXT: setp.eq.b32 %p19, %r84, 7;
-; CHECK-NEXT: selp.b32 %r105, 2143289344, %r104, %p19;
-; CHECK-NEXT: setp.eq.b32 %p20, %r95, 15;
-; CHECK-NEXT: selp.b32 %r106, %r105, %r104, %p20;
-; CHECK-NEXT: st.param.v4.b32 [func_retval0], {%r106, %r80, %r54, %r28};
+; CHECK-NEXT: selp.b32 %r28, 2143289344, %r27, %p4;
+; CHECK-NEXT: setp.eq.b32 %p5, %r18, 15;
+; CHECK-NEXT: selp.b32 %r29, %r28, %r27, %p5;
+; CHECK-NEXT: prmt.b32 %r30, %r1, 0, 0x7772U;
+; CHECK-NEXT: shl.b32 %r31, %r30, 24;
+; CHECK-NEXT: and.b32 %r32, %r31, -2147483648;
+; CHECK-NEXT: and.b32 %r33, %r30, 7;
+; CHECK-NEXT: clz.b32 %r34, %r33;
+; CHECK-NEXT: sub.s32 %r35, 149, %r34;
+; CHECK-NEXT: shl.b32 %r36, %r35, 23;
+; CHECK-NEXT: or.b32 %r37, %r32, %r36;
+; CHECK-NEXT: sub.s32 %r38, 31, %r34;
+; CHECK-NEXT: shl.b32 %r39, %r11, %r38;
+; CHECK-NEXT: xor.b32 %r40, %r33, %r39;
+; CHECK-NEXT: add.s32 %r41, %r34, -8;
+; CHECK-NEXT: shl.b32 %r42, %r40, %r41;
+; CHECK-NEXT: or.b32 %r43, %r37, %r42;
+; CHECK-NEXT: shr.u32 %r44, %r30, 3;
+; CHECK-NEXT: and.b32 %r45, %r44, 15;
+; CHECK-NEXT: shl.b32 %r46, %r45, 23;
+; CHECK-NEXT: or.b32 %r47, %r46, %r32;
+; CHECK-NEXT: shl.b32 %r48, %r33, 20;
+; CHECK-NEXT: or.b32 %r49, %r47, %r48;
+; CHECK-NEXT: add.s32 %r50, %r49, 1006632960;
+; CHECK-NEXT: setp.ne.b32 %p6, %r33, 0;
+; CHECK-NEXT: selp.b32 %r51, %r43, %r50, %p6;
+; CHECK-NEXT: setp.eq.b32 %p7, %r45, 0;
+; CHECK-NEXT: selp.b32 %r52, %r51, %r50, %p7;
+; CHECK-NEXT: or.b32 %r53, %r45, %r33;
+; CHECK-NEXT: setp.eq.b32 %p8, %r53, 0;
+; CHECK-NEXT: selp.b32 %r54, %r32, %r52, %p8;
+; CHECK-NEXT: setp.eq.b32 %p9, %r33, 7;
+; CHECK-NEXT: selp.b32 %r55, 2143289344, %r54, %p9;
+; CHECK-NEXT: setp.eq.b32 %p10, %r45, 15;
+; CHECK-NEXT: selp.b32 %r56, %r55, %r54, %p10;
+; CHECK-NEXT: prmt.b32 %r57, %r1, 0, 0x7771U;
+; CHECK-NEXT: shl.b32 %r58, %r57, 24;
+; CHECK-NEXT: and.b32 %r59, %r58, -2147483648;
+; CHECK-NEXT: and.b32 %r60, %r57, 7;
+; CHECK-NEXT: clz.b32 %r61, %r60;
+; CHECK-NEXT: sub.s32 %r62, 149, %r61;
+; CHECK-NEXT: shl.b32 %r63, %r62, 23;
+; CHECK-NEXT: or.b32 %r64, %r59, %r63;
+; CHECK-NEXT: sub.s32 %r65, 31, %r61;
+; CHECK-NEXT: shl.b32 %r66, %r11, %r65;
+; CHECK-NEXT: xor.b32 %r67, %r60, %r66;
+; CHECK-NEXT: add.s32 %r68, %r61, -8;
+; CHECK-NEXT: shl.b32 %r69, %r67, %r68;
+; CHECK-NEXT: or.b32 %r70, %r64, %r69;
+; CHECK-NEXT: shr.u32 %r71, %r57, 3;
+; CHECK-NEXT: and.b32 %r72, %r71, 15;
+; CHECK-NEXT: shl.b32 %r73, %r72, 23;
+; CHECK-NEXT: or.b32 %r74, %r73, %r59;
+; CHECK-NEXT: shl.b32 %r75, %r60, 20;
+; CHECK-NEXT: or.b32 %r76, %r74, %r75;
+; CHECK-NEXT: add.s32 %r77, %r76, 1006632960;
+; CHECK-NEXT: setp.ne.b32 %p11, %r60, 0;
+; CHECK-NEXT: selp.b32 %r78, %r70, %r77, %p11;
+; CHECK-NEXT: setp.eq.b32 %p12, %r72, 0;
+; CHECK-NEXT: selp.b32 %r79, %r78, %r77, %p12;
+; CHECK-NEXT: or.b32 %r80, %r72, %r60;
+; CHECK-NEXT: setp.eq.b32 %p13, %r80, 0;
+; CHECK-NEXT: selp.b32 %r81, %r59, %r79, %p13;
+; CHECK-NEXT: setp.eq.b32 %p14, %r60, 7;
+; CHECK-NEXT: selp.b32 %r82, 2143289344, %r81, %p14;
+; CHECK-NEXT: setp.eq.b32 %p15, %r72, 15;
+; CHECK-NEXT: selp.b32 %r83, %r82, %r81, %p15;
+; CHECK-NEXT: prmt.b32 %r84, %r1, 0, 0x7770U;
+; CHECK-NEXT: shl.b32 %r85, %r84, 24;
+; CHECK-NEXT: and.b32 %r86, %r85, -2147483648;
+; CHECK-NEXT: and.b32 %r87, %r84, 7;
+; CHECK-NEXT: clz.b32 %r88, %r87;
+; CHECK-NEXT: sub.s32 %r89, 149, %r88;
+; CHECK-NEXT: shl.b32 %r90, %r89, 23;
+; CHECK-NEXT: or.b32 %r91, %r86, %r90;
+; CHECK-NEXT: sub.s32 %r92, 31, %r88;
+; CHECK-NEXT: shl.b32 %r93, %r11, %r92;
+; CHECK-NEXT: xor.b32 %r94, %r87, %r93;
+; CHECK-NEXT: add.s32 %r95, %r88, -8;
+; CHECK-NEXT: shl.b32 %r96, %r94, %r95;
+; CHECK-NEXT: or.b32 %r97, %r91, %r96;
+; CHECK-NEXT: shr.u32 %r98, %r84, 3;
+; CHECK-NEXT: and.b32 %r99, %r98, 15;
+; CHECK-NEXT: shl.b32 %r100, %r99, 23;
+; CHECK-NEXT: or.b32 %r101, %r100, %r86;
+; CHECK-NEXT: shl.b32 %r102, %r87, 20;
+; CHECK-NEXT: or.b32 %r103, %r101, %r102;
+; CHECK-NEXT: add.s32 %r104, %r103, 1006632960;
+; CHECK-NEXT: setp.ne.b32 %p16, %r87, 0;
+; CHECK-NEXT: selp.b32 %r105, %r97, %r104, %p16;
+; CHECK-NEXT: setp.eq.b32 %p17, %r99, 0;
+; CHECK-NEXT: selp.b32 %r106, %r105, %r104, %p17;
+; CHECK-NEXT: or.b32 %r107, %r99, %r87;
+; CHECK-NEXT: setp.eq.b32 %p18, %r107, 0;
+; CHECK-NEXT: selp.b32 %r108, %r86, %r106, %p18;
+; CHECK-NEXT: setp.eq.b32 %p19, %r87, 7;
+; CHECK-NEXT: selp.b32 %r109, 2143289344, %r108, %p19;
+; CHECK-NEXT: setp.eq.b32 %p20, %r99, 15;
+; CHECK-NEXT: selp.b32 %r110, %r109, %r108, %p20;
+; CHECK-NEXT: st.param.v4.b32 [func_retval0], {%r110, %r83, %r56, %r29};
; CHECK-NEXT: ret;
%r = call <4 x float> @llvm.convert.from.arbitrary.fp.v4f32.v4i8(<4 x i8> %x, metadata !"Float8E4M3FN")
ret <4 x float> %r
@@ -1154,7 +1174,7 @@ define <4 x float> @from_f8e5m2_v4f32(<4 x i8> %x) {
; CHECK-LABEL: from_f8e5m2_v4f32(
; CHECK: {
; CHECK-NEXT: .reg .pred %p<17>;
-; CHECK-NEXT: .reg .b32 %r<119>;
+; CHECK-NEXT: .reg .b32 %r<123>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
; CHECK-NEXT: ld.param.b32 %r1, [from_f8e5m2_v4f32_param_0];
@@ -1173,125 +1193,129 @@ define <4 x float> @from_f8e5m2_v4f32(<4 x i8> %x) {
; CHECK-NEXT: add.s32 %r14, %r6, -8;
; CHECK-NEXT: shl.b32 %r15, %r13, %r14;
; CHECK-NEXT: or.b32 %r16, %r9, %r15;
-; CHECK-NEXT: bfe.u32 %r17, %r2, 2, 5;
-; CHECK-NEXT: shl.b32 %r18, %r17, 23;
-; CHECK-NEXT: or.b32 %r19, %r18, %r4;
-; CHECK-NEXT: shl.b32 %r20, %r5, 21;
-; CHECK-NEXT: or.b32 %r21, %r19, %r20;
-; CHECK-NEXT: add.s32 %r22, %r21, 939524096;
+; CHECK-NEXT: shr.u32 %r17, %r2, 2;
+; CHECK-NEXT: and.b32 %r18, %r17, 31;
+; CHECK-NEXT: shl.b32 %r19, %r18, 23;
+; CHECK-NEXT: or.b32 %r20, %r19, %r4;
+; CHECK-NEXT: shl.b32 %r21, %r5, 21;
+; CHECK-NEXT: or.b32 %r22, %r20, %r21;
+; CHECK-NEXT: add.s32 %r23, %r22, 939524096;
; CHECK-NEXT: setp.ne.b32 %p1, %r5, 0;
-; CHECK-NEXT: selp.b32 %r23, %r16, %r22, %p1;
-; CHECK-NEXT: setp.eq.b32 %p2, %r17, 0;
-; CHECK-NEXT: selp.b32 %r24, %r23, %r22, %p2;
-; CHECK-NEXT: or.b32 %r25, %r17, %r5;
-; CHECK-NEXT: setp.eq.b32 %p3, %r25, 0;
-; CHECK-NEXT: selp.b32 %r26, %r4, %r24, %p3;
-; CHECK-NEXT: or.b32 %r27, %r4, 2139095040;
-; CHECK-NEXT: selp.b32 %r28, %r26, %r27, %p1;
-; CHECK-NEXT: setp.eq.b32 %p4, %r17, 31;
-; CHECK-NEXT: selp.b32 %r29, %r28, %r26, %p4;
-; CHECK-NEXT: selp.b32 %r30, 2143289344, %r29, %p1;
-; CHECK-NEXT: selp.b32 %r31, %r30, %r29, %p4;
-; CHECK-NEXT: prmt.b32 %r32, %r1, 0, 0x7772U;
-; CHECK-NEXT: shl.b32 %r33, %r32, 24;
-; CHECK-NEXT: and.b32 %r34, %r33, -2147483648;
-; CHECK-NEXT: and.b32 %r35, %r32, 3;
-; CHECK-NEXT: clz.b32 %r36, %r35;
-; CHECK-NEXT: sub.s32 %r37, 142, %r36;
-; CHECK-NEXT: shl.b32 %r38, %r37, 23;
-; CHECK-NEXT: or.b32 %r39, %r34, %r38;
-; CHECK-NEXT: sub.s32 %r40, 31, %r36;
-; CHECK-NEXT: shl.b32 %r41, %r11, %r40;
-; CHECK-NEXT: xor.b32 %r42, %r35, %r41;
-; CHECK-NEXT: add.s32 %r43, %r36, -8;
-; CHECK-NEXT: shl.b32 %r44, %r42, %r43;
-; CHECK-NEXT: or.b32 %r45, %r39, %r44;
-; CHECK-NEXT: bfe.u32 %r46, %r32, 2, 5;
-; CHECK-NEXT: shl.b32 %r47, %r46, 23;
-; CHECK-NEXT: or.b32 %r48, %r47, %r34;
-; CHECK-NEXT: shl.b32 %r49, %r35, 21;
-; CHECK-NEXT: or.b32 %r50, %r48, %r49;
-; CHECK-NEXT: add.s32 %r51, %r50, 939524096;
-; CHECK-NEXT: setp.ne.b32 %p5, %r35, 0;
-; CHECK-NEXT: selp.b32 %r52, %r45, %r51, %p5;
-; CHECK-NEXT: setp.eq.b32 %p6, %r46, 0;
-; CHECK-NEXT: selp.b32 %r53, %r52, %r51, %p6;
-; CHECK-NEXT: or.b32 %r54, %r46, %r35;
-; CHECK-NEXT: setp.eq.b32 %p7, %r54, 0;
-; CHECK-NEXT: selp.b32 %r55, %r34, %r53, %p7;
-; CHECK-NEXT: or.b32 %r56, %r34, 2139095040;
-; CHECK-NEXT: selp.b32 %r57, %r55, %r56, %p5;
-; CHECK-NEXT: setp.eq.b32 %p8, %r46, 31;
-; CHECK-NEXT: selp.b32 %r58, %r57, %r55, %p8;
-; CHECK-NEXT: selp.b32 %r59, 2143289344, %r58, %p5;
-; CHECK-NEXT: selp.b32 %r60, %r59, %r58, %p8;
-; CHECK-NEXT: prmt.b32 %r61, %r1, 0, 0x7771U;
-; CHECK-NEXT: shl.b32 %r62, %r61, 24;
-; CHECK-NEXT: and.b32 %r63, %r62, -2147483648;
-; CHECK-NEXT: and.b32 %r64, %r61, 3;
-; CHECK-NEXT: clz.b32 %r65, %r64;
-; CHECK-NEXT: sub.s32 %r66, 142, %r65;
-; CHECK-NEXT: shl.b32 %r67, %r66, 23;
-; CHECK-NEXT: or.b32 %r68, %r63, %r67;
-; CHECK-NEXT: sub.s32 %r69, 31, %r65;
-; CHECK-NEXT: shl.b32 %r70, %r11, %r69;
-; CHECK-NEXT: xor.b32 %r71, %r64, %r70;
-; CHECK-NEXT: add.s32 %r72, %r65, -8;
-; CHECK-NEXT: shl.b32 %r73, %r71, %r72;
-; CHECK-NEXT: or.b32 %r74, %r68, %r73;
-; CHECK-NEXT: bfe.u32 %r75, %r61, 2, 5;
-; CHECK-NEXT: shl.b32 %r76, %r75, 23;
-; CHECK-NEXT: or.b32 %r77, %r76, %r63;
-; CHECK-NEXT: shl.b32 %r78, %r64, 21;
-; CHECK-NEXT: or.b32 %r79, %r77, %r78;
-; CHECK-NEXT: add.s32 %r80, %r79, 939524096;
-; CHECK-NEXT: setp.ne.b32 %p9, %r64, 0;
-; CHECK-NEXT: selp.b32 %r81, %r74, %r80, %p9;
-; CHECK-NEXT: setp.eq.b32 %p10, %r75, 0;
-; CHECK-NEXT: selp.b32 %r82, %r81, %r80, %p10;
-; CHECK-NEXT: or.b32 %r83, %r75, %r64;
-; CHECK-NEXT: setp.eq.b32 %p11, %r83, 0;
-; CHECK-NEXT: selp.b32 %r84, %r63, %r82, %p11;
-; CHECK-NEXT: or.b32 %r85, %r63, 2139095040;
-; CHECK-NEXT: selp.b32 %r86, %r84, %r85, %p9;
-; CHECK-NEXT: setp.eq.b32 %p12, %r75, 31;
-; CHECK-NEXT: selp.b32 %r87, %r86, %r84, %p12;
-; CHECK-NEXT: selp.b32 %r88, 2143289344, %r87, %p9;
-; CHECK-NEXT: selp.b32 %r89, %r88, %r87, %p12;
-; CHECK-NEXT: prmt.b32 %r90, %r1, 0, 0x7770U;
-; CHECK-NEXT: shl.b32 %r91, %r90, 24;
-; CHECK-NEXT: and.b32 %r92, %r91, -2147483648;
-; CHECK-NEXT: and.b32 %r93, %r90, 3;
-; CHECK-NEXT: clz.b32 %r94, %r93;
-; CHECK-NEXT: sub.s32 %r95, 142, %r94;
-; CHECK-NEXT: shl.b32 %r96, %r95, 23;
-; CHECK-NEXT: or.b32 %r97, %r92, %r96;
-; CHECK-NEXT: sub.s32 %r98, 31, %r94;
-; CHECK-NEXT: shl.b32 %r99, %r11, %r98;
-; CHECK-NEXT: xor.b32 %r100, %r93, %r99;
-; CHECK-NEXT: add.s32 %r101, %r94, -8;
-; CHECK-NEXT: shl.b32 %r102, %r100, %r101;
-; CHECK-NEXT: or.b32 %r103, %r97, %r102;
-; CHECK-NEXT: bfe.u32 %r104, %r90, 2, 5;
-; CHECK-NEXT: shl.b32 %r105, %r104, 23;
-; CHECK-NEXT: or.b32 %r106, %r105, %r92;
-; CHECK-NEXT: shl.b32 %r107, %r93, 21;
-; CHECK-NEXT: or.b32 %r108, %r106, %r107;
-; CHECK-NEXT: add.s32 %r109, %r108, 939524096;
-; CHECK-NEXT: setp.ne.b32 %p13, %r93, 0;
-; CHECK-NEXT: selp.b32 %r110, %r103, %r109, %p13;
-; CHECK-NEXT: setp.eq.b32 %p14, %r104, 0;
-; CHECK-NEXT: selp.b32 %r111, %r110, %r109, %p14;
-; CHECK-NEXT: or.b32 %r112, %r104, %r93;
-; CHECK-NEXT: setp.eq.b32 %p15, %r112, 0;
-; CHECK-NEXT: selp.b32 %r113, %r92, %r111, %p15;
-; CHECK-NEXT: or.b32 %r114, %r92, 2139095040;
-; CHECK-NEXT: selp.b32 %r115, %r113, %r114, %p13;
-; CHECK-NEXT: setp.eq.b32 %p16, %r104, 31;
-; CHECK-NEXT: selp.b32 %r116, %r115, %r113, %p16;
-; CHECK-NEXT: selp.b32 %r117, 2143289344, %r116, %p13;
-; CHECK-NEXT: selp.b32 %r118, %r117, %r116, %p16;
-; CHECK-NEXT: st.param.v4.b32 [func_retval0], {%r118, %r89, %r60, %r31};
+; CHECK-NEXT: selp.b32 %r24, %r16, %r23, %p1;
+; CHECK-NEXT: setp.eq.b32 %p2, %r18, 0;
+; CHECK-NEXT: selp.b32 %r25, %r24, %r23, %p2;
+; CHECK-NEXT: or.b32 %r26, %r18, %r5;
+; CHECK-NEXT: setp.eq.b32 %p3, %r26, 0;
+; CHECK-NEXT: selp.b32 %r27, %r4, %r25, %p3;
+; CHECK-NEXT: or.b32 %r28, %r4, 2139095040;
+; CHECK-NEXT: selp.b32 %r29, %r27, %r28, %p1;
+; CHECK-NEXT: setp.eq.b32 %p4, %r18, 31;
+; CHECK-NEXT: selp.b32 %r30, %r29, %r27, %p4;
+; CHECK-NEXT: selp.b32 %r31, 2143289344, %r30, %p1;
+; CHECK-NEXT: selp.b32 %r32, %r31, %r30, %p4;
+; CHECK-NEXT: prmt.b32 %r33, %r1, 0, 0x7772U;
+; CHECK-NEXT: shl.b32 %r34, %r33, 24;
+; CHECK-NEXT: and.b32 %r35, %r34, -2147483648;
+; CHECK-NEXT: and.b32 %r36, %r33, 3;
+; CHECK-NEXT: clz.b32 %r37, %r36;
+; CHECK-NEXT: sub.s32 %r38, 142, %r37;
+; CHECK-NEXT: shl.b32 %r39, %r38, 23;
+; CHECK-NEXT: or.b32 %r40, %r35, %r39;
+; CHECK-NEXT: sub.s32 %r41, 31, %r37;
+; CHECK-NEXT: shl.b32 %r42, %r11, %r41;
+; CHECK-NEXT: xor.b32 %r43, %r36, %r42;
+; CHECK-NEXT: add.s32 %r44, %r37, -8;
+; CHECK-NEXT: shl.b32 %r45, %r43, %r44;
+; CHECK-NEXT: or.b32 %r46, %r40, %r45;
+; CHECK-NEXT: shr.u32 %r47, %r33, 2;
+; CHECK-NEXT: and.b32 %r48, %r47, 31;
+; CHECK-NEXT: shl.b32 %r49, %r48, 23;
+; CHECK-NEXT: or.b32 %r50, %r49, %r35;
+; CHECK-NEXT: shl.b32 %r51, %r36, 21;
+; CHECK-NEXT: or.b32 %r52, %r50, %r51;
+; CHECK-NEXT: add.s32 %r53, %r52, 939524096;
+; CHECK-NEXT: setp.ne.b32 %p5, %r36, 0;
+; CHECK-NEXT: selp.b32 %r54, %r46, %r53, %p5;
+; CHECK-NEXT: setp.eq.b32 %p6, %r48, 0;
+; CHECK-NEXT: selp.b32 %r55, %r54, %r53, %p6;
+; CHECK-NEXT: or.b32 %r56, %r48, %r36;
+; CHECK-NEXT: setp.eq.b32 %p7, %r56, 0;
+; CHECK-NEXT: selp.b32 %r57, %r35, %r55, %p7;
+; CHECK-NEXT: or.b32 %r58, %r35, 2139095040;
+; CHECK-NEXT: selp.b32 %r59, %r57, %r58, %p5;
+; CHECK-NEXT: setp.eq.b32 %p8, %r48, 31;
+; CHECK-NEXT: selp.b32 %r60, %r59, %r57, %p8;
+; CHECK-NEXT: selp.b32 %r61, 2143289344, %r60, %p5;
+; CHECK-NEXT: selp.b32 %r62, %r61, %r60, %p8;
+; CHECK-NEXT: prmt.b32 %r63, %r1, 0, 0x7771U;
+; CHECK-NEXT: shl.b32 %r64, %r63, 24;
+; CHECK-NEXT: and.b32 %r65, %r64, -2147483648;
+; CHECK-NEXT: and.b32 %r66, %r63, 3;
+; CHECK-NEXT: clz.b32 %r67, %r66;
+; CHECK-NEXT: sub.s32 %r68, 142, %r67;
+; CHECK-NEXT: shl.b32 %r69, %r68, 23;
+; CHECK-NEXT: or.b32 %r70, %r65, %r69;
+; CHECK-NEXT: sub.s32 %r71, 31, %r67;
+; CHECK-NEXT: shl.b32 %r72, %r11, %r71;
+; CHECK-NEXT: xor.b32 %r73, %r66, %r72;
+; CHECK-NEXT: add.s32 %r74, %r67, -8;
+; CHECK-NEXT: shl.b32 %r75, %r73, %r74;
+; CHECK-NEXT: or.b32 %r76, %r70, %r75;
+; CHECK-NEXT: shr.u32 %r77, %r63, 2;
+; CHECK-NEXT: and.b32 %r78, %r77, 31;
+; CHECK-NEXT: shl.b32 %r79, %r78, 23;
+; CHECK-NEXT: or.b32 %r80, %r79, %r65;
+; CHECK-NEXT: shl.b32 %r81, %r66, 21;
+; CHECK-NEXT: or.b32 %r82, %r80, %r81;
+; CHECK-NEXT: add.s32 %r83, %r82, 939524096;
+; CHECK-NEXT: setp.ne.b32 %p9, %r66, 0;
+; CHECK-NEXT: selp.b32 %r84, %r76, %r83, %p9;
+; CHECK-NEXT: setp.eq.b32 %p10, %r78, 0;
+; CHECK-NEXT: selp.b32 %r85, %r84, %r83, %p10;
+; CHECK-NEXT: or.b32 %r86, %r78, %r66;
+; CHECK-NEXT: setp.eq.b32 %p11, %r86, 0;
+; CHECK-NEXT: selp.b32 %r87, %r65, %r85, %p11;
+; CHECK-NEXT: or.b32 %r88, %r65, 2139095040;
+; CHECK-NEXT: selp.b32 %r89, %r87, %r88, %p9;
+; CHECK-NEXT: setp.eq.b32 %p12, %r78, 31;
+; CHECK-NEXT: selp.b32 %r90, %r89, %r87, %p12;
+; CHECK-NEXT: selp.b32 %r91, 2143289344, %r90, %p9;
+; CHECK-NEXT: selp.b32 %r92, %r91, %r90, %p12;
+; CHECK-NEXT: prmt.b32 %r93, %r1, 0, 0x7770U;
+; CHECK-NEXT: shl.b32 %r94, %r93, 24;
+; CHECK-NEXT: and.b32 %r95, %r94, -2147483648;
+; CHECK-NEXT: and.b32 %r96, %r93, 3;
+; CHECK-NEXT: clz.b32 %r97, %r96;
+; CHECK-NEXT: sub.s32 %r98, 142, %r97;
+; CHECK-NEXT: shl.b32 %r99, %r98, 23;
+; CHECK-NEXT: or.b32 %r100, %r95, %r99;
+; CHECK-NEXT: sub.s32 %r101, 31, %r97;
+; CHECK-NEXT: shl.b32 %r102, %r11, %r101;
+; CHECK-NEXT: xor.b32 %r103, %r96, %r102;
+; CHECK-NEXT: add.s32 %r104, %r97, -8;
+; CHECK-NEXT: shl.b32 %r105, %r103, %r104;
+; CHECK-NEXT: or.b32 %r106, %r100, %r105;
+; CHECK-NEXT: shr.u32 %r107, %r93, 2;
+; CHECK-NEXT: and.b32 %r108, %r107, 31;
+; CHECK-NEXT: shl.b32 %r109, %r108, 23;
+; CHECK-NEXT: or.b32 %r110, %r109, %r95;
+; CHECK-NEXT: shl.b32 %r111, %r96, 21;
+; CHECK-NEXT: or.b32 %r112, %r110, %r111;
+; CHECK-NEXT: add.s32 %r113, %r112, 939524096;
+; CHECK-NEXT: setp.ne.b32 %p13, %r96, 0;
+; CHECK-NEXT: selp.b32 %r114, %r106, %r113, %p13;
+; CHECK-NEXT: setp.eq.b32 %p14, %r108, 0;
+; CHECK-NEXT: selp.b32 %r115, %r114, %r113, %p14;
+; CHECK-NEXT: or.b32 %r116, %r108, %r96;
+; CHECK-NEXT: setp.eq.b32 %p15, %r116, 0;
+; CHECK-NEXT: selp.b32 %r117, %r95, %r115, %p15;
+; CHECK-NEXT: or.b32 %r118, %r95, 2139095040;
+; CHECK-NEXT: selp.b32 %r119, %r117, %r118, %p13;
+; CHECK-NEXT: setp.eq.b32 %p16, %r108, 31;
+; CHECK-NEXT: selp.b32 %r120, %r119, %r117, %p16;
+; CHECK-NEXT: selp.b32 %r121, 2143289344, %r120, %p13;
+; CHECK-NEXT: selp.b32 %r122, %r121, %r120, %p16;
+; CHECK-NEXT: st.param.v4.b32 [func_retval0], {%r122, %r92, %r62, %r32};
; CHECK-NEXT: ret;
%r = call <4 x float> @llvm.convert.from.arbitrary.fp.v4f32.v4i8(<4 x i8> %x, metadata !"Float8E5M2")
ret <4 x float> %r
diff --git a/llvm/test/CodeGen/NVPTX/atomicrmw-sm70.ll b/llvm/test/CodeGen/NVPTX/atomicrmw-sm70.ll
index 00fbbf4db9ff2..19ffb12f9ab05 100644
--- a/llvm/test/CodeGen/NVPTX/atomicrmw-sm70.ll
+++ b/llvm/test/CodeGen/NVPTX/atomicrmw-sm70.ll
@@ -3235,7 +3235,7 @@ define bfloat @fadd_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat %va
; SM70-NOFTZ-DISALLOW: {
; SM70-NOFTZ-DISALLOW-NEXT: .reg .pred %p<3>;
; SM70-NOFTZ-DISALLOW-NEXT: .reg .b16 %rs<2>;
-; SM70-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<24>;
+; SM70-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<25>;
; SM70-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
; SM70-NOFTZ-DISALLOW-EMPTY:
; SM70-NOFTZ-DISALLOW-NEXT: // %bb.0:
@@ -3249,39 +3249,40 @@ define bfloat @fadd_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat %va
; SM70-NOFTZ-DISALLOW-NEXT: mov.b32 %r6, 65535;
; SM70-NOFTZ-DISALLOW-NEXT: shl.b32 %r7, %r6, %r1;
; SM70-NOFTZ-DISALLOW-NEXT: not.b32 %r2, %r7;
-; SM70-NOFTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b32 %r23, [%rd1];
+; SM70-NOFTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b32 %r24, [%rd1];
; SM70-NOFTZ-DISALLOW-NEXT: cvt.u32.u16 %r10, %rs1;
; SM70-NOFTZ-DISALLOW-NEXT: shl.b32 %r11, %r10, 16;
; SM70-NOFTZ-DISALLOW-NEXT: $L__BB78_1: // %atomicrmw.start
; SM70-NOFTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NOFTZ-DISALLOW-NEXT: shr.u32 %r8, %r23, %r1;
+; SM70-NOFTZ-DISALLOW-NEXT: shr.u32 %r8, %r24, %r1;
; SM70-NOFTZ-DISALLOW-NEXT: shl.b32 %r9, %r8, 16;
; SM70-NOFTZ-DISALLOW-NEXT: add.rn.f32 %r12, %r9, %r11;
-; SM70-NOFTZ-DISALLOW-NEXT: bfe.u32 %r13, %r12, 16, 1;
-; SM70-NOFTZ-DISALLOW-NEXT: add.s32 %r14, %r13, %r12;
-; SM70-NOFTZ-DISALLOW-NEXT: add.s32 %r15, %r14, 32767;
+; SM70-NOFTZ-DISALLOW-NEXT: shr.u32 %r13, %r12, 16;
+; SM70-NOFTZ-DISALLOW-NEXT: and.b32 %r14, %r13, 1;
+; SM70-NOFTZ-DISALLOW-NEXT: add.s32 %r15, %r14, %r12;
+; SM70-NOFTZ-DISALLOW-NEXT: add.s32 %r16, %r15, 32767;
; SM70-NOFTZ-DISALLOW-NEXT: setp.nan.f32 %p1, %r12, %r12;
-; SM70-NOFTZ-DISALLOW-NEXT: or.b32 %r16, %r12, 4194304;
-; SM70-NOFTZ-DISALLOW-NEXT: selp.b32 %r17, %r16, %r15, %p1;
-; SM70-NOFTZ-DISALLOW-NEXT: shr.u32 %r18, %r17, 16;
-; SM70-NOFTZ-DISALLOW-NEXT: shl.b32 %r19, %r18, %r1;
-; SM70-NOFTZ-DISALLOW-NEXT: and.b32 %r20, %r23, %r2;
-; SM70-NOFTZ-DISALLOW-NEXT: or.b32 %r21, %r20, %r19;
-; SM70-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r23, %r21;
-; SM70-NOFTZ-DISALLOW-NEXT: setp.ne.b32 %p2, %r3, %r23;
-; SM70-NOFTZ-DISALLOW-NEXT: mov.b32 %r23, %r3;
+; SM70-NOFTZ-DISALLOW-NEXT: or.b32 %r17, %r12, 4194304;
+; SM70-NOFTZ-DISALLOW-NEXT: selp.b32 %r18, %r17, %r16, %p1;
+; SM70-NOFTZ-DISALLOW-NEXT: shr.u32 %r19, %r18, 16;
+; SM70-NOFTZ-DISALLOW-NEXT: shl.b32 %r20, %r19, %r1;
+; SM70-NOFTZ-DISALLOW-NEXT: and.b32 %r21, %r24, %r2;
+; SM70-NOFTZ-DISALLOW-NEXT: or.b32 %r22, %r21, %r20;
+; SM70-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r24, %r22;
+; SM70-NOFTZ-DISALLOW-NEXT: setp.ne.b32 %p2, %r3, %r24;
+; SM70-NOFTZ-DISALLOW-NEXT: mov.b32 %r24, %r3;
; SM70-NOFTZ-DISALLOW-NEXT: @%p2 bra $L__BB78_1;
; SM70-NOFTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
-; SM70-NOFTZ-DISALLOW-NEXT: shr.u32 %r22, %r3, %r1;
+; SM70-NOFTZ-DISALLOW-NEXT: shr.u32 %r23, %r3, %r1;
; SM70-NOFTZ-DISALLOW-NEXT: fence.acq_rel.cta;
-; SM70-NOFTZ-DISALLOW-NEXT: st.param.b16 [func_retval0], %r22;
+; SM70-NOFTZ-DISALLOW-NEXT: st.param.b16 [func_retval0], %r23;
; SM70-NOFTZ-DISALLOW-NEXT: ret;
;
; SM70-NOFTZ-ALLOW-LABEL: fadd_acq_rel_bfloat_global_cta(
; SM70-NOFTZ-ALLOW: {
; SM70-NOFTZ-ALLOW-NEXT: .reg .pred %p<3>;
; SM70-NOFTZ-ALLOW-NEXT: .reg .b16 %rs<2>;
-; SM70-NOFTZ-ALLOW-NEXT: .reg .b32 %r<24>;
+; SM70-NOFTZ-ALLOW-NEXT: .reg .b32 %r<25>;
; SM70-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
; SM70-NOFTZ-ALLOW-EMPTY:
; SM70-NOFTZ-ALLOW-NEXT: // %bb.0:
@@ -3295,39 +3296,40 @@ define bfloat @fadd_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat %va
; SM70-NOFTZ-ALLOW-NEXT: mov.b32 %r6, 65535;
; SM70-NOFTZ-ALLOW-NEXT: shl.b32 %r7, %r6, %r1;
; SM70-NOFTZ-ALLOW-NEXT: not.b32 %r2, %r7;
-; SM70-NOFTZ-ALLOW-NEXT: ld.relaxed.cta.global.b32 %r23, [%rd1];
+; SM70-NOFTZ-ALLOW-NEXT: ld.relaxed.cta.global.b32 %r24, [%rd1];
; SM70-NOFTZ-ALLOW-NEXT: cvt.u32.u16 %r10, %rs1;
; SM70-NOFTZ-ALLOW-NEXT: shl.b32 %r11, %r10, 16;
; SM70-NOFTZ-ALLOW-NEXT: $L__BB78_1: // %atomicrmw.start
; SM70-NOFTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NOFTZ-ALLOW-NEXT: shr.u32 %r8, %r23, %r1;
+; SM70-NOFTZ-ALLOW-NEXT: shr.u32 %r8, %r24, %r1;
; SM70-NOFTZ-ALLOW-NEXT: shl.b32 %r9, %r8, 16;
; SM70-NOFTZ-ALLOW-NEXT: add.rn.f32 %r12, %r9, %r11;
-; SM70-NOFTZ-ALLOW-NEXT: bfe.u32 %r13, %r12, 16, 1;
-; SM70-NOFTZ-ALLOW-NEXT: add.s32 %r14, %r13, %r12;
-; SM70-NOFTZ-ALLOW-NEXT: add.s32 %r15, %r14, 32767;
+; SM70-NOFTZ-ALLOW-NEXT: shr.u32 %r13, %r12, 16;
+; SM70-NOFTZ-ALLOW-NEXT: and.b32 %r14, %r13, 1;
+; SM70-NOFTZ-ALLOW-NEXT: add.s32 %r15, %r14, %r12;
+; SM70-NOFTZ-ALLOW-NEXT: add.s32 %r16, %r15, 32767;
; SM70-NOFTZ-ALLOW-NEXT: setp.nan.f32 %p1, %r12, %r12;
-; SM70-NOFTZ-ALLOW-NEXT: or.b32 %r16, %r12, 4194304;
-; SM70-NOFTZ-ALLOW-NEXT: selp.b32 %r17, %r16, %r15, %p1;
-; SM70-NOFTZ-ALLOW-NEXT: shr.u32 %r18, %r17, 16;
-; SM70-NOFTZ-ALLOW-NEXT: shl.b32 %r19, %r18, %r1;
-; SM70-NOFTZ-ALLOW-NEXT: and.b32 %r20, %r23, %r2;
-; SM70-NOFTZ-ALLOW-NEXT: or.b32 %r21, %r20, %r19;
-; SM70-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r23, %r21;
-; SM70-NOFTZ-ALLOW-NEXT: setp.ne.b32 %p2, %r3, %r23;
-; SM70-NOFTZ-ALLOW-NEXT: mov.b32 %r23, %r3;
+; SM70-NOFTZ-ALLOW-NEXT: or.b32 %r17, %r12, 4194304;
+; SM70-NOFTZ-ALLOW-NEXT: selp.b32 %r18, %r17, %r16, %p1;
+; SM70-NOFTZ-ALLOW-NEXT: shr.u32 %r19, %r18, 16;
+; SM70-NOFTZ-ALLOW-NEXT: shl.b32 %r20, %r19, %r1;
+; SM70-NOFTZ-ALLOW-NEXT: and.b32 %r21, %r24, %r2;
+; SM70-NOFTZ-ALLOW-NEXT: or.b32 %r22, %r21, %r20;
+; SM70-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r24, %r22;
+; SM70-NOFTZ-ALLOW-NEXT: setp.ne.b32 %p2, %r3, %r24;
+; SM70-NOFTZ-ALLOW-NEXT: mov.b32 %r24, %r3;
; SM70-NOFTZ-ALLOW-NEXT: @%p2 bra $L__BB78_1;
; SM70-NOFTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
-; SM70-NOFTZ-ALLOW-NEXT: shr.u32 %r22, %r3, %r1;
+; SM70-NOFTZ-ALLOW-NEXT: shr.u32 %r23, %r3, %r1;
; SM70-NOFTZ-ALLOW-NEXT: fence.acq_rel.cta;
-; SM70-NOFTZ-ALLOW-NEXT: st.param.b16 [func_retval0], %r22;
+; SM70-NOFTZ-ALLOW-NEXT: st.param.b16 [func_retval0], %r23;
; SM70-NOFTZ-ALLOW-NEXT: ret;
;
; SM70-FTZ-DISALLOW-LABEL: fadd_acq_rel_bfloat_global_cta(
; SM70-FTZ-DISALLOW: {
; SM70-FTZ-DISALLOW-NEXT: .reg .pred %p<3>;
; SM70-FTZ-DISALLOW-NEXT: .reg .b16 %rs<2>;
-; SM70-FTZ-DISALLOW-NEXT: .reg .b32 %r<24>;
+; SM70-FTZ-DISALLOW-NEXT: .reg .b32 %r<25>;
; SM70-FTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
; SM70-FTZ-DISALLOW-EMPTY:
; SM70-FTZ-DISALLOW-NEXT: // %bb.0:
@@ -3341,39 +3343,40 @@ define bfloat @fadd_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat %va
; SM70-FTZ-DISALLOW-NEXT: mov.b32 %r6, 65535;
; SM70-FTZ-DISALLOW-NEXT: shl.b32 %r7, %r6, %r1;
; SM70-FTZ-DISALLOW-NEXT: not.b32 %r2, %r7;
-; SM70-FTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b32 %r23, [%rd1];
+; SM70-FTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b32 %r24, [%rd1];
; SM70-FTZ-DISALLOW-NEXT: cvt.u32.u16 %r10, %rs1;
; SM70-FTZ-DISALLOW-NEXT: shl.b32 %r11, %r10, 16;
; SM70-FTZ-DISALLOW-NEXT: $L__BB78_1: // %atomicrmw.start
; SM70-FTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-FTZ-DISALLOW-NEXT: shr.u32 %r8, %r23, %r1;
+; SM70-FTZ-DISALLOW-NEXT: shr.u32 %r8, %r24, %r1;
; SM70-FTZ-DISALLOW-NEXT: shl.b32 %r9, %r8, 16;
; SM70-FTZ-DISALLOW-NEXT: add.rn.ftz.f32 %r12, %r9, %r11;
-; SM70-FTZ-DISALLOW-NEXT: bfe.u32 %r13, %r12, 16, 1;
-; SM70-FTZ-DISALLOW-NEXT: add.s32 %r14, %r13, %r12;
-; SM70-FTZ-DISALLOW-NEXT: add.s32 %r15, %r14, 32767;
+; SM70-FTZ-DISALLOW-NEXT: shr.u32 %r13, %r12, 16;
+; SM70-FTZ-DISALLOW-NEXT: and.b32 %r14, %r13, 1;
+; SM70-FTZ-DISALLOW-NEXT: add.s32 %r15, %r14, %r12;
+; SM70-FTZ-DISALLOW-NEXT: add.s32 %r16, %r15, 32767;
; SM70-FTZ-DISALLOW-NEXT: setp.nan.ftz.f32 %p1, %r12, %r12;
-; SM70-FTZ-DISALLOW-NEXT: or.b32 %r16, %r12, 4194304;
-; SM70-FTZ-DISALLOW-NEXT: selp.b32 %r17, %r16, %r15, %p1;
-; SM70-FTZ-DISALLOW-NEXT: shr.u32 %r18, %r17, 16;
-; SM70-FTZ-DISALLOW-NEXT: shl.b32 %r19, %r18, %r1;
-; SM70-FTZ-DISALLOW-NEXT: and.b32 %r20, %r23, %r2;
-; SM70-FTZ-DISALLOW-NEXT: or.b32 %r21, %r20, %r19;
-; SM70-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r23, %r21;
-; SM70-FTZ-DISALLOW-NEXT: setp.ne.b32 %p2, %r3, %r23;
-; SM70-FTZ-DISALLOW-NEXT: mov.b32 %r23, %r3;
+; SM70-FTZ-DISALLOW-NEXT: or.b32 %r17, %r12, 4194304;
+; SM70-FTZ-DISALLOW-NEXT: selp.b32 %r18, %r17, %r16, %p1;
+; SM70-FTZ-DISALLOW-NEXT: shr.u32 %r19, %r18, 16;
+; SM70-FTZ-DISALLOW-NEXT: shl.b32 %r20, %r19, %r1;
+; SM70-FTZ-DISALLOW-NEXT: and.b32 %r21, %r24, %r2;
+; SM70-FTZ-DISALLOW-NEXT: or.b32 %r22, %r21, %r20;
+; SM70-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r24, %r22;
+; SM70-FTZ-DISALLOW-NEXT: setp.ne.b32 %p2, %r3, %r24;
+; SM70-FTZ-DISALLOW-NEXT: mov.b32 %r24, %r3;
; SM70-FTZ-DISALLOW-NEXT: @%p2 bra $L__BB78_1;
; SM70-FTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
-; SM70-FTZ-DISALLOW-NEXT: shr.u32 %r22, %r3, %r1;
+; SM70-FTZ-DISALLOW-NEXT: shr.u32 %r23, %r3, %r1;
; SM70-FTZ-DISALLOW-NEXT: fence.acq_rel.cta;
-; SM70-FTZ-DISALLOW-NEXT: st.param.b16 [func_retval0], %r22;
+; SM70-FTZ-DISALLOW-NEXT: st.param.b16 [func_retval0], %r23;
; SM70-FTZ-DISALLOW-NEXT: ret;
;
; SM70-FTZ-ALLOW-LABEL: fadd_acq_rel_bfloat_global_cta(
; SM70-FTZ-ALLOW: {
; SM70-FTZ-ALLOW-NEXT: .reg .pred %p<3>;
; SM70-FTZ-ALLOW-NEXT: .reg .b16 %rs<2>;
-; SM70-FTZ-ALLOW-NEXT: .reg .b32 %r<24>;
+; SM70-FTZ-ALLOW-NEXT: .reg .b32 %r<25>;
; SM70-FTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
; SM70-FTZ-ALLOW-EMPTY:
; SM70-FTZ-ALLOW-NEXT: // %bb.0:
@@ -3387,32 +3390,33 @@ define bfloat @fadd_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat %va
; SM70-FTZ-ALLOW-NEXT: mov.b32 %r6, 65535;
; SM70-FTZ-ALLOW-NEXT: shl.b32 %r7, %r6, %r1;
; SM70-FTZ-ALLOW-NEXT: not.b32 %r2, %r7;
-; SM70-FTZ-ALLOW-NEXT: ld.relaxed.cta.global.b32 %r23, [%rd1];
+; SM70-FTZ-ALLOW-NEXT: ld.relaxed.cta.global.b32 %r24, [%rd1];
; SM70-FTZ-ALLOW-NEXT: cvt.u32.u16 %r10, %rs1;
; SM70-FTZ-ALLOW-NEXT: shl.b32 %r11, %r10, 16;
; SM70-FTZ-ALLOW-NEXT: $L__BB78_1: // %atomicrmw.start
; SM70-FTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-FTZ-ALLOW-NEXT: shr.u32 %r8, %r23, %r1;
+; SM70-FTZ-ALLOW-NEXT: shr.u32 %r8, %r24, %r1;
; SM70-FTZ-ALLOW-NEXT: shl.b32 %r9, %r8, 16;
; SM70-FTZ-ALLOW-NEXT: add.rn.ftz.f32 %r12, %r9, %r11;
-; SM70-FTZ-ALLOW-NEXT: bfe.u32 %r13, %r12, 16, 1;
-; SM70-FTZ-ALLOW-NEXT: add.s32 %r14, %r13, %r12;
-; SM70-FTZ-ALLOW-NEXT: add.s32 %r15, %r14, 32767;
+; SM70-FTZ-ALLOW-NEXT: shr.u32 %r13, %r12, 16;
+; SM70-FTZ-ALLOW-NEXT: and.b32 %r14, %r13, 1;
+; SM70-FTZ-ALLOW-NEXT: add.s32 %r15, %r14, %r12;
+; SM70-FTZ-ALLOW-NEXT: add.s32 %r16, %r15, 32767;
; SM70-FTZ-ALLOW-NEXT: setp.nan.ftz.f32 %p1, %r12, %r12;
-; SM70-FTZ-ALLOW-NEXT: or.b32 %r16, %r12, 4194304;
-; SM70-FTZ-ALLOW-NEXT: selp.b32 %r17, %r16, %r15, %p1;
-; SM70-FTZ-ALLOW-NEXT: shr.u32 %r18, %r17, 16;
-; SM70-FTZ-ALLOW-NEXT: shl.b32 %r19, %r18, %r1;
-; SM70-FTZ-ALLOW-NEXT: and.b32 %r20, %r23, %r2;
-; SM70-FTZ-ALLOW-NEXT: or.b32 %r21, %r20, %r19;
-; SM70-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r23, %r21;
-; SM70-FTZ-ALLOW-NEXT: setp.ne.b32 %p2, %r3, %r23;
-; SM70-FTZ-ALLOW-NEXT: mov.b32 %r23, %r3;
+; SM70-FTZ-ALLOW-NEXT: or.b32 %r17, %r12, 4194304;
+; SM70-FTZ-ALLOW-NEXT: selp.b32 %r18, %r17, %r16, %p1;
+; SM70-FTZ-ALLOW-NEXT: shr.u32 %r19, %r18, 16;
+; SM70-FTZ-ALLOW-NEXT: shl.b32 %r20, %r19, %r1;
+; SM70-FTZ-ALLOW-NEXT: and.b32 %r21, %r24, %r2;
+; SM70-FTZ-ALLOW-NEXT: or.b32 %r22, %r21, %r20;
+; SM70-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r24, %r22;
+; SM70-FTZ-ALLOW-NEXT: setp.ne.b32 %p2, %r3, %r24;
+; SM70-FTZ-ALLOW-NEXT: mov.b32 %r24, %r3;
; SM70-FTZ-ALLOW-NEXT: @%p2 bra $L__BB78_1;
; SM70-FTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
-; SM70-FTZ-ALLOW-NEXT: shr.u32 %r22, %r3, %r1;
+; SM70-FTZ-ALLOW-NEXT: shr.u32 %r23, %r3, %r1;
; SM70-FTZ-ALLOW-NEXT: fence.acq_rel.cta;
-; SM70-FTZ-ALLOW-NEXT: st.param.b16 [func_retval0], %r22;
+; SM70-FTZ-ALLOW-NEXT: st.param.b16 [func_retval0], %r23;
; SM70-FTZ-ALLOW-NEXT: ret;
%retval = atomicrmw fadd ptr addrspace(1) %addr, bfloat %val syncscope("block") acq_rel
ret bfloat %retval
@@ -3423,7 +3427,7 @@ define bfloat @fsub_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat %va
; SM70-NOFTZ-DISALLOW: {
; SM70-NOFTZ-DISALLOW-NEXT: .reg .pred %p<3>;
; SM70-NOFTZ-DISALLOW-NEXT: .reg .b16 %rs<2>;
-; SM70-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<24>;
+; SM70-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<25>;
; SM70-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
; SM70-NOFTZ-DISALLOW-EMPTY:
; SM70-NOFTZ-DISALLOW-NEXT: // %bb.0:
@@ -3437,39 +3441,40 @@ define bfloat @fsub_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat %va
; SM70-NOFTZ-DISALLOW-NEXT: mov.b32 %r6, 65535;
; SM70-NOFTZ-DISALLOW-NEXT: shl.b32 %r7, %r6, %r1;
; SM70-NOFTZ-DISALLOW-NEXT: not.b32 %r2, %r7;
-; SM70-NOFTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b32 %r23, [%rd1];
+; SM70-NOFTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b32 %r24, [%rd1];
; SM70-NOFTZ-DISALLOW-NEXT: cvt.u32.u16 %r10, %rs1;
; SM70-NOFTZ-DISALLOW-NEXT: shl.b32 %r11, %r10, 16;
; SM70-NOFTZ-DISALLOW-NEXT: $L__BB79_1: // %atomicrmw.start
; SM70-NOFTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NOFTZ-DISALLOW-NEXT: shr.u32 %r8, %r23, %r1;
+; SM70-NOFTZ-DISALLOW-NEXT: shr.u32 %r8, %r24, %r1;
; SM70-NOFTZ-DISALLOW-NEXT: shl.b32 %r9, %r8, 16;
; SM70-NOFTZ-DISALLOW-NEXT: sub.rn.f32 %r12, %r9, %r11;
-; SM70-NOFTZ-DISALLOW-NEXT: bfe.u32 %r13, %r12, 16, 1;
-; SM70-NOFTZ-DISALLOW-NEXT: add.s32 %r14, %r13, %r12;
-; SM70-NOFTZ-DISALLOW-NEXT: add.s32 %r15, %r14, 32767;
+; SM70-NOFTZ-DISALLOW-NEXT: shr.u32 %r13, %r12, 16;
+; SM70-NOFTZ-DISALLOW-NEXT: and.b32 %r14, %r13, 1;
+; SM70-NOFTZ-DISALLOW-NEXT: add.s32 %r15, %r14, %r12;
+; SM70-NOFTZ-DISALLOW-NEXT: add.s32 %r16, %r15, 32767;
; SM70-NOFTZ-DISALLOW-NEXT: setp.nan.f32 %p1, %r12, %r12;
-; SM70-NOFTZ-DISALLOW-NEXT: or.b32 %r16, %r12, 4194304;
-; SM70-NOFTZ-DISALLOW-NEXT: selp.b32 %r17, %r16, %r15, %p1;
-; SM70-NOFTZ-DISALLOW-NEXT: shr.u32 %r18, %r17, 16;
-; SM70-NOFTZ-DISALLOW-NEXT: shl.b32 %r19, %r18, %r1;
-; SM70-NOFTZ-DISALLOW-NEXT: and.b32 %r20, %r23, %r2;
-; SM70-NOFTZ-DISALLOW-NEXT: or.b32 %r21, %r20, %r19;
-; SM70-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r23, %r21;
-; SM70-NOFTZ-DISALLOW-NEXT: setp.ne.b32 %p2, %r3, %r23;
-; SM70-NOFTZ-DISALLOW-NEXT: mov.b32 %r23, %r3;
+; SM70-NOFTZ-DISALLOW-NEXT: or.b32 %r17, %r12, 4194304;
+; SM70-NOFTZ-DISALLOW-NEXT: selp.b32 %r18, %r17, %r16, %p1;
+; SM70-NOFTZ-DISALLOW-NEXT: shr.u32 %r19, %r18, 16;
+; SM70-NOFTZ-DISALLOW-NEXT: shl.b32 %r20, %r19, %r1;
+; SM70-NOFTZ-DISALLOW-NEXT: and.b32 %r21, %r24, %r2;
+; SM70-NOFTZ-DISALLOW-NEXT: or.b32 %r22, %r21, %r20;
+; SM70-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r24, %r22;
+; SM70-NOFTZ-DISALLOW-NEXT: setp.ne.b32 %p2, %r3, %r24;
+; SM70-NOFTZ-DISALLOW-NEXT: mov.b32 %r24, %r3;
; SM70-NOFTZ-DISALLOW-NEXT: @%p2 bra $L__BB79_1;
; SM70-NOFTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
-; SM70-NOFTZ-DISALLOW-NEXT: shr.u32 %r22, %r3, %r1;
+; SM70-NOFTZ-DISALLOW-NEXT: shr.u32 %r23, %r3, %r1;
; SM70-NOFTZ-DISALLOW-NEXT: fence.acq_rel.cta;
-; SM70-NOFTZ-DISALLOW-NEXT: st.param.b16 [func_retval0], %r22;
+; SM70-NOFTZ-DISALLOW-NEXT: st.param.b16 [func_retval0], %r23;
; SM70-NOFTZ-DISALLOW-NEXT: ret;
;
; SM70-NOFTZ-ALLOW-LABEL: fsub_acq_rel_bfloat_global_cta(
; SM70-NOFTZ-ALLOW: {
; SM70-NOFTZ-ALLOW-NEXT: .reg .pred %p<3>;
; SM70-NOFTZ-ALLOW-NEXT: .reg .b16 %rs<2>;
-; SM70-NOFTZ-ALLOW-NEXT: .reg .b32 %r<24>;
+; SM70-NOFTZ-ALLOW-NEXT: .reg .b32 %r<25>;
; SM70-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
; SM70-NOFTZ-ALLOW-EMPTY:
; SM70-NOFTZ-ALLOW-NEXT: // %bb.0:
@@ -3483,39 +3488,40 @@ define bfloat @fsub_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat %va
; SM70-NOFTZ-ALLOW-NEXT: mov.b32 %r6, 65535;
; SM70-NOFTZ-ALLOW-NEXT: shl.b32 %r7, %r6, %r1;
; SM70-NOFTZ-ALLOW-NEXT: not.b32 %r2, %r7;
-; SM70-NOFTZ-ALLOW-NEXT: ld.relaxed.cta.global.b32 %r23, [%rd1];
+; SM70-NOFTZ-ALLOW-NEXT: ld.relaxed.cta.global.b32 %r24, [%rd1];
; SM70-NOFTZ-ALLOW-NEXT: cvt.u32.u16 %r10, %rs1;
; SM70-NOFTZ-ALLOW-NEXT: shl.b32 %r11, %r10, 16;
; SM70-NOFTZ-ALLOW-NEXT: $L__BB79_1: // %atomicrmw.start
; SM70-NOFTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NOFTZ-ALLOW-NEXT: shr.u32 %r8, %r23, %r1;
+; SM70-NOFTZ-ALLOW-NEXT: shr.u32 %r8, %r24, %r1;
; SM70-NOFTZ-ALLOW-NEXT: shl.b32 %r9, %r8, 16;
; SM70-NOFTZ-ALLOW-NEXT: sub.rn.f32 %r12, %r9, %r11;
-; SM70-NOFTZ-ALLOW-NEXT: bfe.u32 %r13, %r12, 16, 1;
-; SM70-NOFTZ-ALLOW-NEXT: add.s32 %r14, %r13, %r12;
-; SM70-NOFTZ-ALLOW-NEXT: add.s32 %r15, %r14, 32767;
+; SM70-NOFTZ-ALLOW-NEXT: shr.u32 %r13, %r12, 16;
+; SM70-NOFTZ-ALLOW-NEXT: and.b32 %r14, %r13, 1;
+; SM70-NOFTZ-ALLOW-NEXT: add.s32 %r15, %r14, %r12;
+; SM70-NOFTZ-ALLOW-NEXT: add.s32 %r16, %r15, 32767;
; SM70-NOFTZ-ALLOW-NEXT: setp.nan.f32 %p1, %r12, %r12;
-; SM70-NOFTZ-ALLOW-NEXT: or.b32 %r16, %r12, 4194304;
-; SM70-NOFTZ-ALLOW-NEXT: selp.b32 %r17, %r16, %r15, %p1;
-; SM70-NOFTZ-ALLOW-NEXT: shr.u32 %r18, %r17, 16;
-; SM70-NOFTZ-ALLOW-NEXT: shl.b32 %r19, %r18, %r1;
-; SM70-NOFTZ-ALLOW-NEXT: and.b32 %r20, %r23, %r2;
-; SM70-NOFTZ-ALLOW-NEXT: or.b32 %r21, %r20, %r19;
-; SM70-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r23, %r21;
-; SM70-NOFTZ-ALLOW-NEXT: setp.ne.b32 %p2, %r3, %r23;
-; SM70-NOFTZ-ALLOW-NEXT: mov.b32 %r23, %r3;
+; SM70-NOFTZ-ALLOW-NEXT: or.b32 %r17, %r12, 4194304;
+; SM70-NOFTZ-ALLOW-NEXT: selp.b32 %r18, %r17, %r16, %p1;
+; SM70-NOFTZ-ALLOW-NEXT: shr.u32 %r19, %r18, 16;
+; SM70-NOFTZ-ALLOW-NEXT: shl.b32 %r20, %r19, %r1;
+; SM70-NOFTZ-ALLOW-NEXT: and.b32 %r21, %r24, %r2;
+; SM70-NOFTZ-ALLOW-NEXT: or.b32 %r22, %r21, %r20;
+; SM70-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r24, %r22;
+; SM70-NOFTZ-ALLOW-NEXT: setp.ne.b32 %p2, %r3, %r24;
+; SM70-NOFTZ-ALLOW-NEXT: mov.b32 %r24, %r3;
; SM70-NOFTZ-ALLOW-NEXT: @%p2 bra $L__BB79_1;
; SM70-NOFTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
-; SM70-NOFTZ-ALLOW-NEXT: shr.u32 %r22, %r3, %r1;
+; SM70-NOFTZ-ALLOW-NEXT: shr.u32 %r23, %r3, %r1;
; SM70-NOFTZ-ALLOW-NEXT: fence.acq_rel.cta;
-; SM70-NOFTZ-ALLOW-NEXT: st.param.b16 [func_retval0], %r22;
+; SM70-NOFTZ-ALLOW-NEXT: st.param.b16 [func_retval0], %r23;
; SM70-NOFTZ-ALLOW-NEXT: ret;
;
; SM70-FTZ-DISALLOW-LABEL: fsub_acq_rel_bfloat_global_cta(
; SM70-FTZ-DISALLOW: {
; SM70-FTZ-DISALLOW-NEXT: .reg .pred %p<3>;
; SM70-FTZ-DISALLOW-NEXT: .reg .b16 %rs<2>;
-; SM70-FTZ-DISALLOW-NEXT: .reg .b32 %r<24>;
+; SM70-FTZ-DISALLOW-NEXT: .reg .b32 %r<25>;
; SM70-FTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
; SM70-FTZ-DISALLOW-EMPTY:
; SM70-FTZ-DISALLOW-NEXT: // %bb.0:
@@ -3529,39 +3535,40 @@ define bfloat @fsub_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat %va
; SM70-FTZ-DISALLOW-NEXT: mov.b32 %r6, 65535;
; SM70-FTZ-DISALLOW-NEXT: shl.b32 %r7, %r6, %r1;
; SM70-FTZ-DISALLOW-NEXT: not.b32 %r2, %r7;
-; SM70-FTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b32 %r23, [%rd1];
+; SM70-FTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b32 %r24, [%rd1];
; SM70-FTZ-DISALLOW-NEXT: cvt.u32.u16 %r10, %rs1;
; SM70-FTZ-DISALLOW-NEXT: shl.b32 %r11, %r10, 16;
; SM70-FTZ-DISALLOW-NEXT: $L__BB79_1: // %atomicrmw.start
; SM70-FTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-FTZ-DISALLOW-NEXT: shr.u32 %r8, %r23, %r1;
+; SM70-FTZ-DISALLOW-NEXT: shr.u32 %r8, %r24, %r1;
; SM70-FTZ-DISALLOW-NEXT: shl.b32 %r9, %r8, 16;
; SM70-FTZ-DISALLOW-NEXT: sub.rn.ftz.f32 %r12, %r9, %r11;
-; SM70-FTZ-DISALLOW-NEXT: bfe.u32 %r13, %r12, 16, 1;
-; SM70-FTZ-DISALLOW-NEXT: add.s32 %r14, %r13, %r12;
-; SM70-FTZ-DISALLOW-NEXT: add.s32 %r15, %r14, 32767;
+; SM70-FTZ-DISALLOW-NEXT: shr.u32 %r13, %r12, 16;
+; SM70-FTZ-DISALLOW-NEXT: and.b32 %r14, %r13, 1;
+; SM70-FTZ-DISALLOW-NEXT: add.s32 %r15, %r14, %r12;
+; SM70-FTZ-DISALLOW-NEXT: add.s32 %r16, %r15, 32767;
; SM70-FTZ-DISALLOW-NEXT: setp.nan.ftz.f32 %p1, %r12, %r12;
-; SM70-FTZ-DISALLOW-NEXT: or.b32 %r16, %r12, 4194304;
-; SM70-FTZ-DISALLOW-NEXT: selp.b32 %r17, %r16, %r15, %p1;
-; SM70-FTZ-DISALLOW-NEXT: shr.u32 %r18, %r17, 16;
-; SM70-FTZ-DISALLOW-NEXT: shl.b32 %r19, %r18, %r1;
-; SM70-FTZ-DISALLOW-NEXT: and.b32 %r20, %r23, %r2;
-; SM70-FTZ-DISALLOW-NEXT: or.b32 %r21, %r20, %r19;
-; SM70-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r23, %r21;
-; SM70-FTZ-DISALLOW-NEXT: setp.ne.b32 %p2, %r3, %r23;
-; SM70-FTZ-DISALLOW-NEXT: mov.b32 %r23, %r3;
+; SM70-FTZ-DISALLOW-NEXT: or.b32 %r17, %r12, 4194304;
+; SM70-FTZ-DISALLOW-NEXT: selp.b32 %r18, %r17, %r16, %p1;
+; SM70-FTZ-DISALLOW-NEXT: shr.u32 %r19, %r18, 16;
+; SM70-FTZ-DISALLOW-NEXT: shl.b32 %r20, %r19, %r1;
+; SM70-FTZ-DISALLOW-NEXT: and.b32 %r21, %r24, %r2;
+; SM70-FTZ-DISALLOW-NEXT: or.b32 %r22, %r21, %r20;
+; SM70-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r24, %r22;
+; SM70-FTZ-DISALLOW-NEXT: setp.ne.b32 %p2, %r3, %r24;
+; SM70-FTZ-DISALLOW-NEXT: mov.b32 %r24, %r3;
; SM70-FTZ-DISALLOW-NEXT: @%p2 bra $L__BB79_1;
; SM70-FTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
-; SM70-FTZ-DISALLOW-NEXT: shr.u32 %r22, %r3, %r1;
+; SM70-FTZ-DISALLOW-NEXT: shr.u32 %r23, %r3, %r1;
; SM70-FTZ-DISALLOW-NEXT: fence.acq_rel.cta;
-; SM70-FTZ-DISALLOW-NEXT: st.param.b16 [func_retval0], %r22;
+; SM70-FTZ-DISALLOW-NEXT: st.param.b16 [func_retval0], %r23;
; SM70-FTZ-DISALLOW-NEXT: ret;
;
; SM70-FTZ-ALLOW-LABEL: fsub_acq_rel_bfloat_global_cta(
; SM70-FTZ-ALLOW: {
; SM70-FTZ-ALLOW-NEXT: .reg .pred %p<3>;
; SM70-FTZ-ALLOW-NEXT: .reg .b16 %rs<2>;
-; SM70-FTZ-ALLOW-NEXT: .reg .b32 %r<24>;
+; SM70-FTZ-ALLOW-NEXT: .reg .b32 %r<25>;
; SM70-FTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
; SM70-FTZ-ALLOW-EMPTY:
; SM70-FTZ-ALLOW-NEXT: // %bb.0:
@@ -3575,32 +3582,33 @@ define bfloat @fsub_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat %va
; SM70-FTZ-ALLOW-NEXT: mov.b32 %r6, 65535;
; SM70-FTZ-ALLOW-NEXT: shl.b32 %r7, %r6, %r1;
; SM70-FTZ-ALLOW-NEXT: not.b32 %r2, %r7;
-; SM70-FTZ-ALLOW-NEXT: ld.relaxed.cta.global.b32 %r23, [%rd1];
+; SM70-FTZ-ALLOW-NEXT: ld.relaxed.cta.global.b32 %r24, [%rd1];
; SM70-FTZ-ALLOW-NEXT: cvt.u32.u16 %r10, %rs1;
; SM70-FTZ-ALLOW-NEXT: shl.b32 %r11, %r10, 16;
; SM70-FTZ-ALLOW-NEXT: $L__BB79_1: // %atomicrmw.start
; SM70-FTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-FTZ-ALLOW-NEXT: shr.u32 %r8, %r23, %r1;
+; SM70-FTZ-ALLOW-NEXT: shr.u32 %r8, %r24, %r1;
; SM70-FTZ-ALLOW-NEXT: shl.b32 %r9, %r8, 16;
; SM70-FTZ-ALLOW-NEXT: sub.rn.ftz.f32 %r12, %r9, %r11;
-; SM70-FTZ-ALLOW-NEXT: bfe.u32 %r13, %r12, 16, 1;
-; SM70-FTZ-ALLOW-NEXT: add.s32 %r14, %r13, %r12;
-; SM70-FTZ-ALLOW-NEXT: add.s32 %r15, %r14, 32767;
+; SM70-FTZ-ALLOW-NEXT: shr.u32 %r13, %r12, 16;
+; SM70-FTZ-ALLOW-NEXT: and.b32 %r14, %r13, 1;
+; SM70-FTZ-ALLOW-NEXT: add.s32 %r15, %r14, %r12;
+; SM70-FTZ-ALLOW-NEXT: add.s32 %r16, %r15, 32767;
; SM70-FTZ-ALLOW-NEXT: setp.nan.ftz.f32 %p1, %r12, %r12;
-; SM70-FTZ-ALLOW-NEXT: or.b32 %r16, %r12, 4194304;
-; SM70-FTZ-ALLOW-NEXT: selp.b32 %r17, %r16, %r15, %p1;
-; SM70-FTZ-ALLOW-NEXT: shr.u32 %r18, %r17, 16;
-; SM70-FTZ-ALLOW-NEXT: shl.b32 %r19, %r18, %r1;
-; SM70-FTZ-ALLOW-NEXT: and.b32 %r20, %r23, %r2;
-; SM70-FTZ-ALLOW-NEXT: or.b32 %r21, %r20, %r19;
-; SM70-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r23, %r21;
-; SM70-FTZ-ALLOW-NEXT: setp.ne.b32 %p2, %r3, %r23;
-; SM70-FTZ-ALLOW-NEXT: mov.b32 %r23, %r3;
+; SM70-FTZ-ALLOW-NEXT: or.b32 %r17, %r12, 4194304;
+; SM70-FTZ-ALLOW-NEXT: selp.b32 %r18, %r17, %r16, %p1;
+; SM70-FTZ-ALLOW-NEXT: shr.u32 %r19, %r18, 16;
+; SM70-FTZ-ALLOW-NEXT: shl.b32 %r20, %r19, %r1;
+; SM70-FTZ-ALLOW-NEXT: and.b32 %r21, %r24, %r2;
+; SM70-FTZ-ALLOW-NEXT: or.b32 %r22, %r21, %r20;
+; SM70-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r24, %r22;
+; SM70-FTZ-ALLOW-NEXT: setp.ne.b32 %p2, %r3, %r24;
+; SM70-FTZ-ALLOW-NEXT: mov.b32 %r24, %r3;
; SM70-FTZ-ALLOW-NEXT: @%p2 bra $L__BB79_1;
; SM70-FTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
-; SM70-FTZ-ALLOW-NEXT: shr.u32 %r22, %r3, %r1;
+; SM70-FTZ-ALLOW-NEXT: shr.u32 %r23, %r3, %r1;
; SM70-FTZ-ALLOW-NEXT: fence.acq_rel.cta;
-; SM70-FTZ-ALLOW-NEXT: st.param.b16 [func_retval0], %r22;
+; SM70-FTZ-ALLOW-NEXT: st.param.b16 [func_retval0], %r23;
; SM70-FTZ-ALLOW-NEXT: ret;
%retval = atomicrmw fsub ptr addrspace(1) %addr, bfloat %val syncscope("block") acq_rel
ret bfloat %retval
@@ -3611,7 +3619,7 @@ define bfloat @fmin_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat %va
; SM70-NOFTZ-DISALLOW: {
; SM70-NOFTZ-DISALLOW-NEXT: .reg .pred %p<3>;
; SM70-NOFTZ-DISALLOW-NEXT: .reg .b16 %rs<2>;
-; SM70-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<24>;
+; SM70-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<25>;
; SM70-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
; SM70-NOFTZ-DISALLOW-EMPTY:
; SM70-NOFTZ-DISALLOW-NEXT: // %bb.0:
@@ -3625,39 +3633,40 @@ define bfloat @fmin_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat %va
; SM70-NOFTZ-DISALLOW-NEXT: mov.b32 %r6, 65535;
; SM70-NOFTZ-DISALLOW-NEXT: shl.b32 %r7, %r6, %r1;
; SM70-NOFTZ-DISALLOW-NEXT: not.b32 %r2, %r7;
-; SM70-NOFTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b32 %r23, [%rd1];
+; SM70-NOFTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b32 %r24, [%rd1];
; SM70-NOFTZ-DISALLOW-NEXT: cvt.u32.u16 %r10, %rs1;
; SM70-NOFTZ-DISALLOW-NEXT: shl.b32 %r11, %r10, 16;
; SM70-NOFTZ-DISALLOW-NEXT: $L__BB80_1: // %atomicrmw.start
; SM70-NOFTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NOFTZ-DISALLOW-NEXT: shr.u32 %r8, %r23, %r1;
+; SM70-NOFTZ-DISALLOW-NEXT: shr.u32 %r8, %r24, %r1;
; SM70-NOFTZ-DISALLOW-NEXT: shl.b32 %r9, %r8, 16;
; SM70-NOFTZ-DISALLOW-NEXT: min.f32 %r12, %r9, %r11;
-; SM70-NOFTZ-DISALLOW-NEXT: bfe.u32 %r13, %r12, 16, 1;
-; SM70-NOFTZ-DISALLOW-NEXT: add.s32 %r14, %r13, %r12;
-; SM70-NOFTZ-DISALLOW-NEXT: add.s32 %r15, %r14, 32767;
+; SM70-NOFTZ-DISALLOW-NEXT: shr.u32 %r13, %r12, 16;
+; SM70-NOFTZ-DISALLOW-NEXT: and.b32 %r14, %r13, 1;
+; SM70-NOFTZ-DISALLOW-NEXT: add.s32 %r15, %r14, %r12;
+; SM70-NOFTZ-DISALLOW-NEXT: add.s32 %r16, %r15, 32767;
; SM70-NOFTZ-DISALLOW-NEXT: setp.nan.f32 %p1, %r12, %r12;
-; SM70-NOFTZ-DISALLOW-NEXT: or.b32 %r16, %r12, 4194304;
-; SM70-NOFTZ-DISALLOW-NEXT: selp.b32 %r17, %r16, %r15, %p1;
-; SM70-NOFTZ-DISALLOW-NEXT: shr.u32 %r18, %r17, 16;
-; SM70-NOFTZ-DISALLOW-NEXT: shl.b32 %r19, %r18, %r1;
-; SM70-NOFTZ-DISALLOW-NEXT: and.b32 %r20, %r23, %r2;
-; SM70-NOFTZ-DISALLOW-NEXT: or.b32 %r21, %r20, %r19;
-; SM70-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r23, %r21;
-; SM70-NOFTZ-DISALLOW-NEXT: setp.ne.b32 %p2, %r3, %r23;
-; SM70-NOFTZ-DISALLOW-NEXT: mov.b32 %r23, %r3;
+; SM70-NOFTZ-DISALLOW-NEXT: or.b32 %r17, %r12, 4194304;
+; SM70-NOFTZ-DISALLOW-NEXT: selp.b32 %r18, %r17, %r16, %p1;
+; SM70-NOFTZ-DISALLOW-NEXT: shr.u32 %r19, %r18, 16;
+; SM70-NOFTZ-DISALLOW-NEXT: shl.b32 %r20, %r19, %r1;
+; SM70-NOFTZ-DISALLOW-NEXT: and.b32 %r21, %r24, %r2;
+; SM70-NOFTZ-DISALLOW-NEXT: or.b32 %r22, %r21, %r20;
+; SM70-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r24, %r22;
+; SM70-NOFTZ-DISALLOW-NEXT: setp.ne.b32 %p2, %r3, %r24;
+; SM70-NOFTZ-DISALLOW-NEXT: mov.b32 %r24, %r3;
; SM70-NOFTZ-DISALLOW-NEXT: @%p2 bra $L__BB80_1;
; SM70-NOFTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
-; SM70-NOFTZ-DISALLOW-NEXT: shr.u32 %r22, %r3, %r1;
+; SM70-NOFTZ-DISALLOW-NEXT: shr.u32 %r23, %r3, %r1;
; SM70-NOFTZ-DISALLOW-NEXT: fence.acq_rel.cta;
-; SM70-NOFTZ-DISALLOW-NEXT: st.param.b16 [func_retval0], %r22;
+; SM70-NOFTZ-DISALLOW-NEXT: st.param.b16 [func_retval0], %r23;
; SM70-NOFTZ-DISALLOW-NEXT: ret;
;
; SM70-NOFTZ-ALLOW-LABEL: fmin_acq_rel_bfloat_global_cta(
; SM70-NOFTZ-ALLOW: {
; SM70-NOFTZ-ALLOW-NEXT: .reg .pred %p<3>;
; SM70-NOFTZ-ALLOW-NEXT: .reg .b16 %rs<2>;
-; SM70-NOFTZ-ALLOW-NEXT: .reg .b32 %r<24>;
+; SM70-NOFTZ-ALLOW-NEXT: .reg .b32 %r<25>;
; SM70-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
; SM70-NOFTZ-ALLOW-EMPTY:
; SM70-NOFTZ-ALLOW-NEXT: // %bb.0:
@@ -3671,39 +3680,40 @@ define bfloat @fmin_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat %va
; SM70-NOFTZ-ALLOW-NEXT: mov.b32 %r6, 65535;
; SM70-NOFTZ-ALLOW-NEXT: shl.b32 %r7, %r6, %r1;
; SM70-NOFTZ-ALLOW-NEXT: not.b32 %r2, %r7;
-; SM70-NOFTZ-ALLOW-NEXT: ld.relaxed.cta.global.b32 %r23, [%rd1];
+; SM70-NOFTZ-ALLOW-NEXT: ld.relaxed.cta.global.b32 %r24, [%rd1];
; SM70-NOFTZ-ALLOW-NEXT: cvt.u32.u16 %r10, %rs1;
; SM70-NOFTZ-ALLOW-NEXT: shl.b32 %r11, %r10, 16;
; SM70-NOFTZ-ALLOW-NEXT: $L__BB80_1: // %atomicrmw.start
; SM70-NOFTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NOFTZ-ALLOW-NEXT: shr.u32 %r8, %r23, %r1;
+; SM70-NOFTZ-ALLOW-NEXT: shr.u32 %r8, %r24, %r1;
; SM70-NOFTZ-ALLOW-NEXT: shl.b32 %r9, %r8, 16;
; SM70-NOFTZ-ALLOW-NEXT: min.f32 %r12, %r9, %r11;
-; SM70-NOFTZ-ALLOW-NEXT: bfe.u32 %r13, %r12, 16, 1;
-; SM70-NOFTZ-ALLOW-NEXT: add.s32 %r14, %r13, %r12;
-; SM70-NOFTZ-ALLOW-NEXT: add.s32 %r15, %r14, 32767;
+; SM70-NOFTZ-ALLOW-NEXT: shr.u32 %r13, %r12, 16;
+; SM70-NOFTZ-ALLOW-NEXT: and.b32 %r14, %r13, 1;
+; SM70-NOFTZ-ALLOW-NEXT: add.s32 %r15, %r14, %r12;
+; SM70-NOFTZ-ALLOW-NEXT: add.s32 %r16, %r15, 32767;
; SM70-NOFTZ-ALLOW-NEXT: setp.nan.f32 %p1, %r12, %r12;
-; SM70-NOFTZ-ALLOW-NEXT: or.b32 %r16, %r12, 4194304;
-; SM70-NOFTZ-ALLOW-NEXT: selp.b32 %r17, %r16, %r15, %p1;
-; SM70-NOFTZ-ALLOW-NEXT: shr.u32 %r18, %r17, 16;
-; SM70-NOFTZ-ALLOW-NEXT: shl.b32 %r19, %r18, %r1;
-; SM70-NOFTZ-ALLOW-NEXT: and.b32 %r20, %r23, %r2;
-; SM70-NOFTZ-ALLOW-NEXT: or.b32 %r21, %r20, %r19;
-; SM70-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r23, %r21;
-; SM70-NOFTZ-ALLOW-NEXT: setp.ne.b32 %p2, %r3, %r23;
-; SM70-NOFTZ-ALLOW-NEXT: mov.b32 %r23, %r3;
+; SM70-NOFTZ-ALLOW-NEXT: or.b32 %r17, %r12, 4194304;
+; SM70-NOFTZ-ALLOW-NEXT: selp.b32 %r18, %r17, %r16, %p1;
+; SM70-NOFTZ-ALLOW-NEXT: shr.u32 %r19, %r18, 16;
+; SM70-NOFTZ-ALLOW-NEXT: shl.b32 %r20, %r19, %r1;
+; SM70-NOFTZ-ALLOW-NEXT: and.b32 %r21, %r24, %r2;
+; SM70-NOFTZ-ALLOW-NEXT: or.b32 %r22, %r21, %r20;
+; SM70-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r24, %r22;
+; SM70-NOFTZ-ALLOW-NEXT: setp.ne.b32 %p2, %r3, %r24;
+; SM70-NOFTZ-ALLOW-NEXT: mov.b32 %r24, %r3;
; SM70-NOFTZ-ALLOW-NEXT: @%p2 bra $L__BB80_1;
; SM70-NOFTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
-; SM70-NOFTZ-ALLOW-NEXT: shr.u32 %r22, %r3, %r1;
+; SM70-NOFTZ-ALLOW-NEXT: shr.u32 %r23, %r3, %r1;
; SM70-NOFTZ-ALLOW-NEXT: fence.acq_rel.cta;
-; SM70-NOFTZ-ALLOW-NEXT: st.param.b16 [func_retval0], %r22;
+; SM70-NOFTZ-ALLOW-NEXT: st.param.b16 [func_retval0], %r23;
; SM70-NOFTZ-ALLOW-NEXT: ret;
;
; SM70-FTZ-DISALLOW-LABEL: fmin_acq_rel_bfloat_global_cta(
; SM70-FTZ-DISALLOW: {
; SM70-FTZ-DISALLOW-NEXT: .reg .pred %p<3>;
; SM70-FTZ-DISALLOW-NEXT: .reg .b16 %rs<2>;
-; SM70-FTZ-DISALLOW-NEXT: .reg .b32 %r<24>;
+; SM70-FTZ-DISALLOW-NEXT: .reg .b32 %r<25>;
; SM70-FTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
; SM70-FTZ-DISALLOW-EMPTY:
; SM70-FTZ-DISALLOW-NEXT: // %bb.0:
@@ -3717,39 +3727,40 @@ define bfloat @fmin_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat %va
; SM70-FTZ-DISALLOW-NEXT: mov.b32 %r6, 65535;
; SM70-FTZ-DISALLOW-NEXT: shl.b32 %r7, %r6, %r1;
; SM70-FTZ-DISALLOW-NEXT: not.b32 %r2, %r7;
-; SM70-FTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b32 %r23, [%rd1];
+; SM70-FTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b32 %r24, [%rd1];
; SM70-FTZ-DISALLOW-NEXT: cvt.u32.u16 %r10, %rs1;
; SM70-FTZ-DISALLOW-NEXT: shl.b32 %r11, %r10, 16;
; SM70-FTZ-DISALLOW-NEXT: $L__BB80_1: // %atomicrmw.start
; SM70-FTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-FTZ-DISALLOW-NEXT: shr.u32 %r8, %r23, %r1;
+; SM70-FTZ-DISALLOW-NEXT: shr.u32 %r8, %r24, %r1;
; SM70-FTZ-DISALLOW-NEXT: shl.b32 %r9, %r8, 16;
; SM70-FTZ-DISALLOW-NEXT: min.ftz.f32 %r12, %r9, %r11;
-; SM70-FTZ-DISALLOW-NEXT: bfe.u32 %r13, %r12, 16, 1;
-; SM70-FTZ-DISALLOW-NEXT: add.s32 %r14, %r13, %r12;
-; SM70-FTZ-DISALLOW-NEXT: add.s32 %r15, %r14, 32767;
+; SM70-FTZ-DISALLOW-NEXT: shr.u32 %r13, %r12, 16;
+; SM70-FTZ-DISALLOW-NEXT: and.b32 %r14, %r13, 1;
+; SM70-FTZ-DISALLOW-NEXT: add.s32 %r15, %r14, %r12;
+; SM70-FTZ-DISALLOW-NEXT: add.s32 %r16, %r15, 32767;
; SM70-FTZ-DISALLOW-NEXT: setp.nan.ftz.f32 %p1, %r12, %r12;
-; SM70-FTZ-DISALLOW-NEXT: or.b32 %r16, %r12, 4194304;
-; SM70-FTZ-DISALLOW-NEXT: selp.b32 %r17, %r16, %r15, %p1;
-; SM70-FTZ-DISALLOW-NEXT: shr.u32 %r18, %r17, 16;
-; SM70-FTZ-DISALLOW-NEXT: shl.b32 %r19, %r18, %r1;
-; SM70-FTZ-DISALLOW-NEXT: and.b32 %r20, %r23, %r2;
-; SM70-FTZ-DISALLOW-NEXT: or.b32 %r21, %r20, %r19;
-; SM70-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r23, %r21;
-; SM70-FTZ-DISALLOW-NEXT: setp.ne.b32 %p2, %r3, %r23;
-; SM70-FTZ-DISALLOW-NEXT: mov.b32 %r23, %r3;
+; SM70-FTZ-DISALLOW-NEXT: or.b32 %r17, %r12, 4194304;
+; SM70-FTZ-DISALLOW-NEXT: selp.b32 %r18, %r17, %r16, %p1;
+; SM70-FTZ-DISALLOW-NEXT: shr.u32 %r19, %r18, 16;
+; SM70-FTZ-DISALLOW-NEXT: shl.b32 %r20, %r19, %r1;
+; SM70-FTZ-DISALLOW-NEXT: and.b32 %r21, %r24, %r2;
+; SM70-FTZ-DISALLOW-NEXT: or.b32 %r22, %r21, %r20;
+; SM70-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r24, %r22;
+; SM70-FTZ-DISALLOW-NEXT: setp.ne.b32 %p2, %r3, %r24;
+; SM70-FTZ-DISALLOW-NEXT: mov.b32 %r24, %r3;
; SM70-FTZ-DISALLOW-NEXT: @%p2 bra $L__BB80_1;
; SM70-FTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
-; SM70-FTZ-DISALLOW-NEXT: shr.u32 %r22, %r3, %r1;
+; SM70-FTZ-DISALLOW-NEXT: shr.u32 %r23, %r3, %r1;
; SM70-FTZ-DISALLOW-NEXT: fence.acq_rel.cta;
-; SM70-FTZ-DISALLOW-NEXT: st.param.b16 [func_retval0], %r22;
+; SM70-FTZ-DISALLOW-NEXT: st.param.b16 [func_retval0], %r23;
; SM70-FTZ-DISALLOW-NEXT: ret;
;
; SM70-FTZ-ALLOW-LABEL: fmin_acq_rel_bfloat_global_cta(
; SM70-FTZ-ALLOW: {
; SM70-FTZ-ALLOW-NEXT: .reg .pred %p<3>;
; SM70-FTZ-ALLOW-NEXT: .reg .b16 %rs<2>;
-; SM70-FTZ-ALLOW-NEXT: .reg .b32 %r<24>;
+; SM70-FTZ-ALLOW-NEXT: .reg .b32 %r<25>;
; SM70-FTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
; SM70-FTZ-ALLOW-EMPTY:
; SM70-FTZ-ALLOW-NEXT: // %bb.0:
@@ -3763,32 +3774,33 @@ define bfloat @fmin_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat %va
; SM70-FTZ-ALLOW-NEXT: mov.b32 %r6, 65535;
; SM70-FTZ-ALLOW-NEXT: shl.b32 %r7, %r6, %r1;
; SM70-FTZ-ALLOW-NEXT: not.b32 %r2, %r7;
-; SM70-FTZ-ALLOW-NEXT: ld.relaxed.cta.global.b32 %r23, [%rd1];
+; SM70-FTZ-ALLOW-NEXT: ld.relaxed.cta.global.b32 %r24, [%rd1];
; SM70-FTZ-ALLOW-NEXT: cvt.u32.u16 %r10, %rs1;
; SM70-FTZ-ALLOW-NEXT: shl.b32 %r11, %r10, 16;
; SM70-FTZ-ALLOW-NEXT: $L__BB80_1: // %atomicrmw.start
; SM70-FTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-FTZ-ALLOW-NEXT: shr.u32 %r8, %r23, %r1;
+; SM70-FTZ-ALLOW-NEXT: shr.u32 %r8, %r24, %r1;
; SM70-FTZ-ALLOW-NEXT: shl.b32 %r9, %r8, 16;
; SM70-FTZ-ALLOW-NEXT: min.ftz.f32 %r12, %r9, %r11;
-; SM70-FTZ-ALLOW-NEXT: bfe.u32 %r13, %r12, 16, 1;
-; SM70-FTZ-ALLOW-NEXT: add.s32 %r14, %r13, %r12;
-; SM70-FTZ-ALLOW-NEXT: add.s32 %r15, %r14, 32767;
+; SM70-FTZ-ALLOW-NEXT: shr.u32 %r13, %r12, 16;
+; SM70-FTZ-ALLOW-NEXT: and.b32 %r14, %r13, 1;
+; SM70-FTZ-ALLOW-NEXT: add.s32 %r15, %r14, %r12;
+; SM70-FTZ-ALLOW-NEXT: add.s32 %r16, %r15, 32767;
; SM70-FTZ-ALLOW-NEXT: setp.nan.ftz.f32 %p1, %r12, %r12;
-; SM70-FTZ-ALLOW-NEXT: or.b32 %r16, %r12, 4194304;
-; SM70-FTZ-ALLOW-NEXT: selp.b32 %r17, %r16, %r15, %p1;
-; SM70-FTZ-ALLOW-NEXT: shr.u32 %r18, %r17, 16;
-; SM70-FTZ-ALLOW-NEXT: shl.b32 %r19, %r18, %r1;
-; SM70-FTZ-ALLOW-NEXT: and.b32 %r20, %r23, %r2;
-; SM70-FTZ-ALLOW-NEXT: or.b32 %r21, %r20, %r19;
-; SM70-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r23, %r21;
-; SM70-FTZ-ALLOW-NEXT: setp.ne.b32 %p2, %r3, %r23;
-; SM70-FTZ-ALLOW-NEXT: mov.b32 %r23, %r3;
+; SM70-FTZ-ALLOW-NEXT: or.b32 %r17, %r12, 4194304;
+; SM70-FTZ-ALLOW-NEXT: selp.b32 %r18, %r17, %r16, %p1;
+; SM70-FTZ-ALLOW-NEXT: shr.u32 %r19, %r18, 16;
+; SM70-FTZ-ALLOW-NEXT: shl.b32 %r20, %r19, %r1;
+; SM70-FTZ-ALLOW-NEXT: and.b32 %r21, %r24, %r2;
+; SM70-FTZ-ALLOW-NEXT: or.b32 %r22, %r21, %r20;
+; SM70-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r24, %r22;
+; SM70-FTZ-ALLOW-NEXT: setp.ne.b32 %p2, %r3, %r24;
+; SM70-FTZ-ALLOW-NEXT: mov.b32 %r24, %r3;
; SM70-FTZ-ALLOW-NEXT: @%p2 bra $L__BB80_1;
; SM70-FTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
-; SM70-FTZ-ALLOW-NEXT: shr.u32 %r22, %r3, %r1;
+; SM70-FTZ-ALLOW-NEXT: shr.u32 %r23, %r3, %r1;
; SM70-FTZ-ALLOW-NEXT: fence.acq_rel.cta;
-; SM70-FTZ-ALLOW-NEXT: st.param.b16 [func_retval0], %r22;
+; SM70-FTZ-ALLOW-NEXT: st.param.b16 [func_retval0], %r23;
; SM70-FTZ-ALLOW-NEXT: ret;
%retval = atomicrmw fmin ptr addrspace(1) %addr, bfloat %val syncscope("block") acq_rel
ret bfloat %retval
@@ -3799,7 +3811,7 @@ define bfloat @fmax_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat %va
; SM70-NOFTZ-DISALLOW: {
; SM70-NOFTZ-DISALLOW-NEXT: .reg .pred %p<3>;
; SM70-NOFTZ-DISALLOW-NEXT: .reg .b16 %rs<2>;
-; SM70-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<24>;
+; SM70-NOFTZ-DISALLOW-NEXT: .reg .b32 %r<25>;
; SM70-NOFTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
; SM70-NOFTZ-DISALLOW-EMPTY:
; SM70-NOFTZ-DISALLOW-NEXT: // %bb.0:
@@ -3813,39 +3825,40 @@ define bfloat @fmax_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat %va
; SM70-NOFTZ-DISALLOW-NEXT: mov.b32 %r6, 65535;
; SM70-NOFTZ-DISALLOW-NEXT: shl.b32 %r7, %r6, %r1;
; SM70-NOFTZ-DISALLOW-NEXT: not.b32 %r2, %r7;
-; SM70-NOFTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b32 %r23, [%rd1];
+; SM70-NOFTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b32 %r24, [%rd1];
; SM70-NOFTZ-DISALLOW-NEXT: cvt.u32.u16 %r10, %rs1;
; SM70-NOFTZ-DISALLOW-NEXT: shl.b32 %r11, %r10, 16;
; SM70-NOFTZ-DISALLOW-NEXT: $L__BB81_1: // %atomicrmw.start
; SM70-NOFTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NOFTZ-DISALLOW-NEXT: shr.u32 %r8, %r23, %r1;
+; SM70-NOFTZ-DISALLOW-NEXT: shr.u32 %r8, %r24, %r1;
; SM70-NOFTZ-DISALLOW-NEXT: shl.b32 %r9, %r8, 16;
; SM70-NOFTZ-DISALLOW-NEXT: max.f32 %r12, %r9, %r11;
-; SM70-NOFTZ-DISALLOW-NEXT: bfe.u32 %r13, %r12, 16, 1;
-; SM70-NOFTZ-DISALLOW-NEXT: add.s32 %r14, %r13, %r12;
-; SM70-NOFTZ-DISALLOW-NEXT: add.s32 %r15, %r14, 32767;
+; SM70-NOFTZ-DISALLOW-NEXT: shr.u32 %r13, %r12, 16;
+; SM70-NOFTZ-DISALLOW-NEXT: and.b32 %r14, %r13, 1;
+; SM70-NOFTZ-DISALLOW-NEXT: add.s32 %r15, %r14, %r12;
+; SM70-NOFTZ-DISALLOW-NEXT: add.s32 %r16, %r15, 32767;
; SM70-NOFTZ-DISALLOW-NEXT: setp.nan.f32 %p1, %r12, %r12;
-; SM70-NOFTZ-DISALLOW-NEXT: or.b32 %r16, %r12, 4194304;
-; SM70-NOFTZ-DISALLOW-NEXT: selp.b32 %r17, %r16, %r15, %p1;
-; SM70-NOFTZ-DISALLOW-NEXT: shr.u32 %r18, %r17, 16;
-; SM70-NOFTZ-DISALLOW-NEXT: shl.b32 %r19, %r18, %r1;
-; SM70-NOFTZ-DISALLOW-NEXT: and.b32 %r20, %r23, %r2;
-; SM70-NOFTZ-DISALLOW-NEXT: or.b32 %r21, %r20, %r19;
-; SM70-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r23, %r21;
-; SM70-NOFTZ-DISALLOW-NEXT: setp.ne.b32 %p2, %r3, %r23;
-; SM70-NOFTZ-DISALLOW-NEXT: mov.b32 %r23, %r3;
+; SM70-NOFTZ-DISALLOW-NEXT: or.b32 %r17, %r12, 4194304;
+; SM70-NOFTZ-DISALLOW-NEXT: selp.b32 %r18, %r17, %r16, %p1;
+; SM70-NOFTZ-DISALLOW-NEXT: shr.u32 %r19, %r18, 16;
+; SM70-NOFTZ-DISALLOW-NEXT: shl.b32 %r20, %r19, %r1;
+; SM70-NOFTZ-DISALLOW-NEXT: and.b32 %r21, %r24, %r2;
+; SM70-NOFTZ-DISALLOW-NEXT: or.b32 %r22, %r21, %r20;
+; SM70-NOFTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r24, %r22;
+; SM70-NOFTZ-DISALLOW-NEXT: setp.ne.b32 %p2, %r3, %r24;
+; SM70-NOFTZ-DISALLOW-NEXT: mov.b32 %r24, %r3;
; SM70-NOFTZ-DISALLOW-NEXT: @%p2 bra $L__BB81_1;
; SM70-NOFTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
-; SM70-NOFTZ-DISALLOW-NEXT: shr.u32 %r22, %r3, %r1;
+; SM70-NOFTZ-DISALLOW-NEXT: shr.u32 %r23, %r3, %r1;
; SM70-NOFTZ-DISALLOW-NEXT: fence.acq_rel.cta;
-; SM70-NOFTZ-DISALLOW-NEXT: st.param.b16 [func_retval0], %r22;
+; SM70-NOFTZ-DISALLOW-NEXT: st.param.b16 [func_retval0], %r23;
; SM70-NOFTZ-DISALLOW-NEXT: ret;
;
; SM70-NOFTZ-ALLOW-LABEL: fmax_acq_rel_bfloat_global_cta(
; SM70-NOFTZ-ALLOW: {
; SM70-NOFTZ-ALLOW-NEXT: .reg .pred %p<3>;
; SM70-NOFTZ-ALLOW-NEXT: .reg .b16 %rs<2>;
-; SM70-NOFTZ-ALLOW-NEXT: .reg .b32 %r<24>;
+; SM70-NOFTZ-ALLOW-NEXT: .reg .b32 %r<25>;
; SM70-NOFTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
; SM70-NOFTZ-ALLOW-EMPTY:
; SM70-NOFTZ-ALLOW-NEXT: // %bb.0:
@@ -3859,39 +3872,40 @@ define bfloat @fmax_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat %va
; SM70-NOFTZ-ALLOW-NEXT: mov.b32 %r6, 65535;
; SM70-NOFTZ-ALLOW-NEXT: shl.b32 %r7, %r6, %r1;
; SM70-NOFTZ-ALLOW-NEXT: not.b32 %r2, %r7;
-; SM70-NOFTZ-ALLOW-NEXT: ld.relaxed.cta.global.b32 %r23, [%rd1];
+; SM70-NOFTZ-ALLOW-NEXT: ld.relaxed.cta.global.b32 %r24, [%rd1];
; SM70-NOFTZ-ALLOW-NEXT: cvt.u32.u16 %r10, %rs1;
; SM70-NOFTZ-ALLOW-NEXT: shl.b32 %r11, %r10, 16;
; SM70-NOFTZ-ALLOW-NEXT: $L__BB81_1: // %atomicrmw.start
; SM70-NOFTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-NOFTZ-ALLOW-NEXT: shr.u32 %r8, %r23, %r1;
+; SM70-NOFTZ-ALLOW-NEXT: shr.u32 %r8, %r24, %r1;
; SM70-NOFTZ-ALLOW-NEXT: shl.b32 %r9, %r8, 16;
; SM70-NOFTZ-ALLOW-NEXT: max.f32 %r12, %r9, %r11;
-; SM70-NOFTZ-ALLOW-NEXT: bfe.u32 %r13, %r12, 16, 1;
-; SM70-NOFTZ-ALLOW-NEXT: add.s32 %r14, %r13, %r12;
-; SM70-NOFTZ-ALLOW-NEXT: add.s32 %r15, %r14, 32767;
+; SM70-NOFTZ-ALLOW-NEXT: shr.u32 %r13, %r12, 16;
+; SM70-NOFTZ-ALLOW-NEXT: and.b32 %r14, %r13, 1;
+; SM70-NOFTZ-ALLOW-NEXT: add.s32 %r15, %r14, %r12;
+; SM70-NOFTZ-ALLOW-NEXT: add.s32 %r16, %r15, 32767;
; SM70-NOFTZ-ALLOW-NEXT: setp.nan.f32 %p1, %r12, %r12;
-; SM70-NOFTZ-ALLOW-NEXT: or.b32 %r16, %r12, 4194304;
-; SM70-NOFTZ-ALLOW-NEXT: selp.b32 %r17, %r16, %r15, %p1;
-; SM70-NOFTZ-ALLOW-NEXT: shr.u32 %r18, %r17, 16;
-; SM70-NOFTZ-ALLOW-NEXT: shl.b32 %r19, %r18, %r1;
-; SM70-NOFTZ-ALLOW-NEXT: and.b32 %r20, %r23, %r2;
-; SM70-NOFTZ-ALLOW-NEXT: or.b32 %r21, %r20, %r19;
-; SM70-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r23, %r21;
-; SM70-NOFTZ-ALLOW-NEXT: setp.ne.b32 %p2, %r3, %r23;
-; SM70-NOFTZ-ALLOW-NEXT: mov.b32 %r23, %r3;
+; SM70-NOFTZ-ALLOW-NEXT: or.b32 %r17, %r12, 4194304;
+; SM70-NOFTZ-ALLOW-NEXT: selp.b32 %r18, %r17, %r16, %p1;
+; SM70-NOFTZ-ALLOW-NEXT: shr.u32 %r19, %r18, 16;
+; SM70-NOFTZ-ALLOW-NEXT: shl.b32 %r20, %r19, %r1;
+; SM70-NOFTZ-ALLOW-NEXT: and.b32 %r21, %r24, %r2;
+; SM70-NOFTZ-ALLOW-NEXT: or.b32 %r22, %r21, %r20;
+; SM70-NOFTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r24, %r22;
+; SM70-NOFTZ-ALLOW-NEXT: setp.ne.b32 %p2, %r3, %r24;
+; SM70-NOFTZ-ALLOW-NEXT: mov.b32 %r24, %r3;
; SM70-NOFTZ-ALLOW-NEXT: @%p2 bra $L__BB81_1;
; SM70-NOFTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
-; SM70-NOFTZ-ALLOW-NEXT: shr.u32 %r22, %r3, %r1;
+; SM70-NOFTZ-ALLOW-NEXT: shr.u32 %r23, %r3, %r1;
; SM70-NOFTZ-ALLOW-NEXT: fence.acq_rel.cta;
-; SM70-NOFTZ-ALLOW-NEXT: st.param.b16 [func_retval0], %r22;
+; SM70-NOFTZ-ALLOW-NEXT: st.param.b16 [func_retval0], %r23;
; SM70-NOFTZ-ALLOW-NEXT: ret;
;
; SM70-FTZ-DISALLOW-LABEL: fmax_acq_rel_bfloat_global_cta(
; SM70-FTZ-DISALLOW: {
; SM70-FTZ-DISALLOW-NEXT: .reg .pred %p<3>;
; SM70-FTZ-DISALLOW-NEXT: .reg .b16 %rs<2>;
-; SM70-FTZ-DISALLOW-NEXT: .reg .b32 %r<24>;
+; SM70-FTZ-DISALLOW-NEXT: .reg .b32 %r<25>;
; SM70-FTZ-DISALLOW-NEXT: .reg .b64 %rd<3>;
; SM70-FTZ-DISALLOW-EMPTY:
; SM70-FTZ-DISALLOW-NEXT: // %bb.0:
@@ -3905,39 +3919,40 @@ define bfloat @fmax_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat %va
; SM70-FTZ-DISALLOW-NEXT: mov.b32 %r6, 65535;
; SM70-FTZ-DISALLOW-NEXT: shl.b32 %r7, %r6, %r1;
; SM70-FTZ-DISALLOW-NEXT: not.b32 %r2, %r7;
-; SM70-FTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b32 %r23, [%rd1];
+; SM70-FTZ-DISALLOW-NEXT: ld.relaxed.cta.global.b32 %r24, [%rd1];
; SM70-FTZ-DISALLOW-NEXT: cvt.u32.u16 %r10, %rs1;
; SM70-FTZ-DISALLOW-NEXT: shl.b32 %r11, %r10, 16;
; SM70-FTZ-DISALLOW-NEXT: $L__BB81_1: // %atomicrmw.start
; SM70-FTZ-DISALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-FTZ-DISALLOW-NEXT: shr.u32 %r8, %r23, %r1;
+; SM70-FTZ-DISALLOW-NEXT: shr.u32 %r8, %r24, %r1;
; SM70-FTZ-DISALLOW-NEXT: shl.b32 %r9, %r8, 16;
; SM70-FTZ-DISALLOW-NEXT: max.ftz.f32 %r12, %r9, %r11;
-; SM70-FTZ-DISALLOW-NEXT: bfe.u32 %r13, %r12, 16, 1;
-; SM70-FTZ-DISALLOW-NEXT: add.s32 %r14, %r13, %r12;
-; SM70-FTZ-DISALLOW-NEXT: add.s32 %r15, %r14, 32767;
+; SM70-FTZ-DISALLOW-NEXT: shr.u32 %r13, %r12, 16;
+; SM70-FTZ-DISALLOW-NEXT: and.b32 %r14, %r13, 1;
+; SM70-FTZ-DISALLOW-NEXT: add.s32 %r15, %r14, %r12;
+; SM70-FTZ-DISALLOW-NEXT: add.s32 %r16, %r15, 32767;
; SM70-FTZ-DISALLOW-NEXT: setp.nan.ftz.f32 %p1, %r12, %r12;
-; SM70-FTZ-DISALLOW-NEXT: or.b32 %r16, %r12, 4194304;
-; SM70-FTZ-DISALLOW-NEXT: selp.b32 %r17, %r16, %r15, %p1;
-; SM70-FTZ-DISALLOW-NEXT: shr.u32 %r18, %r17, 16;
-; SM70-FTZ-DISALLOW-NEXT: shl.b32 %r19, %r18, %r1;
-; SM70-FTZ-DISALLOW-NEXT: and.b32 %r20, %r23, %r2;
-; SM70-FTZ-DISALLOW-NEXT: or.b32 %r21, %r20, %r19;
-; SM70-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r23, %r21;
-; SM70-FTZ-DISALLOW-NEXT: setp.ne.b32 %p2, %r3, %r23;
-; SM70-FTZ-DISALLOW-NEXT: mov.b32 %r23, %r3;
+; SM70-FTZ-DISALLOW-NEXT: or.b32 %r17, %r12, 4194304;
+; SM70-FTZ-DISALLOW-NEXT: selp.b32 %r18, %r17, %r16, %p1;
+; SM70-FTZ-DISALLOW-NEXT: shr.u32 %r19, %r18, 16;
+; SM70-FTZ-DISALLOW-NEXT: shl.b32 %r20, %r19, %r1;
+; SM70-FTZ-DISALLOW-NEXT: and.b32 %r21, %r24, %r2;
+; SM70-FTZ-DISALLOW-NEXT: or.b32 %r22, %r21, %r20;
+; SM70-FTZ-DISALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r24, %r22;
+; SM70-FTZ-DISALLOW-NEXT: setp.ne.b32 %p2, %r3, %r24;
+; SM70-FTZ-DISALLOW-NEXT: mov.b32 %r24, %r3;
; SM70-FTZ-DISALLOW-NEXT: @%p2 bra $L__BB81_1;
; SM70-FTZ-DISALLOW-NEXT: // %bb.2: // %atomicrmw.end
-; SM70-FTZ-DISALLOW-NEXT: shr.u32 %r22, %r3, %r1;
+; SM70-FTZ-DISALLOW-NEXT: shr.u32 %r23, %r3, %r1;
; SM70-FTZ-DISALLOW-NEXT: fence.acq_rel.cta;
-; SM70-FTZ-DISALLOW-NEXT: st.param.b16 [func_retval0], %r22;
+; SM70-FTZ-DISALLOW-NEXT: st.param.b16 [func_retval0], %r23;
; SM70-FTZ-DISALLOW-NEXT: ret;
;
; SM70-FTZ-ALLOW-LABEL: fmax_acq_rel_bfloat_global_cta(
; SM70-FTZ-ALLOW: {
; SM70-FTZ-ALLOW-NEXT: .reg .pred %p<3>;
; SM70-FTZ-ALLOW-NEXT: .reg .b16 %rs<2>;
-; SM70-FTZ-ALLOW-NEXT: .reg .b32 %r<24>;
+; SM70-FTZ-ALLOW-NEXT: .reg .b32 %r<25>;
; SM70-FTZ-ALLOW-NEXT: .reg .b64 %rd<3>;
; SM70-FTZ-ALLOW-EMPTY:
; SM70-FTZ-ALLOW-NEXT: // %bb.0:
@@ -3951,32 +3966,33 @@ define bfloat @fmax_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat %va
; SM70-FTZ-ALLOW-NEXT: mov.b32 %r6, 65535;
; SM70-FTZ-ALLOW-NEXT: shl.b32 %r7, %r6, %r1;
; SM70-FTZ-ALLOW-NEXT: not.b32 %r2, %r7;
-; SM70-FTZ-ALLOW-NEXT: ld.relaxed.cta.global.b32 %r23, [%rd1];
+; SM70-FTZ-ALLOW-NEXT: ld.relaxed.cta.global.b32 %r24, [%rd1];
; SM70-FTZ-ALLOW-NEXT: cvt.u32.u16 %r10, %rs1;
; SM70-FTZ-ALLOW-NEXT: shl.b32 %r11, %r10, 16;
; SM70-FTZ-ALLOW-NEXT: $L__BB81_1: // %atomicrmw.start
; SM70-FTZ-ALLOW-NEXT: // =>This Inner Loop Header: Depth=1
-; SM70-FTZ-ALLOW-NEXT: shr.u32 %r8, %r23, %r1;
+; SM70-FTZ-ALLOW-NEXT: shr.u32 %r8, %r24, %r1;
; SM70-FTZ-ALLOW-NEXT: shl.b32 %r9, %r8, 16;
; SM70-FTZ-ALLOW-NEXT: max.ftz.f32 %r12, %r9, %r11;
-; SM70-FTZ-ALLOW-NEXT: bfe.u32 %r13, %r12, 16, 1;
-; SM70-FTZ-ALLOW-NEXT: add.s32 %r14, %r13, %r12;
-; SM70-FTZ-ALLOW-NEXT: add.s32 %r15, %r14, 32767;
+; SM70-FTZ-ALLOW-NEXT: shr.u32 %r13, %r12, 16;
+; SM70-FTZ-ALLOW-NEXT: and.b32 %r14, %r13, 1;
+; SM70-FTZ-ALLOW-NEXT: add.s32 %r15, %r14, %r12;
+; SM70-FTZ-ALLOW-NEXT: add.s32 %r16, %r15, 32767;
; SM70-FTZ-ALLOW-NEXT: setp.nan.ftz.f32 %p1, %r12, %r12;
-; SM70-FTZ-ALLOW-NEXT: or.b32 %r16, %r12, 4194304;
-; SM70-FTZ-ALLOW-NEXT: selp.b32 %r17, %r16, %r15, %p1;
-; SM70-FTZ-ALLOW-NEXT: shr.u32 %r18, %r17, 16;
-; SM70-FTZ-ALLOW-NEXT: shl.b32 %r19, %r18, %r1;
-; SM70-FTZ-ALLOW-NEXT: and.b32 %r20, %r23, %r2;
-; SM70-FTZ-ALLOW-NEXT: or.b32 %r21, %r20, %r19;
-; SM70-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r23, %r21;
-; SM70-FTZ-ALLOW-NEXT: setp.ne.b32 %p2, %r3, %r23;
-; SM70-FTZ-ALLOW-NEXT: mov.b32 %r23, %r3;
+; SM70-FTZ-ALLOW-NEXT: or.b32 %r17, %r12, 4194304;
+; SM70-FTZ-ALLOW-NEXT: selp.b32 %r18, %r17, %r16, %p1;
+; SM70-FTZ-ALLOW-NEXT: shr.u32 %r19, %r18, 16;
+; SM70-FTZ-ALLOW-NEXT: shl.b32 %r20, %r19, %r1;
+; SM70-FTZ-ALLOW-NEXT: and.b32 %r21, %r24, %r2;
+; SM70-FTZ-ALLOW-NEXT: or.b32 %r22, %r21, %r20;
+; SM70-FTZ-ALLOW-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r24, %r22;
+; SM70-FTZ-ALLOW-NEXT: setp.ne.b32 %p2, %r3, %r24;
+; SM70-FTZ-ALLOW-NEXT: mov.b32 %r24, %r3;
; SM70-FTZ-ALLOW-NEXT: @%p2 bra $L__BB81_1;
; SM70-FTZ-ALLOW-NEXT: // %bb.2: // %atomicrmw.end
-; SM70-FTZ-ALLOW-NEXT: shr.u32 %r22, %r3, %r1;
+; SM70-FTZ-ALLOW-NEXT: shr.u32 %r23, %r3, %r1;
; SM70-FTZ-ALLOW-NEXT: fence.acq_rel.cta;
-; SM70-FTZ-ALLOW-NEXT: st.param.b16 [func_retval0], %r22;
+; SM70-FTZ-ALLOW-NEXT: st.param.b16 [func_retval0], %r23;
; SM70-FTZ-ALLOW-NEXT: ret;
%retval = atomicrmw fmax ptr addrspace(1) %addr, bfloat %val syncscope("block") acq_rel
ret bfloat %retval
diff --git a/llvm/test/CodeGen/NVPTX/bf16-instructions.ll b/llvm/test/CodeGen/NVPTX/bf16-instructions.ll
index 8548ed7ac5a45..a6ad0086661e2 100644
--- a/llvm/test/CodeGen/NVPTX/bf16-instructions.ll
+++ b/llvm/test/CodeGen/NVPTX/bf16-instructions.ll
@@ -18,7 +18,7 @@ define bfloat @test_fadd(bfloat %0, bfloat %1) {
; SM70-LABEL: test_fadd(
; SM70: {
; SM70-NEXT: .reg .pred %p<2>;
-; SM70-NEXT: .reg .b32 %r<12>;
+; SM70-NEXT: .reg .b32 %r<13>;
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
; SM70-NEXT: ld.param.b16 %r1, [test_fadd_param_1];
@@ -26,14 +26,15 @@ define bfloat @test_fadd(bfloat %0, bfloat %1) {
; SM70-NEXT: ld.param.b16 %r3, [test_fadd_param_0];
; SM70-NEXT: shl.b32 %r4, %r3, 16;
; SM70-NEXT: add.rn.f32 %r5, %r4, %r2;
-; SM70-NEXT: bfe.u32 %r6, %r5, 16, 1;
-; SM70-NEXT: add.s32 %r7, %r6, %r5;
-; SM70-NEXT: add.s32 %r8, %r7, 32767;
+; SM70-NEXT: shr.u32 %r6, %r5, 16;
+; SM70-NEXT: and.b32 %r7, %r6, 1;
+; SM70-NEXT: add.s32 %r8, %r7, %r5;
+; SM70-NEXT: add.s32 %r9, %r8, 32767;
; SM70-NEXT: setp.nan.f32 %p1, %r5, %r5;
-; SM70-NEXT: or.b32 %r9, %r5, 4194304;
-; SM70-NEXT: selp.b32 %r10, %r9, %r8, %p1;
-; SM70-NEXT: shr.u32 %r11, %r10, 16;
-; SM70-NEXT: st.param.b16 [func_retval0], %r11;
+; SM70-NEXT: or.b32 %r10, %r5, 4194304;
+; SM70-NEXT: selp.b32 %r11, %r10, %r9, %p1;
+; SM70-NEXT: shr.u32 %r12, %r11, 16;
+; SM70-NEXT: st.param.b16 [func_retval0], %r12;
; SM70-NEXT: ret;
;
; SM80-LABEL: test_fadd(
@@ -92,7 +93,7 @@ define bfloat @test_fsub(bfloat %0, bfloat %1) {
; SM70-LABEL: test_fsub(
; SM70: {
; SM70-NEXT: .reg .pred %p<2>;
-; SM70-NEXT: .reg .b32 %r<12>;
+; SM70-NEXT: .reg .b32 %r<13>;
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
; SM70-NEXT: ld.param.b16 %r1, [test_fsub_param_1];
@@ -100,14 +101,15 @@ define bfloat @test_fsub(bfloat %0, bfloat %1) {
; SM70-NEXT: ld.param.b16 %r3, [test_fsub_param_0];
; SM70-NEXT: shl.b32 %r4, %r3, 16;
; SM70-NEXT: sub.rn.f32 %r5, %r4, %r2;
-; SM70-NEXT: bfe.u32 %r6, %r5, 16, 1;
-; SM70-NEXT: add.s32 %r7, %r6, %r5;
-; SM70-NEXT: add.s32 %r8, %r7, 32767;
+; SM70-NEXT: shr.u32 %r6, %r5, 16;
+; SM70-NEXT: and.b32 %r7, %r6, 1;
+; SM70-NEXT: add.s32 %r8, %r7, %r5;
+; SM70-NEXT: add.s32 %r9, %r8, 32767;
; SM70-NEXT: setp.nan.f32 %p1, %r5, %r5;
-; SM70-NEXT: or.b32 %r9, %r5, 4194304;
-; SM70-NEXT: selp.b32 %r10, %r9, %r8, %p1;
-; SM70-NEXT: shr.u32 %r11, %r10, 16;
-; SM70-NEXT: st.param.b16 [func_retval0], %r11;
+; SM70-NEXT: or.b32 %r10, %r5, 4194304;
+; SM70-NEXT: selp.b32 %r11, %r10, %r9, %p1;
+; SM70-NEXT: shr.u32 %r12, %r11, 16;
+; SM70-NEXT: st.param.b16 [func_retval0], %r12;
; SM70-NEXT: ret;
;
; SM80-LABEL: test_fsub(
@@ -167,7 +169,7 @@ define <2 x bfloat> @test_faddx2(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
; SM70: {
; SM70-NEXT: .reg .pred %p<3>;
; SM70-NEXT: .reg .b16 %rs<5>;
-; SM70-NEXT: .reg .b32 %r<22>;
+; SM70-NEXT: .reg .b32 %r<24>;
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
; SM70-NEXT: ld.param.v2.b16 {%rs1, %rs2}, [test_faddx2_param_0];
@@ -177,25 +179,27 @@ define <2 x bfloat> @test_faddx2(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
; SM70-NEXT: cvt.u32.u16 %r3, %rs2;
; SM70-NEXT: shl.b32 %r4, %r3, 16;
; SM70-NEXT: add.rn.f32 %r5, %r4, %r2;
-; SM70-NEXT: bfe.u32 %r6, %r5, 16, 1;
-; SM70-NEXT: add.s32 %r7, %r6, %r5;
-; SM70-NEXT: add.s32 %r8, %r7, 32767;
+; SM70-NEXT: shr.u32 %r6, %r5, 16;
+; SM70-NEXT: and.b32 %r7, %r6, 1;
+; SM70-NEXT: add.s32 %r8, %r7, %r5;
+; SM70-NEXT: add.s32 %r9, %r8, 32767;
; SM70-NEXT: setp.nan.f32 %p1, %r5, %r5;
-; SM70-NEXT: or.b32 %r9, %r5, 4194304;
-; SM70-NEXT: selp.b32 %r10, %r9, %r8, %p1;
-; SM70-NEXT: cvt.u32.u16 %r11, %rs3;
-; SM70-NEXT: shl.b32 %r12, %r11, 16;
-; SM70-NEXT: cvt.u32.u16 %r13, %rs1;
-; SM70-NEXT: shl.b32 %r14, %r13, 16;
-; SM70-NEXT: add.rn.f32 %r15, %r14, %r12;
-; SM70-NEXT: bfe.u32 %r16, %r15, 16, 1;
-; SM70-NEXT: add.s32 %r17, %r16, %r15;
-; SM70-NEXT: add.s32 %r18, %r17, 32767;
-; SM70-NEXT: setp.nan.f32 %p2, %r15, %r15;
-; SM70-NEXT: or.b32 %r19, %r15, 4194304;
-; SM70-NEXT: selp.b32 %r20, %r19, %r18, %p2;
-; SM70-NEXT: prmt.b32 %r21, %r20, %r10, 0x7632U;
-; SM70-NEXT: st.param.b32 [func_retval0], %r21;
+; SM70-NEXT: or.b32 %r10, %r5, 4194304;
+; SM70-NEXT: selp.b32 %r11, %r10, %r9, %p1;
+; SM70-NEXT: cvt.u32.u16 %r12, %rs3;
+; SM70-NEXT: shl.b32 %r13, %r12, 16;
+; SM70-NEXT: cvt.u32.u16 %r14, %rs1;
+; SM70-NEXT: shl.b32 %r15, %r14, 16;
+; SM70-NEXT: add.rn.f32 %r16, %r15, %r13;
+; SM70-NEXT: shr.u32 %r17, %r16, 16;
+; SM70-NEXT: and.b32 %r18, %r17, 1;
+; SM70-NEXT: add.s32 %r19, %r18, %r16;
+; SM70-NEXT: add.s32 %r20, %r19, 32767;
+; SM70-NEXT: setp.nan.f32 %p2, %r16, %r16;
+; SM70-NEXT: or.b32 %r21, %r16, 4194304;
+; SM70-NEXT: selp.b32 %r22, %r21, %r20, %p2;
+; SM70-NEXT: prmt.b32 %r23, %r22, %r11, 0x7632U;
+; SM70-NEXT: st.param.b32 [func_retval0], %r23;
; SM70-NEXT: ret;
;
; SM80-LABEL: test_faddx2(
@@ -258,7 +262,7 @@ define <2 x bfloat> @test_fsubx2(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
; SM70: {
; SM70-NEXT: .reg .pred %p<3>;
; SM70-NEXT: .reg .b16 %rs<5>;
-; SM70-NEXT: .reg .b32 %r<22>;
+; SM70-NEXT: .reg .b32 %r<24>;
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
; SM70-NEXT: ld.param.v2.b16 {%rs1, %rs2}, [test_fsubx2_param_0];
@@ -268,25 +272,27 @@ define <2 x bfloat> @test_fsubx2(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
; SM70-NEXT: cvt.u32.u16 %r3, %rs2;
; SM70-NEXT: shl.b32 %r4, %r3, 16;
; SM70-NEXT: sub.rn.f32 %r5, %r4, %r2;
-; SM70-NEXT: bfe.u32 %r6, %r5, 16, 1;
-; SM70-NEXT: add.s32 %r7, %r6, %r5;
-; SM70-NEXT: add.s32 %r8, %r7, 32767;
+; SM70-NEXT: shr.u32 %r6, %r5, 16;
+; SM70-NEXT: and.b32 %r7, %r6, 1;
+; SM70-NEXT: add.s32 %r8, %r7, %r5;
+; SM70-NEXT: add.s32 %r9, %r8, 32767;
; SM70-NEXT: setp.nan.f32 %p1, %r5, %r5;
-; SM70-NEXT: or.b32 %r9, %r5, 4194304;
-; SM70-NEXT: selp.b32 %r10, %r9, %r8, %p1;
-; SM70-NEXT: cvt.u32.u16 %r11, %rs3;
-; SM70-NEXT: shl.b32 %r12, %r11, 16;
-; SM70-NEXT: cvt.u32.u16 %r13, %rs1;
-; SM70-NEXT: shl.b32 %r14, %r13, 16;
-; SM70-NEXT: sub.rn.f32 %r15, %r14, %r12;
-; SM70-NEXT: bfe.u32 %r16, %r15, 16, 1;
-; SM70-NEXT: add.s32 %r17, %r16, %r15;
-; SM70-NEXT: add.s32 %r18, %r17, 32767;
-; SM70-NEXT: setp.nan.f32 %p2, %r15, %r15;
-; SM70-NEXT: or.b32 %r19, %r15, 4194304;
-; SM70-NEXT: selp.b32 %r20, %r19, %r18, %p2;
-; SM70-NEXT: prmt.b32 %r21, %r20, %r10, 0x7632U;
-; SM70-NEXT: st.param.b32 [func_retval0], %r21;
+; SM70-NEXT: or.b32 %r10, %r5, 4194304;
+; SM70-NEXT: selp.b32 %r11, %r10, %r9, %p1;
+; SM70-NEXT: cvt.u32.u16 %r12, %rs3;
+; SM70-NEXT: shl.b32 %r13, %r12, 16;
+; SM70-NEXT: cvt.u32.u16 %r14, %rs1;
+; SM70-NEXT: shl.b32 %r15, %r14, 16;
+; SM70-NEXT: sub.rn.f32 %r16, %r15, %r13;
+; SM70-NEXT: shr.u32 %r17, %r16, 16;
+; SM70-NEXT: and.b32 %r18, %r17, 1;
+; SM70-NEXT: add.s32 %r19, %r18, %r16;
+; SM70-NEXT: add.s32 %r20, %r19, 32767;
+; SM70-NEXT: setp.nan.f32 %p2, %r16, %r16;
+; SM70-NEXT: or.b32 %r21, %r16, 4194304;
+; SM70-NEXT: selp.b32 %r22, %r21, %r20, %p2;
+; SM70-NEXT: prmt.b32 %r23, %r22, %r11, 0x7632U;
+; SM70-NEXT: st.param.b32 [func_retval0], %r23;
; SM70-NEXT: ret;
;
; SM80-LABEL: test_fsubx2(
@@ -349,7 +355,7 @@ define <2 x bfloat> @test_fmulx2(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
; SM70: {
; SM70-NEXT: .reg .pred %p<3>;
; SM70-NEXT: .reg .b16 %rs<5>;
-; SM70-NEXT: .reg .b32 %r<22>;
+; SM70-NEXT: .reg .b32 %r<24>;
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
; SM70-NEXT: ld.param.v2.b16 {%rs1, %rs2}, [test_fmulx2_param_0];
@@ -359,25 +365,27 @@ define <2 x bfloat> @test_fmulx2(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
; SM70-NEXT: cvt.u32.u16 %r3, %rs2;
; SM70-NEXT: shl.b32 %r4, %r3, 16;
; SM70-NEXT: mul.rn.f32 %r5, %r4, %r2;
-; SM70-NEXT: bfe.u32 %r6, %r5, 16, 1;
-; SM70-NEXT: add.s32 %r7, %r6, %r5;
-; SM70-NEXT: add.s32 %r8, %r7, 32767;
+; SM70-NEXT: shr.u32 %r6, %r5, 16;
+; SM70-NEXT: and.b32 %r7, %r6, 1;
+; SM70-NEXT: add.s32 %r8, %r7, %r5;
+; SM70-NEXT: add.s32 %r9, %r8, 32767;
; SM70-NEXT: setp.nan.f32 %p1, %r5, %r5;
-; SM70-NEXT: or.b32 %r9, %r5, 4194304;
-; SM70-NEXT: selp.b32 %r10, %r9, %r8, %p1;
-; SM70-NEXT: cvt.u32.u16 %r11, %rs3;
-; SM70-NEXT: shl.b32 %r12, %r11, 16;
-; SM70-NEXT: cvt.u32.u16 %r13, %rs1;
-; SM70-NEXT: shl.b32 %r14, %r13, 16;
-; SM70-NEXT: mul.rn.f32 %r15, %r14, %r12;
-; SM70-NEXT: bfe.u32 %r16, %r15, 16, 1;
-; SM70-NEXT: add.s32 %r17, %r16, %r15;
-; SM70-NEXT: add.s32 %r18, %r17, 32767;
-; SM70-NEXT: setp.nan.f32 %p2, %r15, %r15;
-; SM70-NEXT: or.b32 %r19, %r15, 4194304;
-; SM70-NEXT: selp.b32 %r20, %r19, %r18, %p2;
-; SM70-NEXT: prmt.b32 %r21, %r20, %r10, 0x7632U;
-; SM70-NEXT: st.param.b32 [func_retval0], %r21;
+; SM70-NEXT: or.b32 %r10, %r5, 4194304;
+; SM70-NEXT: selp.b32 %r11, %r10, %r9, %p1;
+; SM70-NEXT: cvt.u32.u16 %r12, %rs3;
+; SM70-NEXT: shl.b32 %r13, %r12, 16;
+; SM70-NEXT: cvt.u32.u16 %r14, %rs1;
+; SM70-NEXT: shl.b32 %r15, %r14, 16;
+; SM70-NEXT: mul.rn.f32 %r16, %r15, %r13;
+; SM70-NEXT: shr.u32 %r17, %r16, 16;
+; SM70-NEXT: and.b32 %r18, %r17, 1;
+; SM70-NEXT: add.s32 %r19, %r18, %r16;
+; SM70-NEXT: add.s32 %r20, %r19, 32767;
+; SM70-NEXT: setp.nan.f32 %p2, %r16, %r16;
+; SM70-NEXT: or.b32 %r21, %r16, 4194304;
+; SM70-NEXT: selp.b32 %r22, %r21, %r20, %p2;
+; SM70-NEXT: prmt.b32 %r23, %r22, %r11, 0x7632U;
+; SM70-NEXT: st.param.b32 [func_retval0], %r23;
; SM70-NEXT: ret;
;
; SM80-LABEL: test_fmulx2(
@@ -440,7 +448,7 @@ define <2 x bfloat> @test_fdiv(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
; SM70: {
; SM70-NEXT: .reg .pred %p<3>;
; SM70-NEXT: .reg .b16 %rs<5>;
-; SM70-NEXT: .reg .b32 %r<22>;
+; SM70-NEXT: .reg .b32 %r<24>;
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
; SM70-NEXT: ld.param.v2.b16 {%rs1, %rs2}, [test_fdiv_param_0];
@@ -450,25 +458,27 @@ define <2 x bfloat> @test_fdiv(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
; SM70-NEXT: cvt.u32.u16 %r3, %rs2;
; SM70-NEXT: shl.b32 %r4, %r3, 16;
; SM70-NEXT: div.rn.f32 %r5, %r4, %r2;
-; SM70-NEXT: bfe.u32 %r6, %r5, 16, 1;
-; SM70-NEXT: add.s32 %r7, %r6, %r5;
-; SM70-NEXT: add.s32 %r8, %r7, 32767;
+; SM70-NEXT: shr.u32 %r6, %r5, 16;
+; SM70-NEXT: and.b32 %r7, %r6, 1;
+; SM70-NEXT: add.s32 %r8, %r7, %r5;
+; SM70-NEXT: add.s32 %r9, %r8, 32767;
; SM70-NEXT: setp.nan.f32 %p1, %r5, %r5;
-; SM70-NEXT: or.b32 %r9, %r5, 4194304;
-; SM70-NEXT: selp.b32 %r10, %r9, %r8, %p1;
-; SM70-NEXT: cvt.u32.u16 %r11, %rs3;
-; SM70-NEXT: shl.b32 %r12, %r11, 16;
-; SM70-NEXT: cvt.u32.u16 %r13, %rs1;
-; SM70-NEXT: shl.b32 %r14, %r13, 16;
-; SM70-NEXT: div.rn.f32 %r15, %r14, %r12;
-; SM70-NEXT: bfe.u32 %r16, %r15, 16, 1;
-; SM70-NEXT: add.s32 %r17, %r16, %r15;
-; SM70-NEXT: add.s32 %r18, %r17, 32767;
-; SM70-NEXT: setp.nan.f32 %p2, %r15, %r15;
-; SM70-NEXT: or.b32 %r19, %r15, 4194304;
-; SM70-NEXT: selp.b32 %r20, %r19, %r18, %p2;
-; SM70-NEXT: prmt.b32 %r21, %r20, %r10, 0x7632U;
-; SM70-NEXT: st.param.b32 [func_retval0], %r21;
+; SM70-NEXT: or.b32 %r10, %r5, 4194304;
+; SM70-NEXT: selp.b32 %r11, %r10, %r9, %p1;
+; SM70-NEXT: cvt.u32.u16 %r12, %rs3;
+; SM70-NEXT: shl.b32 %r13, %r12, 16;
+; SM70-NEXT: cvt.u32.u16 %r14, %rs1;
+; SM70-NEXT: shl.b32 %r15, %r14, 16;
+; SM70-NEXT: div.rn.f32 %r16, %r15, %r13;
+; SM70-NEXT: shr.u32 %r17, %r16, 16;
+; SM70-NEXT: and.b32 %r18, %r17, 1;
+; SM70-NEXT: add.s32 %r19, %r18, %r16;
+; SM70-NEXT: add.s32 %r20, %r19, 32767;
+; SM70-NEXT: setp.nan.f32 %p2, %r16, %r16;
+; SM70-NEXT: or.b32 %r21, %r16, 4194304;
+; SM70-NEXT: selp.b32 %r22, %r21, %r20, %p2;
+; SM70-NEXT: prmt.b32 %r23, %r22, %r11, 0x7632U;
+; SM70-NEXT: st.param.b32 [func_retval0], %r23;
; SM70-NEXT: ret;
;
; SM80-LABEL: test_fdiv(
@@ -634,18 +644,19 @@ define bfloat @test_fptrunc_float(float %a) #0 {
; SM70-LABEL: test_fptrunc_float(
; SM70: {
; SM70-NEXT: .reg .pred %p<2>;
-; SM70-NEXT: .reg .b32 %r<8>;
+; SM70-NEXT: .reg .b32 %r<9>;
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
; SM70-NEXT: ld.param.b32 %r1, [test_fptrunc_float_param_0];
-; SM70-NEXT: bfe.u32 %r2, %r1, 16, 1;
-; SM70-NEXT: add.s32 %r3, %r2, %r1;
-; SM70-NEXT: add.s32 %r4, %r3, 32767;
+; SM70-NEXT: shr.u32 %r2, %r1, 16;
+; SM70-NEXT: and.b32 %r3, %r2, 1;
+; SM70-NEXT: add.s32 %r4, %r3, %r1;
+; SM70-NEXT: add.s32 %r5, %r4, 32767;
; SM70-NEXT: setp.nan.f32 %p1, %r1, %r1;
-; SM70-NEXT: or.b32 %r5, %r1, 4194304;
-; SM70-NEXT: selp.b32 %r6, %r5, %r4, %p1;
-; SM70-NEXT: shr.u32 %r7, %r6, 16;
-; SM70-NEXT: st.param.b16 [func_retval0], %r7;
+; SM70-NEXT: or.b32 %r6, %r1, 4194304;
+; SM70-NEXT: selp.b32 %r7, %r6, %r5, %p1;
+; SM70-NEXT: shr.u32 %r8, %r7, 16;
+; SM70-NEXT: st.param.b16 [func_retval0], %r8;
; SM70-NEXT: ret;
;
; SM80-LABEL: test_fptrunc_float(
@@ -699,20 +710,21 @@ define bfloat @test_fadd_imm_1(bfloat %a) #0 {
; SM70-LABEL: test_fadd_imm_1(
; SM70: {
; SM70-NEXT: .reg .pred %p<2>;
-; SM70-NEXT: .reg .b32 %r<10>;
+; SM70-NEXT: .reg .b32 %r<11>;
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
; SM70-NEXT: ld.param.b16 %r1, [test_fadd_imm_1_param_0];
; SM70-NEXT: shl.b32 %r2, %r1, 16;
; SM70-NEXT: add.rn.f32 %r3, %r2, 0f3F800000;
-; SM70-NEXT: bfe.u32 %r4, %r3, 16, 1;
-; SM70-NEXT: add.s32 %r5, %r4, %r3;
-; SM70-NEXT: add.s32 %r6, %r5, 32767;
+; SM70-NEXT: shr.u32 %r4, %r3, 16;
+; SM70-NEXT: and.b32 %r5, %r4, 1;
+; SM70-NEXT: add.s32 %r6, %r5, %r3;
+; SM70-NEXT: add.s32 %r7, %r6, 32767;
; SM70-NEXT: setp.nan.f32 %p1, %r3, %r3;
-; SM70-NEXT: or.b32 %r7, %r3, 4194304;
-; SM70-NEXT: selp.b32 %r8, %r7, %r6, %p1;
-; SM70-NEXT: shr.u32 %r9, %r8, 16;
-; SM70-NEXT: st.param.b16 [func_retval0], %r9;
+; SM70-NEXT: or.b32 %r8, %r3, 4194304;
+; SM70-NEXT: selp.b32 %r9, %r8, %r7, %p1;
+; SM70-NEXT: shr.u32 %r10, %r9, 16;
+; SM70-NEXT: st.param.b16 [func_retval0], %r10;
; SM70-NEXT: ret;
;
; SM80-LABEL: test_fadd_imm_1(
@@ -1063,19 +1075,20 @@ define bfloat @test_sitofp_i16(i16 %a) {
; SM70: {
; SM70-NEXT: .reg .pred %p<2>;
; SM70-NEXT: .reg .b16 %rs<2>;
-; SM70-NEXT: .reg .b32 %r<8>;
+; SM70-NEXT: .reg .b32 %r<9>;
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
; SM70-NEXT: ld.param.b16 %rs1, [test_sitofp_i16_param_0];
; SM70-NEXT: cvt.rn.f32.s16 %r1, %rs1;
-; SM70-NEXT: bfe.u32 %r2, %r1, 16, 1;
-; SM70-NEXT: add.s32 %r3, %r2, %r1;
-; SM70-NEXT: add.s32 %r4, %r3, 32767;
+; SM70-NEXT: shr.u32 %r2, %r1, 16;
+; SM70-NEXT: and.b32 %r3, %r2, 1;
+; SM70-NEXT: add.s32 %r4, %r3, %r1;
+; SM70-NEXT: add.s32 %r5, %r4, 32767;
; SM70-NEXT: setp.nan.f32 %p1, %r1, %r1;
-; SM70-NEXT: or.b32 %r5, %r1, 4194304;
-; SM70-NEXT: selp.b32 %r6, %r5, %r4, %p1;
-; SM70-NEXT: shr.u32 %r7, %r6, 16;
-; SM70-NEXT: st.param.b16 [func_retval0], %r7;
+; SM70-NEXT: or.b32 %r6, %r1, 4194304;
+; SM70-NEXT: selp.b32 %r7, %r6, %r5, %p1;
+; SM70-NEXT: shr.u32 %r8, %r7, 16;
+; SM70-NEXT: st.param.b16 [func_retval0], %r8;
; SM70-NEXT: ret;
;
; SM80-LABEL: test_sitofp_i16(
@@ -1130,19 +1143,20 @@ define bfloat @test_uitofp_i8(i8 %a) {
; SM70: {
; SM70-NEXT: .reg .pred %p<2>;
; SM70-NEXT: .reg .b16 %rs<2>;
-; SM70-NEXT: .reg .b32 %r<8>;
+; SM70-NEXT: .reg .b32 %r<9>;
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
; SM70-NEXT: ld.param.b8 %rs1, [test_uitofp_i8_param_0];
; SM70-NEXT: cvt.rn.f32.u16 %r1, %rs1;
-; SM70-NEXT: bfe.u32 %r2, %r1, 16, 1;
-; SM70-NEXT: add.s32 %r3, %r2, %r1;
-; SM70-NEXT: add.s32 %r4, %r3, 32767;
+; SM70-NEXT: shr.u32 %r2, %r1, 16;
+; SM70-NEXT: and.b32 %r3, %r2, 1;
+; SM70-NEXT: add.s32 %r4, %r3, %r1;
+; SM70-NEXT: add.s32 %r5, %r4, 32767;
; SM70-NEXT: setp.nan.f32 %p1, %r1, %r1;
-; SM70-NEXT: or.b32 %r5, %r1, 4194304;
-; SM70-NEXT: selp.b32 %r6, %r5, %r4, %p1;
-; SM70-NEXT: shr.u32 %r7, %r6, 16;
-; SM70-NEXT: st.param.b16 [func_retval0], %r7;
+; SM70-NEXT: or.b32 %r6, %r1, 4194304;
+; SM70-NEXT: selp.b32 %r7, %r6, %r5, %p1;
+; SM70-NEXT: shr.u32 %r8, %r7, 16;
+; SM70-NEXT: st.param.b16 [func_retval0], %r8;
; SM70-NEXT: ret;
;
; SM80-LABEL: test_uitofp_i8(
@@ -1197,7 +1211,7 @@ define bfloat @test_uitofp_i1(i1 %a) {
; SM70: {
; SM70-NEXT: .reg .pred %p<3>;
; SM70-NEXT: .reg .b16 %rs<3>;
-; SM70-NEXT: .reg .b32 %r<9>;
+; SM70-NEXT: .reg .b32 %r<10>;
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
; SM70-NEXT: ld.param.b8 %rs1, [test_uitofp_i1_param_0];
@@ -1205,14 +1219,15 @@ define bfloat @test_uitofp_i1(i1 %a) {
; SM70-NEXT: setp.ne.b16 %p1, %rs2, 0;
; SM70-NEXT: selp.b32 %r1, 1, 0, %p1;
; SM70-NEXT: cvt.rn.f32.u32 %r2, %r1;
-; SM70-NEXT: bfe.u32 %r3, %r2, 16, 1;
-; SM70-NEXT: add.s32 %r4, %r3, %r2;
-; SM70-NEXT: add.s32 %r5, %r4, 32767;
+; SM70-NEXT: shr.u32 %r3, %r2, 16;
+; SM70-NEXT: and.b32 %r4, %r3, 1;
+; SM70-NEXT: add.s32 %r5, %r4, %r2;
+; SM70-NEXT: add.s32 %r6, %r5, 32767;
; SM70-NEXT: setp.nan.f32 %p2, %r2, %r2;
-; SM70-NEXT: or.b32 %r6, %r2, 4194304;
-; SM70-NEXT: selp.b32 %r7, %r6, %r5, %p2;
-; SM70-NEXT: shr.u32 %r8, %r7, 16;
-; SM70-NEXT: st.param.b16 [func_retval0], %r8;
+; SM70-NEXT: or.b32 %r7, %r2, 4194304;
+; SM70-NEXT: selp.b32 %r8, %r7, %r6, %p2;
+; SM70-NEXT: shr.u32 %r9, %r8, 16;
+; SM70-NEXT: st.param.b16 [func_retval0], %r9;
; SM70-NEXT: ret;
;
; SM80-LABEL: test_uitofp_i1(
@@ -1285,19 +1300,20 @@ define bfloat @test_uitofp_i16(i16 %a) {
; SM70: {
; SM70-NEXT: .reg .pred %p<2>;
; SM70-NEXT: .reg .b16 %rs<2>;
-; SM70-NEXT: .reg .b32 %r<8>;
+; SM70-NEXT: .reg .b32 %r<9>;
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
; SM70-NEXT: ld.param.b16 %rs1, [test_uitofp_i16_param_0];
; SM70-NEXT: cvt.rn.f32.u16 %r1, %rs1;
-; SM70-NEXT: bfe.u32 %r2, %r1, 16, 1;
-; SM70-NEXT: add.s32 %r3, %r2, %r1;
-; SM70-NEXT: add.s32 %r4, %r3, 32767;
+; SM70-NEXT: shr.u32 %r2, %r1, 16;
+; SM70-NEXT: and.b32 %r3, %r2, 1;
+; SM70-NEXT: add.s32 %r4, %r3, %r1;
+; SM70-NEXT: add.s32 %r5, %r4, 32767;
; SM70-NEXT: setp.nan.f32 %p1, %r1, %r1;
-; SM70-NEXT: or.b32 %r5, %r1, 4194304;
-; SM70-NEXT: selp.b32 %r6, %r5, %r4, %p1;
-; SM70-NEXT: shr.u32 %r7, %r6, 16;
-; SM70-NEXT: st.param.b16 [func_retval0], %r7;
+; SM70-NEXT: or.b32 %r6, %r1, 4194304;
+; SM70-NEXT: selp.b32 %r7, %r6, %r5, %p1;
+; SM70-NEXT: shr.u32 %r8, %r7, 16;
+; SM70-NEXT: st.param.b16 [func_retval0], %r8;
; SM70-NEXT: ret;
;
; SM80-LABEL: test_uitofp_i16(
@@ -1351,19 +1367,20 @@ define bfloat @test_uitofp_i32(i32 %a) {
; SM70-LABEL: test_uitofp_i32(
; SM70: {
; SM70-NEXT: .reg .pred %p<2>;
-; SM70-NEXT: .reg .b32 %r<9>;
+; SM70-NEXT: .reg .b32 %r<10>;
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
; SM70-NEXT: ld.param.b32 %r1, [test_uitofp_i32_param_0];
; SM70-NEXT: cvt.rn.f32.u32 %r2, %r1;
-; SM70-NEXT: bfe.u32 %r3, %r2, 16, 1;
-; SM70-NEXT: add.s32 %r4, %r3, %r2;
-; SM70-NEXT: add.s32 %r5, %r4, 32767;
+; SM70-NEXT: shr.u32 %r3, %r2, 16;
+; SM70-NEXT: and.b32 %r4, %r3, 1;
+; SM70-NEXT: add.s32 %r5, %r4, %r2;
+; SM70-NEXT: add.s32 %r6, %r5, 32767;
; SM70-NEXT: setp.nan.f32 %p1, %r2, %r2;
-; SM70-NEXT: or.b32 %r6, %r2, 4194304;
-; SM70-NEXT: selp.b32 %r7, %r6, %r5, %p1;
-; SM70-NEXT: shr.u32 %r8, %r7, 16;
-; SM70-NEXT: st.param.b16 [func_retval0], %r8;
+; SM70-NEXT: or.b32 %r7, %r2, 4194304;
+; SM70-NEXT: selp.b32 %r8, %r7, %r6, %p1;
+; SM70-NEXT: shr.u32 %r9, %r8, 16;
+; SM70-NEXT: st.param.b16 [func_retval0], %r9;
; SM70-NEXT: ret;
;
; SM80-LABEL: test_uitofp_i32(
@@ -1419,20 +1436,21 @@ define bfloat @test_uitofp_i64(i64 %a) {
; SM70-LABEL: test_uitofp_i64(
; SM70: {
; SM70-NEXT: .reg .pred %p<2>;
-; SM70-NEXT: .reg .b32 %r<8>;
+; SM70-NEXT: .reg .b32 %r<9>;
; SM70-NEXT: .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
; SM70-NEXT: ld.param.b64 %rd1, [test_uitofp_i64_param_0];
; SM70-NEXT: cvt.rn.f32.u64 %r1, %rd1;
-; SM70-NEXT: bfe.u32 %r2, %r1, 16, 1;
-; SM70-NEXT: add.s32 %r3, %r2, %r1;
-; SM70-NEXT: add.s32 %r4, %r3, 32767;
+; SM70-NEXT: shr.u32 %r2, %r1, 16;
+; SM70-NEXT: and.b32 %r3, %r2, 1;
+; SM70-NEXT: add.s32 %r4, %r3, %r1;
+; SM70-NEXT: add.s32 %r5, %r4, 32767;
; SM70-NEXT: setp.nan.f32 %p1, %r1, %r1;
-; SM70-NEXT: or.b32 %r5, %r1, 4194304;
-; SM70-NEXT: selp.b32 %r6, %r5, %r4, %p1;
-; SM70-NEXT: shr.u32 %r7, %r6, 16;
-; SM70-NEXT: st.param.b16 [func_retval0], %r7;
+; SM70-NEXT: or.b32 %r6, %r1, 4194304;
+; SM70-NEXT: selp.b32 %r7, %r6, %r5, %p1;
+; SM70-NEXT: shr.u32 %r8, %r7, 16;
+; SM70-NEXT: st.param.b16 [func_retval0], %r8;
; SM70-NEXT: ret;
;
; SM80-LABEL: test_uitofp_i64(
@@ -1490,20 +1508,21 @@ define bfloat @test_roundeven(bfloat %a) {
; SM70-LABEL: test_roundeven(
; SM70: {
; SM70-NEXT: .reg .pred %p<2>;
-; SM70-NEXT: .reg .b32 %r<10>;
+; SM70-NEXT: .reg .b32 %r<11>;
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
; SM70-NEXT: ld.param.b16 %r1, [test_roundeven_param_0];
; SM70-NEXT: shl.b32 %r2, %r1, 16;
; SM70-NEXT: cvt.rni.f32.f32 %r3, %r2;
-; SM70-NEXT: bfe.u32 %r4, %r3, 16, 1;
-; SM70-NEXT: add.s32 %r5, %r4, %r3;
-; SM70-NEXT: add.s32 %r6, %r5, 32767;
+; SM70-NEXT: shr.u32 %r4, %r3, 16;
+; SM70-NEXT: and.b32 %r5, %r4, 1;
+; SM70-NEXT: add.s32 %r6, %r5, %r3;
+; SM70-NEXT: add.s32 %r7, %r6, 32767;
; SM70-NEXT: setp.nan.f32 %p1, %r3, %r3;
-; SM70-NEXT: or.b32 %r7, %r3, 4194304;
-; SM70-NEXT: selp.b32 %r8, %r7, %r6, %p1;
-; SM70-NEXT: shr.u32 %r9, %r8, 16;
-; SM70-NEXT: st.param.b16 [func_retval0], %r9;
+; SM70-NEXT: or.b32 %r8, %r3, 4194304;
+; SM70-NEXT: selp.b32 %r9, %r8, %r7, %p1;
+; SM70-NEXT: shr.u32 %r10, %r9, 16;
+; SM70-NEXT: st.param.b16 [func_retval0], %r10;
; SM70-NEXT: ret;
;
; SM80-LABEL: test_roundeven(
@@ -1631,7 +1650,7 @@ define bfloat @test_maxnum(bfloat %a, bfloat %b) {
; SM70-LABEL: test_maxnum(
; SM70: {
; SM70-NEXT: .reg .pred %p<2>;
-; SM70-NEXT: .reg .b32 %r<12>;
+; SM70-NEXT: .reg .b32 %r<13>;
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
; SM70-NEXT: ld.param.b16 %r1, [test_maxnum_param_1];
@@ -1639,14 +1658,15 @@ define bfloat @test_maxnum(bfloat %a, bfloat %b) {
; SM70-NEXT: ld.param.b16 %r3, [test_maxnum_param_0];
; SM70-NEXT: shl.b32 %r4, %r3, 16;
; SM70-NEXT: max.f32 %r5, %r4, %r2;
-; SM70-NEXT: bfe.u32 %r6, %r5, 16, 1;
-; SM70-NEXT: add.s32 %r7, %r6, %r5;
-; SM70-NEXT: add.s32 %r8, %r7, 32767;
+; SM70-NEXT: shr.u32 %r6, %r5, 16;
+; SM70-NEXT: and.b32 %r7, %r6, 1;
+; SM70-NEXT: add.s32 %r8, %r7, %r5;
+; SM70-NEXT: add.s32 %r9, %r8, 32767;
; SM70-NEXT: setp.nan.f32 %p1, %r5, %r5;
-; SM70-NEXT: or.b32 %r9, %r5, 4194304;
-; SM70-NEXT: selp.b32 %r10, %r9, %r8, %p1;
-; SM70-NEXT: shr.u32 %r11, %r10, 16;
-; SM70-NEXT: st.param.b16 [func_retval0], %r11;
+; SM70-NEXT: or.b32 %r10, %r5, 4194304;
+; SM70-NEXT: selp.b32 %r11, %r10, %r9, %p1;
+; SM70-NEXT: shr.u32 %r12, %r11, 16;
+; SM70-NEXT: st.param.b16 [func_retval0], %r12;
; SM70-NEXT: ret;
;
; SM80-LABEL: test_maxnum(
@@ -1785,7 +1805,7 @@ define <2 x bfloat> @test_maxnum_v2(<2 x bfloat> %a, <2 x bfloat> %b) {
; SM70: {
; SM70-NEXT: .reg .pred %p<3>;
; SM70-NEXT: .reg .b16 %rs<5>;
-; SM70-NEXT: .reg .b32 %r<22>;
+; SM70-NEXT: .reg .b32 %r<24>;
; SM70-EMPTY:
; SM70-NEXT: // %bb.0:
; SM70-NEXT: ld.param.v2.b16 {%rs1, %rs2}, [test_maxnum_v2_param_0];
@@ -1795,25 +1815,27 @@ define <2 x bfloat> @test_maxnum_v2(<2 x bfloat> %a, <2 x bfloat> %b) {
; SM70-NEXT: cvt.u32.u16 %r3, %rs2;
; SM70-NEXT: shl.b32 %r4, %r3, 16;
; SM70-NEXT: max.f32 %r5, %r4, %r2;
-; SM70-NEXT: bfe.u32 %r6, %r5, 16, 1;
-; SM70-NEXT: add.s32 %r7, %r6, %r5;
-; SM70-NEXT: add.s32 %r8, %r7, 32767;
+; SM70-NEXT: shr.u32 %r6, %r5, 16;
+; SM70-NEXT: and.b32 %r7, %r6, 1;
+; SM70-NEXT: add.s32 %r8, %r7, %r5;
+; SM70-NEXT: add.s32 %r9, %r8, 32767;
; SM70-NEXT: setp.nan.f32 %p1, %r5, %r5;
-; SM70-NEXT: or.b32 %r9, %r5, 4194304;
-; SM70-NEXT: selp.b32 %r10, %r9, %r8, %p1;
-; SM70-NEXT: cvt.u32.u16 %r11, %rs3;
-; SM70-NEXT: shl.b32 %r12, %r11, 16;
-; SM70-NEXT: cvt.u32.u16 %r13, %rs1;
-; SM70-NEXT: shl.b32 %r14, %r13, 16;
-; SM70-NEXT: max.f32 %r15, %r14, %r12;
-; SM70-NEXT: bfe.u32 %r16, %r15, 16, 1;
-; SM70-NEXT: add.s32 %r17, %r16, %r15;
-; SM70-NEXT: add.s32 %r18, %r17, 32767;
-; SM70-NEXT: setp.nan.f32 %p2, %r15, %r15;
-; SM70-NEXT: or.b32 %r19, %r15, 4194304;
-; SM70-NEXT: selp.b32 %r20, %r19, %r18, %p2;
-; SM70-NEXT: prmt.b32 %r21, %r20, %r10, 0x7632U;
-; SM70-NEXT: st.param.b32 [func_retval0], %r21;
+; SM70-NEXT: or.b32 %r10, %r5, 4194304;
+; SM70-NEXT: selp.b32 %r11, %r10, %r9, %p1;
+; SM70-NEXT: cvt.u32.u16 %r12, %rs3;
+; SM70-NEXT: shl.b32 %r13, %r12, 16;
+; SM70-NEXT: cvt.u32.u16 %r14, %rs1;
+; SM70-NEXT: shl.b32 %r15, %r14, 16;
+; SM70-NEXT: max.f32 %r16, %r15, %r13;
+; SM70-NEXT: shr.u32 %r17, %r16, 16;
+; SM70-NEXT: and.b32 %r18, %r17, 1;
+; SM70-NEXT: add.s32 %r19, %r18, %r16;
+; SM70-NEXT: add.s32 %r20, %r19, 32767;
+; SM70-NEXT: setp.nan.f32 %p2, %r16, %r16;
+; SM70-NEXT: or.b32 %r21, %r16, 4194304;
+; SM70-NEXT: selp.b32 %r22, %r21, %r20, %p2;
+; SM70-NEXT: prmt.b32 %r23, %r22, %r11, 0x7632U;
+; SM70-NEXT: st.param.b32 [func_retval0], %r23;
; SM70-NEXT: ret;
;
; SM80-LABEL: test_maxnum_v2(
diff --git a/llvm/test/CodeGen/NVPTX/bfe.ll b/llvm/test/CodeGen/NVPTX/bfe.ll
index 644bf3606e8f6..0b5dab03b1af2 100644
--- a/llvm/test/CodeGen/NVPTX/bfe.ll
+++ b/llvm/test/CodeGen/NVPTX/bfe.ll
@@ -1,6 +1,7 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_20 | FileCheck %s --check-prefixes=CHECK,CHECK-O3
; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_20 -O0 | FileCheck %s --check-prefixes=CHECK,CHECK-O0
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_70 | FileCheck %s --check-prefix=SM70 --implicit-check-not=bfe.
; RUN: %if ptxas %{ llc < %s -mtriple=nvptx64 -mcpu=sm_20 | %ptxas-verify %}
; RUN: %if ptxas %{ llc < %s -mtriple=nvptx64 -mcpu=sm_20 -O0 | %ptxas-verify %}
@@ -16,6 +17,17 @@ define i32 @bfe0(i32 %a) {
; CHECK-NEXT: bfe.u32 %r2, %r1, 4, 4;
; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
; CHECK-NEXT: ret;
+;
+; SM70-LABEL: bfe0(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<4>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b32 %r1, [bfe0_param_0];
+; SM70-NEXT: shr.u32 %r2, %r1, 4;
+; SM70-NEXT: and.b32 %r3, %r2, 15;
+; SM70-NEXT: st.param.b32 [func_retval0], %r3;
+; SM70-NEXT: ret;
%val0 = ashr i32 %a, 4
%val1 = and i32 %val0, 15
ret i32 %val1
@@ -31,6 +43,17 @@ define i32 @bfe1(i32 %a) {
; CHECK-NEXT: bfe.u32 %r2, %r1, 3, 3;
; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
; CHECK-NEXT: ret;
+;
+; SM70-LABEL: bfe1(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<4>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b32 %r1, [bfe1_param_0];
+; SM70-NEXT: shr.u32 %r2, %r1, 3;
+; SM70-NEXT: and.b32 %r3, %r2, 7;
+; SM70-NEXT: st.param.b32 [func_retval0], %r3;
+; SM70-NEXT: ret;
%val0 = ashr i32 %a, 3
%val1 = and i32 %val0, 7
ret i32 %val1
@@ -46,11 +69,93 @@ define i32 @bfe2(i32 %a) {
; CHECK-NEXT: bfe.u32 %r2, %r1, 5, 3;
; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
; CHECK-NEXT: ret;
+;
+; SM70-LABEL: bfe2(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<4>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b32 %r1, [bfe2_param_0];
+; SM70-NEXT: shr.u32 %r2, %r1, 5;
+; SM70-NEXT: and.b32 %r3, %r2, 7;
+; SM70-NEXT: st.param.b32 [func_retval0], %r3;
+; SM70-NEXT: ret;
%val0 = ashr i32 %a, 5
%val1 = and i32 %val0, 7
ret i32 %val1
}
+define i32 @bfe_shifted_mask(i32 %x) {
+; CHECK-LABEL: bfe_shifted_mask(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [bfe_shifted_mask_param_0];
+; CHECK-NEXT: bfe.u32 %r2, %r1, 4, 12;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+;
+; SM70-LABEL: bfe_shifted_mask(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<4>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b32 %r1, [bfe_shifted_mask_param_0];
+; SM70-NEXT: and.b32 %r2, %r1, 65520;
+; SM70-NEXT: shr.u32 %r3, %r2, 4;
+; SM70-NEXT: st.param.b32 [func_retval0], %r3;
+; SM70-NEXT: ret;
+ %masked = and i32 %x, 65520
+ %result = lshr i32 %masked, 4
+ ret i32 %result
+}
+
+define i64 @no_bfe_i64_start_above_32(i64 %a, i64 %b) {
+; CHECK-O3-LABEL: no_bfe_i64_start_above_32(
+; CHECK-O3: {
+; CHECK-O3-NEXT: .reg .b64 %rd<6>;
+; CHECK-O3-EMPTY:
+; CHECK-O3-NEXT: // %bb.0:
+; CHECK-O3-NEXT: ld.param.b64 %rd1, [no_bfe_i64_start_above_32_param_0];
+; CHECK-O3-NEXT: ld.param.b64 %rd2, [no_bfe_i64_start_above_32_param_1];
+; CHECK-O3-NEXT: xor.b64 %rd3, %rd1, %rd2;
+; CHECK-O3-NEXT: shr.u64 %rd4, %rd3, 40;
+; CHECK-O3-NEXT: and.b64 %rd5, %rd4, 255;
+; CHECK-O3-NEXT: st.param.b64 [func_retval0], %rd5;
+; CHECK-O3-NEXT: ret;
+;
+; CHECK-O0-LABEL: no_bfe_i64_start_above_32(
+; CHECK-O0: {
+; CHECK-O0-NEXT: .reg .b64 %rd<6>;
+; CHECK-O0-EMPTY:
+; CHECK-O0-NEXT: // %bb.0:
+; CHECK-O0-NEXT: ld.param.b64 %rd2, [no_bfe_i64_start_above_32_param_1];
+; CHECK-O0-NEXT: ld.param.b64 %rd1, [no_bfe_i64_start_above_32_param_0];
+; CHECK-O0-NEXT: xor.b64 %rd3, %rd1, %rd2;
+; CHECK-O0-NEXT: shr.u64 %rd4, %rd3, 40;
+; CHECK-O0-NEXT: and.b64 %rd5, %rd4, 255;
+; CHECK-O0-NEXT: st.param.b64 [func_retval0], %rd5;
+; CHECK-O0-NEXT: ret;
+;
+; SM70-LABEL: no_bfe_i64_start_above_32(
+; SM70: {
+; SM70-NEXT: .reg .b64 %rd<6>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [no_bfe_i64_start_above_32_param_0];
+; SM70-NEXT: ld.param.b64 %rd2, [no_bfe_i64_start_above_32_param_1];
+; SM70-NEXT: xor.b64 %rd3, %rd1, %rd2;
+; SM70-NEXT: shr.u64 %rd4, %rd3, 40;
+; SM70-NEXT: and.b64 %rd5, %rd4, 255;
+; SM70-NEXT: st.param.b64 [func_retval0], %rd5;
+; SM70-NEXT: ret;
+ %x = xor i64 %a, %b
+ %val0 = lshr i64 %x, 40
+ %val1 = and i64 %val0, 255
+ ret i64 %val1
+}
+
define i32 @no_bfe_on_32bit_overflow(i32 %a) {
; CHECK-LABEL: no_bfe_on_32bit_overflow(
; CHECK: {
@@ -62,6 +167,17 @@ define i32 @no_bfe_on_32bit_overflow(i32 %a) {
; CHECK-NEXT: and.b32 %r3, %r2, 15;
; CHECK-NEXT: st.param.b32 [func_retval0], %r3;
; CHECK-NEXT: ret;
+;
+; SM70-LABEL: no_bfe_on_32bit_overflow(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<4>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b32 %r1, [no_bfe_on_32bit_overflow_param_0];
+; SM70-NEXT: shr.s32 %r2, %r1, 31;
+; SM70-NEXT: and.b32 %r3, %r2, 15;
+; SM70-NEXT: st.param.b32 [func_retval0], %r3;
+; SM70-NEXT: ret;
%val0 = ashr i32 %a, 31
%val1 = and i32 %val0, 15
ret i32 %val1
@@ -78,6 +194,17 @@ define i32 @no_bfe_on_32bit_overflow_shr_and_pair(i32 %a) {
; CHECK-NEXT: and.b32 %r3, %r2, 15;
; CHECK-NEXT: st.param.b32 [func_retval0], %r3;
; CHECK-NEXT: ret;
+;
+; SM70-LABEL: no_bfe_on_32bit_overflow_shr_and_pair(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<4>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b32 %r1, [no_bfe_on_32bit_overflow_shr_and_pair_param_0];
+; SM70-NEXT: shr.s32 %r2, %r1, 31;
+; SM70-NEXT: and.b32 %r3, %r2, 15;
+; SM70-NEXT: st.param.b32 [func_retval0], %r3;
+; SM70-NEXT: ret;
%val0 = ashr i32 %a, 31
%val1 = and i32 %val0, 15
ret i32 %val1
@@ -94,6 +221,17 @@ define i64 @no_bfe_on_64bit_overflow(i64 %a) {
; CHECK-NEXT: and.b64 %rd3, %rd2, 7;
; CHECK-NEXT: st.param.b64 [func_retval0], %rd3;
; CHECK-NEXT: ret;
+;
+; SM70-LABEL: no_bfe_on_64bit_overflow(
+; SM70: {
+; SM70-NEXT: .reg .b64 %rd<4>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [no_bfe_on_64bit_overflow_param_0];
+; SM70-NEXT: shr.s64 %rd2, %rd1, 63;
+; SM70-NEXT: and.b64 %rd3, %rd2, 7;
+; SM70-NEXT: st.param.b64 [func_retval0], %rd3;
+; SM70-NEXT: ret;
%val0 = ashr i64 %a, 63
%val1 = and i64 %val0, 7
ret i64 %val1
@@ -110,6 +248,17 @@ define i64 @no_bfe_on_64bit_overflow_shr_and_pair(i64 %a) {
; CHECK-NEXT: and.b64 %rd3, %rd2, 7;
; CHECK-NEXT: st.param.b64 [func_retval0], %rd3;
; CHECK-NEXT: ret;
+;
+; SM70-LABEL: no_bfe_on_64bit_overflow_shr_and_pair(
+; SM70: {
+; SM70-NEXT: .reg .b64 %rd<4>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [no_bfe_on_64bit_overflow_shr_and_pair_param_0];
+; SM70-NEXT: shr.s64 %rd2, %rd1, 63;
+; SM70-NEXT: and.b64 %rd3, %rd2, 7;
+; SM70-NEXT: st.param.b64 [func_retval0], %rd3;
+; SM70-NEXT: ret;
%val0 = ashr i64 %a, 63
%val1 = and i64 %val0, 7
ret i64 %val1
@@ -135,6 +284,17 @@ define i32 @bfe_ashr_signed_32(i32 %x) {
; CHECK-O0-NEXT: bfe.s32 %r2, %r1, 20, 12;
; CHECK-O0-NEXT: st.param.b32 [func_retval0], %r2;
; CHECK-O0-NEXT: ret;
+;
+; SM70-LABEL: bfe_ashr_signed_32(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<4>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b16 %r1, [bfe_ashr_signed_32_param_0+2];
+; SM70-NEXT: shl.b32 %r2, %r1, 16;
+; SM70-NEXT: shr.s32 %r3, %r2, 20;
+; SM70-NEXT: st.param.b32 [func_retval0], %r3;
+; SM70-NEXT: ret;
%and = and i32 %x, -65536
%shr = ashr exact i32 %and, 20
ret i32 %shr
@@ -150,6 +310,17 @@ define i32 @bfe_ashr_unsigned_32(i32 %x) {
; CHECK-NEXT: bfe.u32 %r2, %r1, 5, 6;
; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
; CHECK-NEXT: ret;
+;
+; SM70-LABEL: bfe_ashr_unsigned_32(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<4>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b32 %r1, [bfe_ashr_unsigned_32_param_0];
+; SM70-NEXT: and.b32 %r2, %r1, 2047;
+; SM70-NEXT: shr.u32 %r3, %r2, 5;
+; SM70-NEXT: st.param.b32 [func_retval0], %r3;
+; SM70-NEXT: ret;
%and = and i32 %x, 2047
%shr = ashr exact i32 %and, 5
ret i32 %shr
@@ -158,13 +329,25 @@ define i32 @bfe_ashr_unsigned_32(i32 %x) {
define i64 @bfe_ashr_signed_64(i64 %x) {
; CHECK-LABEL: bfe_ashr_signed_64(
; CHECK: {
-; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-NEXT: .reg .b64 %rd<4>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
; CHECK-NEXT: ld.param.b64 %rd1, [bfe_ashr_signed_64_param_0];
-; CHECK-NEXT: bfe.s64 %rd2, %rd1, 16, 48;
-; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: and.b64 %rd2, %rd1, -65536;
+; CHECK-NEXT: shr.s64 %rd3, %rd2, 16;
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd3;
; CHECK-NEXT: ret;
+;
+; SM70-LABEL: bfe_ashr_signed_64(
+; SM70: {
+; SM70-NEXT: .reg .b64 %rd<4>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [bfe_ashr_signed_64_param_0];
+; SM70-NEXT: and.b64 %rd2, %rd1, -65536;
+; SM70-NEXT: shr.s64 %rd3, %rd2, 16;
+; SM70-NEXT: st.param.b64 [func_retval0], %rd3;
+; SM70-NEXT: ret;
%and = and i64 %x, -65536
%shr = ashr exact i64 %and, 16
ret i64 %shr
@@ -173,13 +356,25 @@ define i64 @bfe_ashr_signed_64(i64 %x) {
define i64 @bfe_ashr_unsigned_64(i64 %x) {
; CHECK-LABEL: bfe_ashr_unsigned_64(
; CHECK: {
-; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-NEXT: .reg .b64 %rd<4>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
; CHECK-NEXT: ld.param.b64 %rd1, [bfe_ashr_unsigned_64_param_0];
-; CHECK-NEXT: bfe.u64 %rd2, %rd1, 5, 6;
-; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: and.b64 %rd2, %rd1, 2047;
+; CHECK-NEXT: shr.u64 %rd3, %rd2, 5;
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd3;
; CHECK-NEXT: ret;
+;
+; SM70-LABEL: bfe_ashr_unsigned_64(
+; SM70: {
+; SM70-NEXT: .reg .b64 %rd<4>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.b64 %rd1, [bfe_ashr_unsigned_64_param_0];
+; SM70-NEXT: and.b64 %rd2, %rd1, 2047;
+; SM70-NEXT: shr.u64 %rd3, %rd2, 5;
+; SM70-NEXT: st.param.b64 [func_retval0], %rd3;
+; SM70-NEXT: ret;
%and = and i64 %x, 2047
%shr = ashr exact i64 %and, 5
ret i64 %shr
@@ -197,6 +392,19 @@ define i32 @bfe3(i128 %a) {
; CHECK-NEXT: bfe.s32 %r2, %r1, 15, 17;
; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
; CHECK-NEXT: ret;
+;
+; SM70-LABEL: bfe3(
+; SM70: {
+; SM70-NEXT: .reg .b32 %r<4>;
+; SM70-NEXT: .reg .b64 %rd<3>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [bfe3_param_0];
+; SM70-NEXT: cvt.u32.u64 %r1, %rd1;
+; SM70-NEXT: and.b32 %r2, %r1, -32768;
+; SM70-NEXT: shr.s32 %r3, %r2, 15;
+; SM70-NEXT: st.param.b32 [func_retval0], %r3;
+; SM70-NEXT: ret;
%trunc = trunc i128 %a to i32
%and = and i32 %trunc, -32768
%shr = ashr exact i32 %and, 15
@@ -206,16 +414,27 @@ define i32 @bfe3(i128 %a) {
define i64 @bfe4(i128 %a) {
; CHECK-LABEL: bfe4(
; CHECK: {
-; CHECK-NEXT: .reg .b64 %rd<4>;
+; CHECK-NEXT: .reg .b64 %rd<5>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [bfe4_param_0];
-; CHECK-NEXT: bfe.s64 %rd3, %rd1, 17, 47;
-; CHECK-NEXT: st.param.b64 [func_retval0], %rd3;
+; CHECK-NEXT: and.b64 %rd3, %rd1, -131072;
+; CHECK-NEXT: shr.s64 %rd4, %rd3, 17;
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd4;
; CHECK-NEXT: ret;
+;
+; SM70-LABEL: bfe4(
+; SM70: {
+; SM70-NEXT: .reg .b64 %rd<5>;
+; SM70-EMPTY:
+; SM70-NEXT: // %bb.0:
+; SM70-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [bfe4_param_0];
+; SM70-NEXT: and.b64 %rd3, %rd1, -131072;
+; SM70-NEXT: shr.s64 %rd4, %rd3, 17;
+; SM70-NEXT: st.param.b64 [func_retval0], %rd4;
+; SM70-NEXT: ret;
%trunc = trunc i128 %a to i64
%and = and i64 %trunc, -131072
%shr = ashr exact i64 %and, 17
ret i64 %shr
}
-
diff --git a/llvm/test/CodeGen/NVPTX/convert-fp-i128.ll b/llvm/test/CodeGen/NVPTX/convert-fp-i128.ll
index f9eff6a8d6cd1..808a9e5262c84 100644
--- a/llvm/test/CodeGen/NVPTX/convert-fp-i128.ll
+++ b/llvm/test/CodeGen/NVPTX/convert-fp-i128.ll
@@ -32,36 +32,37 @@ define i128 @cvt_u128_f32(float %x) {
; CHECK-LABEL: cvt_u128_f32(
; CHECK: {
; CHECK-NEXT: .reg .pred %p<4>;
-; CHECK-NEXT: .reg .b32 %r<10>;
+; CHECK-NEXT: .reg .b32 %r<11>;
; CHECK-NEXT: .reg .b64 %rd<6>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0: // %fp-to-i-entry
; CHECK-NEXT: ld.param.b32 %r3, [cvt_u128_f32_param_0];
-; CHECK-NEXT: bfe.u32 %r1, %r3, 23, 8;
+; CHECK-NEXT: shr.u32 %r4, %r3, 23;
+; CHECK-NEXT: and.b32 %r1, %r4, 255;
; CHECK-NEXT: setp.lt.u32 %p1, %r1, 127;
; CHECK-NEXT: mov.b64 %rd5, 0;
; CHECK-NEXT: mov.b64 %rd4, %rd5;
; CHECK-NEXT: @%p1 bra $L__BB1_4;
; CHECK-NEXT: // %bb.1: // %fp-to-i-if-check.exp.size
-; CHECK-NEXT: and.b32 %r4, %r3, 8388607;
-; CHECK-NEXT: or.b32 %r2, %r4, 8388608;
+; CHECK-NEXT: and.b32 %r5, %r3, 8388607;
+; CHECK-NEXT: or.b32 %r2, %r5, 8388608;
; CHECK-NEXT: setp.gt.u32 %p2, %r1, 149;
; CHECK-NEXT: @%p2 bra $L__BB1_3;
; CHECK-NEXT: // %bb.2: // %fp-to-i-if-exp.small
-; CHECK-NEXT: sub.s32 %r8, 150, %r1;
-; CHECK-NEXT: shr.u32 %r9, %r2, %r8;
-; CHECK-NEXT: cvt.u64.u32 %rd4, %r9;
+; CHECK-NEXT: sub.s32 %r9, 150, %r1;
+; CHECK-NEXT: shr.u32 %r10, %r2, %r9;
+; CHECK-NEXT: cvt.u64.u32 %rd4, %r10;
; CHECK-NEXT: bra.uni $L__BB1_4;
; CHECK-NEXT: $L__BB1_3: // %fp-to-i-if-exp.large
-; CHECK-NEXT: add.s32 %r5, %r1, -150;
+; CHECK-NEXT: add.s32 %r6, %r1, -150;
; CHECK-NEXT: cvt.u64.u32 %rd1, %r2;
-; CHECK-NEXT: sub.s32 %r6, 214, %r1;
-; CHECK-NEXT: shr.u64 %rd2, %rd1, %r6;
-; CHECK-NEXT: add.s32 %r7, %r1, -214;
-; CHECK-NEXT: shl.b64 %rd3, %rd1, %r7;
-; CHECK-NEXT: setp.gt.s32 %p3, %r5, 63;
+; CHECK-NEXT: sub.s32 %r7, 214, %r1;
+; CHECK-NEXT: shr.u64 %rd2, %rd1, %r7;
+; CHECK-NEXT: add.s32 %r8, %r1, -214;
+; CHECK-NEXT: shl.b64 %rd3, %rd1, %r8;
+; CHECK-NEXT: setp.gt.s32 %p3, %r6, 63;
; CHECK-NEXT: selp.b64 %rd5, %rd3, %rd2, %p3;
-; CHECK-NEXT: shl.b64 %rd4, %rd1, %r5;
+; CHECK-NEXT: shl.b64 %rd4, %rd1, %r6;
; CHECK-NEXT: $L__BB1_4: // %fp-to-i-cleanup
; CHECK-NEXT: st.param.v2.b64 [func_retval0], {%rd4, %rd5};
; CHECK-NEXT: ret;
@@ -116,7 +117,7 @@ define half @cvt_f16_u128(i128 %x) {
; CHECK: {
; CHECK-NEXT: .reg .pred %p<10>;
; CHECK-NEXT: .reg .b16 %rs<2>;
-; CHECK-NEXT: .reg .b32 %r<19>;
+; CHECK-NEXT: .reg .b32 %r<20>;
; CHECK-NEXT: .reg .b64 %rd<28>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0: // %itofp-entry
@@ -135,7 +136,7 @@ define half @cvt_f16_u128(i128 %x) {
; CHECK-NEXT: selp.b64 %rd6, %rd5, %rd4, %p2;
; CHECK-NEXT: cvt.u32.u64 %r1, %rd6;
; CHECK-NEXT: sub.s32 %r2, 128, %r1;
-; CHECK-NEXT: sub.s32 %r18, 127, %r1;
+; CHECK-NEXT: sub.s32 %r19, 127, %r1;
; CHECK-NEXT: setp.lt.s32 %p3, %r2, 25;
; CHECK-NEXT: @%p3 bra $L__BB3_8;
; CHECK-NEXT: // %bb.2: // %itofp-if-then4
@@ -169,33 +170,34 @@ define half @cvt_f16_u128(i128 %x) {
; CHECK-NEXT: or.b64 %rd27, %rd12, %rd20;
; CHECK-NEXT: $L__BB3_6: // %itofp-sw-epilog
; CHECK-NEXT: cvt.u32.u64 %r11, %rd27;
-; CHECK-NEXT: bfe.u32 %r12, %r11, 2, 1;
-; CHECK-NEXT: cvt.u64.u32 %rd21, %r12;
+; CHECK-NEXT: shr.u32 %r12, %r11, 2;
+; CHECK-NEXT: and.b32 %r13, %r12, 1;
+; CHECK-NEXT: cvt.u64.u32 %rd21, %r13;
; CHECK-NEXT: or.b64 %rd22, %rd27, %rd21;
; CHECK-NEXT: add.s64 %rd23, %rd22, 1;
; CHECK-NEXT: shr.u64 %rd24, %rd23, 2;
; CHECK-NEXT: and.b64 %rd25, %rd23, 67108864;
; CHECK-NEXT: setp.eq.b64 %p9, %rd25, 0;
-; CHECK-NEXT: cvt.u32.u64 %r17, %rd24;
+; CHECK-NEXT: cvt.u32.u64 %r18, %rd24;
; CHECK-NEXT: @!%p9 bra $L__BB3_7;
; CHECK-NEXT: $L__BB3_9: // %itofp-if-end26
-; CHECK-NEXT: shl.b32 %r13, %r18, 23;
-; CHECK-NEXT: and.b32 %r14, %r17, 8388607;
-; CHECK-NEXT: or.b32 %r15, %r13, %r14;
-; CHECK-NEXT: add.s32 %r16, %r15, 1065353216;
-; CHECK-NEXT: cvt.rn.f16.f32 %rs1, %r16;
+; CHECK-NEXT: shl.b32 %r14, %r19, 23;
+; CHECK-NEXT: and.b32 %r15, %r18, 8388607;
+; CHECK-NEXT: or.b32 %r16, %r14, %r15;
+; CHECK-NEXT: add.s32 %r17, %r16, 1065353216;
+; CHECK-NEXT: cvt.rn.f16.f32 %rs1, %r17;
; CHECK-NEXT: $L__BB3_10: // %itofp-return
; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
; CHECK-NEXT: ret;
; CHECK-NEXT: $L__BB3_8: // %itofp-if-else
; CHECK-NEXT: add.s32 %r5, %r1, -104;
; CHECK-NEXT: shl.b64 %rd7, %rd27, %r5;
-; CHECK-NEXT: cvt.u32.u64 %r17, %rd7;
+; CHECK-NEXT: cvt.u32.u64 %r18, %rd7;
; CHECK-NEXT: bra.uni $L__BB3_9;
; CHECK-NEXT: $L__BB3_7: // %itofp-if-then20
; CHECK-NEXT: shr.u64 %rd26, %rd23, 3;
-; CHECK-NEXT: cvt.u32.u64 %r17, %rd26;
-; CHECK-NEXT: mov.b32 %r18, %r2;
+; CHECK-NEXT: cvt.u32.u64 %r18, %rd26;
+; CHECK-NEXT: mov.b32 %r19, %r2;
; CHECK-NEXT: bra.uni $L__BB3_9;
; CHECK-NEXT: $L__BB3_4: // %itofp-sw-bb
; CHECK-NEXT: shl.b64 %rd27, %rd27, 1;
@@ -208,14 +210,14 @@ define float @cvt_f32_u128(i128 %x) {
; CHECK-LABEL: cvt_f32_u128(
; CHECK: {
; CHECK-NEXT: .reg .pred %p<10>;
-; CHECK-NEXT: .reg .b32 %r<19>;
+; CHECK-NEXT: .reg .b32 %r<20>;
; CHECK-NEXT: .reg .b64 %rd<28>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0: // %itofp-entry
; CHECK-NEXT: ld.param.v2.b64 {%rd27, %rd1}, [cvt_f32_u128_param_0];
; CHECK-NEXT: or.b64 %rd2, %rd27, %rd1;
; CHECK-NEXT: setp.eq.b64 %p1, %rd2, 0;
-; CHECK-NEXT: mov.b32 %r18, 0f00000000;
+; CHECK-NEXT: mov.b32 %r19, 0f00000000;
; CHECK-NEXT: @%p1 bra $L__BB4_10;
; CHECK-NEXT: // %bb.1: // %itofp-if-end
; CHECK-NEXT: setp.ne.b64 %p2, %rd1, 0;
@@ -227,7 +229,7 @@ define float @cvt_f32_u128(i128 %x) {
; CHECK-NEXT: selp.b64 %rd6, %rd5, %rd4, %p2;
; CHECK-NEXT: cvt.u32.u64 %r1, %rd6;
; CHECK-NEXT: sub.s32 %r2, 128, %r1;
-; CHECK-NEXT: sub.s32 %r17, 127, %r1;
+; CHECK-NEXT: sub.s32 %r18, 127, %r1;
; CHECK-NEXT: setp.lt.s32 %p3, %r2, 25;
; CHECK-NEXT: @%p3 bra $L__BB4_8;
; CHECK-NEXT: // %bb.2: // %itofp-if-then4
@@ -261,32 +263,33 @@ define float @cvt_f32_u128(i128 %x) {
; CHECK-NEXT: or.b64 %rd27, %rd12, %rd20;
; CHECK-NEXT: $L__BB4_6: // %itofp-sw-epilog
; CHECK-NEXT: cvt.u32.u64 %r11, %rd27;
-; CHECK-NEXT: bfe.u32 %r12, %r11, 2, 1;
-; CHECK-NEXT: cvt.u64.u32 %rd21, %r12;
+; CHECK-NEXT: shr.u32 %r12, %r11, 2;
+; CHECK-NEXT: and.b32 %r13, %r12, 1;
+; CHECK-NEXT: cvt.u64.u32 %rd21, %r13;
; CHECK-NEXT: or.b64 %rd22, %rd27, %rd21;
; CHECK-NEXT: add.s64 %rd23, %rd22, 1;
; CHECK-NEXT: shr.u64 %rd24, %rd23, 2;
; CHECK-NEXT: and.b64 %rd25, %rd23, 67108864;
; CHECK-NEXT: setp.eq.b64 %p9, %rd25, 0;
-; CHECK-NEXT: cvt.u32.u64 %r16, %rd24;
+; CHECK-NEXT: cvt.u32.u64 %r17, %rd24;
; CHECK-NEXT: @!%p9 bra $L__BB4_7;
; CHECK-NEXT: $L__BB4_9: // %itofp-if-end26
-; CHECK-NEXT: shl.b32 %r13, %r17, 23;
-; CHECK-NEXT: and.b32 %r14, %r16, 8388607;
-; CHECK-NEXT: or.b32 %r15, %r13, %r14;
-; CHECK-NEXT: add.s32 %r18, %r15, 1065353216;
+; CHECK-NEXT: shl.b32 %r14, %r18, 23;
+; CHECK-NEXT: and.b32 %r15, %r17, 8388607;
+; CHECK-NEXT: or.b32 %r16, %r14, %r15;
+; CHECK-NEXT: add.s32 %r19, %r16, 1065353216;
; CHECK-NEXT: $L__BB4_10: // %itofp-return
-; CHECK-NEXT: st.param.b32 [func_retval0], %r18;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r19;
; CHECK-NEXT: ret;
; CHECK-NEXT: $L__BB4_8: // %itofp-if-else
; CHECK-NEXT: add.s32 %r5, %r1, -104;
; CHECK-NEXT: shl.b64 %rd7, %rd27, %r5;
-; CHECK-NEXT: cvt.u32.u64 %r16, %rd7;
+; CHECK-NEXT: cvt.u32.u64 %r17, %rd7;
; CHECK-NEXT: bra.uni $L__BB4_9;
; CHECK-NEXT: $L__BB4_7: // %itofp-if-then20
; CHECK-NEXT: shr.u64 %rd26, %rd23, 3;
-; CHECK-NEXT: cvt.u32.u64 %r16, %rd26;
-; CHECK-NEXT: mov.b32 %r17, %r2;
+; CHECK-NEXT: cvt.u32.u64 %r17, %rd26;
+; CHECK-NEXT: mov.b32 %r18, %r2;
; CHECK-NEXT: bra.uni $L__BB4_9;
; CHECK-NEXT: $L__BB4_4: // %itofp-sw-bb
; CHECK-NEXT: shl.b64 %rd27, %rd27, 1;
@@ -299,7 +302,7 @@ define double @cvt_f64_u128(i128 %x) {
; CHECK-LABEL: cvt_f64_u128(
; CHECK: {
; CHECK-NEXT: .reg .pred %p<10>;
-; CHECK-NEXT: .reg .b32 %r<35>;
+; CHECK-NEXT: .reg .b32 %r<36>;
; CHECK-NEXT: .reg .b64 %rd<34>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0: // %itofp-entry
@@ -318,7 +321,7 @@ define double @cvt_f64_u128(i128 %x) {
; CHECK-NEXT: selp.b64 %rd6, %rd5, %rd4, %p2;
; CHECK-NEXT: cvt.u32.u64 %r1, %rd6;
; CHECK-NEXT: sub.s32 %r2, 128, %r1;
-; CHECK-NEXT: sub.s32 %r34, 127, %r1;
+; CHECK-NEXT: sub.s32 %r35, 127, %r1;
; CHECK-NEXT: setp.lt.s32 %p3, %r2, 54;
; CHECK-NEXT: @%p3 bra $L__BB5_8;
; CHECK-NEXT: // %bb.2: // %itofp-if-then4
@@ -354,28 +357,29 @@ define double @cvt_f64_u128(i128 %x) {
; CHECK-NEXT: mov.b64 %rd31, %rd1;
; CHECK-NEXT: $L__BB5_6: // %itofp-sw-epilog
; CHECK-NEXT: cvt.u32.u64 %r17, %rd30;
-; CHECK-NEXT: bfe.u32 %r18, %r17, 2, 1;
-; CHECK-NEXT: cvt.u64.u32 %rd20, %r18;
+; CHECK-NEXT: shr.u32 %r18, %r17, 2;
+; CHECK-NEXT: and.b32 %r19, %r18, 1;
+; CHECK-NEXT: cvt.u64.u32 %rd20, %r19;
; CHECK-NEXT: or.b64 %rd21, %rd30, %rd20;
; CHECK-NEXT: add.cc.s64 %rd22, %rd21, 1;
; CHECK-NEXT: addc.cc.s64 %rd23, %rd31, 0;
-; CHECK-NEXT: mov.b64 {%r19, %r20}, %rd22;
-; CHECK-NEXT: shf.l.wrap.b32 %r21, %r19, %r20, 30;
-; CHECK-NEXT: mov.b64 {%r22, %r23}, %rd23;
-; CHECK-NEXT: shf.l.wrap.b32 %r24, %r20, %r22, 30;
-; CHECK-NEXT: mov.b64 %rd32, {%r21, %r24};
+; CHECK-NEXT: mov.b64 {%r20, %r21}, %rd22;
+; CHECK-NEXT: shf.l.wrap.b32 %r22, %r20, %r21, 30;
+; CHECK-NEXT: mov.b64 {%r23, %r24}, %rd23;
+; CHECK-NEXT: shf.l.wrap.b32 %r25, %r21, %r23, 30;
+; CHECK-NEXT: mov.b64 %rd32, {%r22, %r25};
; CHECK-NEXT: and.b64 %rd24, %rd22, 36028797018963968;
; CHECK-NEXT: setp.eq.b64 %p9, %rd24, 0;
-; CHECK-NEXT: shf.l.wrap.b32 %r25, %r22, %r23, 30;
-; CHECK-NEXT: mov.b64 %rd25, {%r24, %r25};
-; CHECK-NEXT: cvt.u32.u64 %r33, %rd25;
+; CHECK-NEXT: shf.l.wrap.b32 %r26, %r23, %r24, 30;
+; CHECK-NEXT: mov.b64 %rd25, {%r25, %r26};
+; CHECK-NEXT: cvt.u32.u64 %r34, %rd25;
; CHECK-NEXT: @!%p9 bra $L__BB5_7;
; CHECK-NEXT: $L__BB5_9: // %itofp-if-end26
-; CHECK-NEXT: shl.b32 %r29, %r34, 20;
-; CHECK-NEXT: and.b32 %r30, %r33, 1048575;
-; CHECK-NEXT: or.b32 %r31, %r29, %r30;
-; CHECK-NEXT: add.s32 %r32, %r31, 1072693248;
-; CHECK-NEXT: cvt.u64.u32 %rd27, %r32;
+; CHECK-NEXT: shl.b32 %r30, %r35, 20;
+; CHECK-NEXT: and.b32 %r31, %r34, 1048575;
+; CHECK-NEXT: or.b32 %r32, %r30, %r31;
+; CHECK-NEXT: add.s32 %r33, %r32, 1072693248;
+; CHECK-NEXT: cvt.u64.u32 %rd27, %r33;
; CHECK-NEXT: shl.b64 %rd28, %rd27, 32;
; CHECK-NEXT: and.b64 %rd29, %rd32, 4294967295;
; CHECK-NEXT: or.b64 %rd33, %rd28, %rd29;
@@ -385,16 +389,16 @@ define double @cvt_f64_u128(i128 %x) {
; CHECK-NEXT: $L__BB5_8: // %itofp-if-else
; CHECK-NEXT: add.s32 %r5, %r1, -75;
; CHECK-NEXT: shl.b64 %rd32, %rd30, %r5;
-; CHECK-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r33}, %rd32; }
+; CHECK-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd32; }
; CHECK-NEXT: bra.uni $L__BB5_9;
; CHECK-NEXT: $L__BB5_7: // %itofp-if-then20
-; CHECK-NEXT: shf.l.wrap.b32 %r26, %r19, %r20, 29;
-; CHECK-NEXT: shf.l.wrap.b32 %r27, %r20, %r22, 29;
-; CHECK-NEXT: mov.b64 %rd32, {%r26, %r27};
-; CHECK-NEXT: shf.l.wrap.b32 %r28, %r22, %r23, 29;
-; CHECK-NEXT: mov.b64 %rd26, {%r27, %r28};
-; CHECK-NEXT: cvt.u32.u64 %r33, %rd26;
-; CHECK-NEXT: mov.b32 %r34, %r2;
+; CHECK-NEXT: shf.l.wrap.b32 %r27, %r20, %r21, 29;
+; CHECK-NEXT: shf.l.wrap.b32 %r28, %r21, %r23, 29;
+; CHECK-NEXT: mov.b64 %rd32, {%r27, %r28};
+; CHECK-NEXT: shf.l.wrap.b32 %r29, %r23, %r24, 29;
+; CHECK-NEXT: mov.b64 %rd26, {%r28, %r29};
+; CHECK-NEXT: cvt.u32.u64 %r34, %rd26;
+; CHECK-NEXT: mov.b32 %r35, %r2;
; CHECK-NEXT: bra.uni $L__BB5_9;
; CHECK-NEXT: $L__BB5_4: // %itofp-sw-bb
; CHECK-NEXT: mov.b64 {%r6, %r7}, %rd30;
@@ -430,12 +434,13 @@ define i128 @cvt_s128_f32(float %x) {
; CHECK-LABEL: cvt_s128_f32(
; CHECK: {
; CHECK-NEXT: .reg .pred %p<4>;
-; CHECK-NEXT: .reg .b32 %r<11>;
+; CHECK-NEXT: .reg .b32 %r<12>;
; CHECK-NEXT: .reg .b64 %rd<14>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0: // %fp-to-i-entry
; CHECK-NEXT: ld.param.b32 %r3, [cvt_s128_f32_param_0];
-; CHECK-NEXT: bfe.u32 %r1, %r3, 23, 8;
+; CHECK-NEXT: shr.u32 %r5, %r3, 23;
+; CHECK-NEXT: and.b32 %r1, %r5, 255;
; CHECK-NEXT: setp.lt.u32 %p1, %r1, 127;
; CHECK-NEXT: mov.b64 %rd12, 0;
; CHECK-NEXT: mov.b64 %rd13, %rd12;
@@ -444,28 +449,28 @@ define i128 @cvt_s128_f32(float %x) {
; CHECK-NEXT: shr.s32 %r4, %r3, 31;
; CHECK-NEXT: cvt.s64.s32 %rd2, %r4;
; CHECK-NEXT: or.b64 %rd1, %rd2, 1;
-; CHECK-NEXT: and.b32 %r5, %r3, 8388607;
-; CHECK-NEXT: or.b32 %r2, %r5, 8388608;
+; CHECK-NEXT: and.b32 %r6, %r3, 8388607;
+; CHECK-NEXT: or.b32 %r2, %r6, 8388608;
; CHECK-NEXT: setp.gt.u32 %p2, %r1, 149;
; CHECK-NEXT: @%p2 bra $L__BB7_3;
; CHECK-NEXT: // %bb.2: // %fp-to-i-if-exp.small
-; CHECK-NEXT: sub.s32 %r9, 150, %r1;
-; CHECK-NEXT: shr.u32 %r10, %r2, %r9;
-; CHECK-NEXT: cvt.u64.u32 %rd10, %r10;
+; CHECK-NEXT: sub.s32 %r10, 150, %r1;
+; CHECK-NEXT: shr.u32 %r11, %r2, %r10;
+; CHECK-NEXT: cvt.u64.u32 %rd10, %r11;
; CHECK-NEXT: mul.hi.u64 %rd11, %rd10, %rd1;
; CHECK-NEXT: mad.lo.s64 %rd13, %rd10, %rd2, %rd11;
; CHECK-NEXT: mul.lo.s64 %rd12, %rd10, %rd1;
; CHECK-NEXT: bra.uni $L__BB7_4;
; CHECK-NEXT: $L__BB7_3: // %fp-to-i-if-exp.large
-; CHECK-NEXT: add.s32 %r6, %r1, -150;
+; CHECK-NEXT: add.s32 %r7, %r1, -150;
; CHECK-NEXT: cvt.u64.u32 %rd3, %r2;
-; CHECK-NEXT: sub.s32 %r7, 214, %r1;
-; CHECK-NEXT: shr.u64 %rd4, %rd3, %r7;
-; CHECK-NEXT: add.s32 %r8, %r1, -214;
-; CHECK-NEXT: shl.b64 %rd5, %rd3, %r8;
-; CHECK-NEXT: setp.gt.s32 %p3, %r6, 63;
+; CHECK-NEXT: sub.s32 %r8, 214, %r1;
+; CHECK-NEXT: shr.u64 %rd4, %rd3, %r8;
+; CHECK-NEXT: add.s32 %r9, %r1, -214;
+; CHECK-NEXT: shl.b64 %rd5, %rd3, %r9;
+; CHECK-NEXT: setp.gt.s32 %p3, %r7, 63;
; CHECK-NEXT: selp.b64 %rd6, %rd5, %rd4, %p3;
-; CHECK-NEXT: shl.b64 %rd7, %rd3, %r6;
+; CHECK-NEXT: shl.b64 %rd7, %rd3, %r7;
; CHECK-NEXT: mul.hi.u64 %rd8, %rd7, %rd1;
; CHECK-NEXT: mad.lo.s64 %rd9, %rd7, %rd2, %rd8;
; CHECK-NEXT: mad.lo.s64 %rd13, %rd6, %rd1, %rd9;
@@ -533,7 +538,7 @@ define half @cvt_f16_s128(i128 %x) {
; CHECK: {
; CHECK-NEXT: .reg .pred %p<11>;
; CHECK-NEXT: .reg .b16 %rs<2>;
-; CHECK-NEXT: .reg .b32 %r<22>;
+; CHECK-NEXT: .reg .b32 %r<23>;
; CHECK-NEXT: .reg .b64 %rd<33>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0: // %itofp-entry
@@ -558,7 +563,7 @@ define half @cvt_f16_s128(i128 %x) {
; CHECK-NEXT: selp.b64 %rd11, %rd8, %rd10, %p3;
; CHECK-NEXT: cvt.u32.u64 %r1, %rd11;
; CHECK-NEXT: sub.s32 %r2, 128, %r1;
-; CHECK-NEXT: sub.s32 %r21, 127, %r1;
+; CHECK-NEXT: sub.s32 %r22, 127, %r1;
; CHECK-NEXT: setp.lt.s32 %p4, %r2, 25;
; CHECK-NEXT: @%p4 bra $L__BB9_8;
; CHECK-NEXT: // %bb.2: // %itofp-if-then4
@@ -592,36 +597,37 @@ define half @cvt_f16_s128(i128 %x) {
; CHECK-NEXT: or.b64 %rd32, %rd17, %rd25;
; CHECK-NEXT: $L__BB9_6: // %itofp-sw-epilog
; CHECK-NEXT: cvt.u32.u64 %r11, %rd32;
-; CHECK-NEXT: bfe.u32 %r12, %r11, 2, 1;
-; CHECK-NEXT: cvt.u64.u32 %rd26, %r12;
+; CHECK-NEXT: shr.u32 %r12, %r11, 2;
+; CHECK-NEXT: and.b32 %r13, %r12, 1;
+; CHECK-NEXT: cvt.u64.u32 %rd26, %r13;
; CHECK-NEXT: or.b64 %rd27, %rd32, %rd26;
; CHECK-NEXT: add.s64 %rd28, %rd27, 1;
; CHECK-NEXT: shr.u64 %rd29, %rd28, 2;
; CHECK-NEXT: and.b64 %rd30, %rd28, 67108864;
; CHECK-NEXT: setp.eq.b64 %p10, %rd30, 0;
-; CHECK-NEXT: cvt.u32.u64 %r20, %rd29;
+; CHECK-NEXT: cvt.u32.u64 %r21, %rd29;
; CHECK-NEXT: @!%p10 bra $L__BB9_7;
; CHECK-NEXT: $L__BB9_9: // %itofp-if-end26
-; CHECK-NEXT: cvt.u32.u64 %r13, %rd3;
-; CHECK-NEXT: and.b32 %r14, %r13, -2147483648;
-; CHECK-NEXT: shl.b32 %r15, %r21, 23;
-; CHECK-NEXT: add.s32 %r16, %r15, 1065353216;
-; CHECK-NEXT: and.b32 %r17, %r20, 8388607;
-; CHECK-NEXT: or.b32 %r18, %r17, %r14;
-; CHECK-NEXT: or.b32 %r19, %r18, %r16;
-; CHECK-NEXT: cvt.rn.f16.f32 %rs1, %r19;
+; CHECK-NEXT: cvt.u32.u64 %r14, %rd3;
+; CHECK-NEXT: and.b32 %r15, %r14, -2147483648;
+; CHECK-NEXT: shl.b32 %r16, %r22, 23;
+; CHECK-NEXT: add.s32 %r17, %r16, 1065353216;
+; CHECK-NEXT: and.b32 %r18, %r21, 8388607;
+; CHECK-NEXT: or.b32 %r19, %r18, %r15;
+; CHECK-NEXT: or.b32 %r20, %r19, %r17;
+; CHECK-NEXT: cvt.rn.f16.f32 %rs1, %r20;
; CHECK-NEXT: $L__BB9_10: // %itofp-return
; CHECK-NEXT: st.param.b16 [func_retval0], %rs1;
; CHECK-NEXT: ret;
; CHECK-NEXT: $L__BB9_8: // %itofp-if-else
; CHECK-NEXT: add.s32 %r5, %r1, -104;
; CHECK-NEXT: shl.b64 %rd12, %rd32, %r5;
-; CHECK-NEXT: cvt.u32.u64 %r20, %rd12;
+; CHECK-NEXT: cvt.u32.u64 %r21, %rd12;
; CHECK-NEXT: bra.uni $L__BB9_9;
; CHECK-NEXT: $L__BB9_7: // %itofp-if-then20
; CHECK-NEXT: shr.u64 %rd31, %rd28, 3;
-; CHECK-NEXT: cvt.u32.u64 %r20, %rd31;
-; CHECK-NEXT: mov.b32 %r21, %r2;
+; CHECK-NEXT: cvt.u32.u64 %r21, %rd31;
+; CHECK-NEXT: mov.b32 %r22, %r2;
; CHECK-NEXT: bra.uni $L__BB9_9;
; CHECK-NEXT: $L__BB9_4: // %itofp-sw-bb
; CHECK-NEXT: shl.b64 %rd32, %rd32, 1;
@@ -634,14 +640,14 @@ define float @cvt_f32_s128(i128 %x) {
; CHECK-LABEL: cvt_f32_s128(
; CHECK: {
; CHECK-NEXT: .reg .pred %p<11>;
-; CHECK-NEXT: .reg .b32 %r<22>;
+; CHECK-NEXT: .reg .b32 %r<23>;
; CHECK-NEXT: .reg .b64 %rd<33>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0: // %itofp-entry
; CHECK-NEXT: ld.param.v2.b64 {%rd1, %rd2}, [cvt_f32_s128_param_0];
; CHECK-NEXT: or.b64 %rd5, %rd1, %rd2;
; CHECK-NEXT: setp.eq.b64 %p1, %rd5, 0;
-; CHECK-NEXT: mov.b32 %r21, 0f00000000;
+; CHECK-NEXT: mov.b32 %r22, 0f00000000;
; CHECK-NEXT: @%p1 bra $L__BB10_10;
; CHECK-NEXT: // %bb.1: // %itofp-if-end
; CHECK-NEXT: shr.s64 %rd3, %rd2, 63;
@@ -659,7 +665,7 @@ define float @cvt_f32_s128(i128 %x) {
; CHECK-NEXT: selp.b64 %rd11, %rd8, %rd10, %p3;
; CHECK-NEXT: cvt.u32.u64 %r1, %rd11;
; CHECK-NEXT: sub.s32 %r2, 128, %r1;
-; CHECK-NEXT: sub.s32 %r20, 127, %r1;
+; CHECK-NEXT: sub.s32 %r21, 127, %r1;
; CHECK-NEXT: setp.lt.s32 %p4, %r2, 25;
; CHECK-NEXT: @%p4 bra $L__BB10_8;
; CHECK-NEXT: // %bb.2: // %itofp-if-then4
@@ -693,35 +699,36 @@ define float @cvt_f32_s128(i128 %x) {
; CHECK-NEXT: or.b64 %rd32, %rd17, %rd25;
; CHECK-NEXT: $L__BB10_6: // %itofp-sw-epilog
; CHECK-NEXT: cvt.u32.u64 %r11, %rd32;
-; CHECK-NEXT: bfe.u32 %r12, %r11, 2, 1;
-; CHECK-NEXT: cvt.u64.u32 %rd26, %r12;
+; CHECK-NEXT: shr.u32 %r12, %r11, 2;
+; CHECK-NEXT: and.b32 %r13, %r12, 1;
+; CHECK-NEXT: cvt.u64.u32 %rd26, %r13;
; CHECK-NEXT: or.b64 %rd27, %rd32, %rd26;
; CHECK-NEXT: add.s64 %rd28, %rd27, 1;
; CHECK-NEXT: shr.u64 %rd29, %rd28, 2;
; CHECK-NEXT: and.b64 %rd30, %rd28, 67108864;
; CHECK-NEXT: setp.eq.b64 %p10, %rd30, 0;
-; CHECK-NEXT: cvt.u32.u64 %r19, %rd29;
+; CHECK-NEXT: cvt.u32.u64 %r20, %rd29;
; CHECK-NEXT: @!%p10 bra $L__BB10_7;
; CHECK-NEXT: $L__BB10_9: // %itofp-if-end26
-; CHECK-NEXT: cvt.u32.u64 %r13, %rd3;
-; CHECK-NEXT: and.b32 %r14, %r13, -2147483648;
-; CHECK-NEXT: shl.b32 %r15, %r20, 23;
-; CHECK-NEXT: add.s32 %r16, %r15, 1065353216;
-; CHECK-NEXT: and.b32 %r17, %r19, 8388607;
-; CHECK-NEXT: or.b32 %r18, %r17, %r14;
-; CHECK-NEXT: or.b32 %r21, %r18, %r16;
+; CHECK-NEXT: cvt.u32.u64 %r14, %rd3;
+; CHECK-NEXT: and.b32 %r15, %r14, -2147483648;
+; CHECK-NEXT: shl.b32 %r16, %r21, 23;
+; CHECK-NEXT: add.s32 %r17, %r16, 1065353216;
+; CHECK-NEXT: and.b32 %r18, %r20, 8388607;
+; CHECK-NEXT: or.b32 %r19, %r18, %r15;
+; CHECK-NEXT: or.b32 %r22, %r19, %r17;
; CHECK-NEXT: $L__BB10_10: // %itofp-return
-; CHECK-NEXT: st.param.b32 [func_retval0], %r21;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r22;
; CHECK-NEXT: ret;
; CHECK-NEXT: $L__BB10_8: // %itofp-if-else
; CHECK-NEXT: add.s32 %r5, %r1, -104;
; CHECK-NEXT: shl.b64 %rd12, %rd32, %r5;
-; CHECK-NEXT: cvt.u32.u64 %r19, %rd12;
+; CHECK-NEXT: cvt.u32.u64 %r20, %rd12;
; CHECK-NEXT: bra.uni $L__BB10_9;
; CHECK-NEXT: $L__BB10_7: // %itofp-if-then20
; CHECK-NEXT: shr.u64 %rd31, %rd28, 3;
-; CHECK-NEXT: cvt.u32.u64 %r19, %rd31;
-; CHECK-NEXT: mov.b32 %r20, %r2;
+; CHECK-NEXT: cvt.u32.u64 %r20, %rd31;
+; CHECK-NEXT: mov.b32 %r21, %r2;
; CHECK-NEXT: bra.uni $L__BB10_9;
; CHECK-NEXT: $L__BB10_4: // %itofp-sw-bb
; CHECK-NEXT: shl.b64 %rd32, %rd32, 1;
@@ -734,7 +741,7 @@ define double @cvt_f64_s128(i128 %x) {
; CHECK-LABEL: cvt_f64_s128(
; CHECK: {
; CHECK-NEXT: .reg .pred %p<11>;
-; CHECK-NEXT: .reg .b32 %r<36>;
+; CHECK-NEXT: .reg .b32 %r<37>;
; CHECK-NEXT: .reg .b64 %rd<37>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0: // %itofp-entry
@@ -759,7 +766,7 @@ define double @cvt_f64_s128(i128 %x) {
; CHECK-NEXT: selp.b64 %rd11, %rd8, %rd10, %p3;
; CHECK-NEXT: cvt.u32.u64 %r1, %rd11;
; CHECK-NEXT: sub.s32 %r2, 128, %r1;
-; CHECK-NEXT: sub.s32 %r35, 127, %r1;
+; CHECK-NEXT: sub.s32 %r36, 127, %r1;
; CHECK-NEXT: setp.lt.s32 %p4, %r2, 54;
; CHECK-NEXT: @%p4 bra $L__BB11_8;
; CHECK-NEXT: // %bb.2: // %itofp-if-then4
@@ -795,29 +802,30 @@ define double @cvt_f64_s128(i128 %x) {
; CHECK-NEXT: mov.b64 %rd34, %rd4;
; CHECK-NEXT: $L__BB11_6: // %itofp-sw-epilog
; CHECK-NEXT: cvt.u32.u64 %r17, %rd33;
-; CHECK-NEXT: bfe.u32 %r18, %r17, 2, 1;
-; CHECK-NEXT: cvt.u64.u32 %rd25, %r18;
+; CHECK-NEXT: shr.u32 %r18, %r17, 2;
+; CHECK-NEXT: and.b32 %r19, %r18, 1;
+; CHECK-NEXT: cvt.u64.u32 %rd25, %r19;
; CHECK-NEXT: or.b64 %rd26, %rd33, %rd25;
; CHECK-NEXT: add.cc.s64 %rd27, %rd26, 1;
; CHECK-NEXT: addc.cc.s64 %rd28, %rd34, 0;
-; CHECK-NEXT: mov.b64 {%r19, %r20}, %rd28;
-; CHECK-NEXT: mov.b64 {%r21, %r22}, %rd27;
-; CHECK-NEXT: shf.l.wrap.b32 %r23, %r22, %r19, 30;
-; CHECK-NEXT: shf.l.wrap.b32 %r24, %r21, %r22, 30;
-; CHECK-NEXT: mov.b64 %rd35, {%r24, %r23};
+; CHECK-NEXT: mov.b64 {%r20, %r21}, %rd28;
+; CHECK-NEXT: mov.b64 {%r22, %r23}, %rd27;
+; CHECK-NEXT: shf.l.wrap.b32 %r24, %r23, %r20, 30;
+; CHECK-NEXT: shf.l.wrap.b32 %r25, %r22, %r23, 30;
+; CHECK-NEXT: mov.b64 %rd35, {%r25, %r24};
; CHECK-NEXT: and.b64 %rd29, %rd27, 36028797018963968;
; CHECK-NEXT: setp.eq.b64 %p10, %rd29, 0;
-; CHECK-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd35; }
+; CHECK-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r35}, %rd35; }
; CHECK-NEXT: @!%p10 bra $L__BB11_7;
; CHECK-NEXT: $L__BB11_9: // %itofp-if-end26
-; CHECK-NEXT: cvt.u32.u64 %r27, %rd3;
-; CHECK-NEXT: and.b32 %r28, %r27, -2147483648;
-; CHECK-NEXT: shl.b32 %r29, %r35, 20;
-; CHECK-NEXT: add.s32 %r30, %r29, 1072693248;
-; CHECK-NEXT: and.b32 %r31, %r34, 1048575;
-; CHECK-NEXT: or.b32 %r32, %r31, %r28;
-; CHECK-NEXT: or.b32 %r33, %r32, %r30;
-; CHECK-NEXT: cvt.u64.u32 %rd30, %r33;
+; CHECK-NEXT: cvt.u32.u64 %r28, %rd3;
+; CHECK-NEXT: and.b32 %r29, %r28, -2147483648;
+; CHECK-NEXT: shl.b32 %r30, %r36, 20;
+; CHECK-NEXT: add.s32 %r31, %r30, 1072693248;
+; CHECK-NEXT: and.b32 %r32, %r35, 1048575;
+; CHECK-NEXT: or.b32 %r33, %r32, %r29;
+; CHECK-NEXT: or.b32 %r34, %r33, %r31;
+; CHECK-NEXT: cvt.u64.u32 %rd30, %r34;
; CHECK-NEXT: shl.b64 %rd31, %rd30, 32;
; CHECK-NEXT: and.b64 %rd32, %rd35, 4294967295;
; CHECK-NEXT: or.b64 %rd36, %rd31, %rd32;
@@ -827,14 +835,14 @@ define double @cvt_f64_s128(i128 %x) {
; CHECK-NEXT: $L__BB11_8: // %itofp-if-else
; CHECK-NEXT: add.s32 %r5, %r1, -75;
; CHECK-NEXT: shl.b64 %rd35, %rd33, %r5;
-; CHECK-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd35; }
+; CHECK-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r35}, %rd35; }
; CHECK-NEXT: bra.uni $L__BB11_9;
; CHECK-NEXT: $L__BB11_7: // %itofp-if-then20
-; CHECK-NEXT: shf.l.wrap.b32 %r25, %r22, %r19, 29;
-; CHECK-NEXT: shf.l.wrap.b32 %r26, %r21, %r22, 29;
-; CHECK-NEXT: mov.b64 %rd35, {%r26, %r25};
-; CHECK-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd35; }
-; CHECK-NEXT: mov.b32 %r35, %r2;
+; CHECK-NEXT: shf.l.wrap.b32 %r26, %r23, %r20, 29;
+; CHECK-NEXT: shf.l.wrap.b32 %r27, %r22, %r23, 29;
+; CHECK-NEXT: mov.b64 %rd35, {%r27, %r26};
+; CHECK-NEXT: { .reg .b32 tmp; mov.b64 {tmp, %r35}, %rd35; }
+; CHECK-NEXT: mov.b32 %r36, %r2;
; CHECK-NEXT: bra.uni $L__BB11_9;
; CHECK-NEXT: $L__BB11_4: // %itofp-sw-bb
; CHECK-NEXT: mov.b64 {%r6, %r7}, %rd33;
diff --git a/llvm/test/CodeGen/NVPTX/fexp2.ll b/llvm/test/CodeGen/NVPTX/fexp2.ll
index 2131014c7c4bd..7c2494e5804d8 100644
--- a/llvm/test/CodeGen/NVPTX/fexp2.ll
+++ b/llvm/test/CodeGen/NVPTX/fexp2.ll
@@ -271,20 +271,21 @@ define bfloat @exp2_bf16_test(bfloat %in) {
; CHECK-FP16-LABEL: exp2_bf16_test(
; CHECK-FP16: {
; CHECK-FP16-NEXT: .reg .pred %p<2>;
-; CHECK-FP16-NEXT: .reg .b32 %r<10>;
+; CHECK-FP16-NEXT: .reg .b32 %r<11>;
; CHECK-FP16-EMPTY:
; CHECK-FP16-NEXT: // %bb.0: // %entry
; CHECK-FP16-NEXT: ld.param.b16 %r1, [exp2_bf16_test_param_0];
; CHECK-FP16-NEXT: shl.b32 %r2, %r1, 16;
; CHECK-FP16-NEXT: ex2.approx.f32 %r3, %r2;
-; CHECK-FP16-NEXT: bfe.u32 %r4, %r3, 16, 1;
-; CHECK-FP16-NEXT: add.s32 %r5, %r4, %r3;
-; CHECK-FP16-NEXT: add.s32 %r6, %r5, 32767;
+; CHECK-FP16-NEXT: shr.u32 %r4, %r3, 16;
+; CHECK-FP16-NEXT: and.b32 %r5, %r4, 1;
+; CHECK-FP16-NEXT: add.s32 %r6, %r5, %r3;
+; CHECK-FP16-NEXT: add.s32 %r7, %r6, 32767;
; CHECK-FP16-NEXT: setp.nan.f32 %p1, %r3, %r3;
-; CHECK-FP16-NEXT: or.b32 %r7, %r3, 4194304;
-; CHECK-FP16-NEXT: selp.b32 %r8, %r7, %r6, %p1;
-; CHECK-FP16-NEXT: shr.u32 %r9, %r8, 16;
-; CHECK-FP16-NEXT: st.param.b16 [func_retval0], %r9;
+; CHECK-FP16-NEXT: or.b32 %r8, %r3, 4194304;
+; CHECK-FP16-NEXT: selp.b32 %r9, %r8, %r7, %p1;
+; CHECK-FP16-NEXT: shr.u32 %r10, %r9, 16;
+; CHECK-FP16-NEXT: st.param.b16 [func_retval0], %r10;
; CHECK-FP16-NEXT: ret;
;
; CHECK-BF16-LABEL: exp2_bf16_test(
@@ -337,30 +338,32 @@ define <2 x bfloat> @exp2_bf16_test_v(<2 x bfloat> %in) {
; CHECK-FP16: {
; CHECK-FP16-NEXT: .reg .pred %p<3>;
; CHECK-FP16-NEXT: .reg .b16 %rs<3>;
-; CHECK-FP16-NEXT: .reg .b32 %r<18>;
+; CHECK-FP16-NEXT: .reg .b32 %r<20>;
; CHECK-FP16-EMPTY:
; CHECK-FP16-NEXT: // %bb.0: // %entry
; CHECK-FP16-NEXT: ld.param.v2.b16 {%rs1, %rs2}, [exp2_bf16_test_v_param_0];
; CHECK-FP16-NEXT: cvt.u32.u16 %r1, %rs2;
; CHECK-FP16-NEXT: shl.b32 %r2, %r1, 16;
; CHECK-FP16-NEXT: ex2.approx.f32 %r3, %r2;
-; CHECK-FP16-NEXT: bfe.u32 %r4, %r3, 16, 1;
-; CHECK-FP16-NEXT: add.s32 %r5, %r4, %r3;
-; CHECK-FP16-NEXT: add.s32 %r6, %r5, 32767;
+; CHECK-FP16-NEXT: shr.u32 %r4, %r3, 16;
+; CHECK-FP16-NEXT: and.b32 %r5, %r4, 1;
+; CHECK-FP16-NEXT: add.s32 %r6, %r5, %r3;
+; CHECK-FP16-NEXT: add.s32 %r7, %r6, 32767;
; CHECK-FP16-NEXT: setp.nan.f32 %p1, %r3, %r3;
-; CHECK-FP16-NEXT: or.b32 %r7, %r3, 4194304;
-; CHECK-FP16-NEXT: selp.b32 %r8, %r7, %r6, %p1;
-; CHECK-FP16-NEXT: cvt.u32.u16 %r9, %rs1;
-; CHECK-FP16-NEXT: shl.b32 %r10, %r9, 16;
-; CHECK-FP16-NEXT: ex2.approx.f32 %r11, %r10;
-; CHECK-FP16-NEXT: bfe.u32 %r12, %r11, 16, 1;
-; CHECK-FP16-NEXT: add.s32 %r13, %r12, %r11;
-; CHECK-FP16-NEXT: add.s32 %r14, %r13, 32767;
-; CHECK-FP16-NEXT: setp.nan.f32 %p2, %r11, %r11;
-; CHECK-FP16-NEXT: or.b32 %r15, %r11, 4194304;
-; CHECK-FP16-NEXT: selp.b32 %r16, %r15, %r14, %p2;
-; CHECK-FP16-NEXT: prmt.b32 %r17, %r16, %r8, 0x7632U;
-; CHECK-FP16-NEXT: st.param.b32 [func_retval0], %r17;
+; CHECK-FP16-NEXT: or.b32 %r8, %r3, 4194304;
+; CHECK-FP16-NEXT: selp.b32 %r9, %r8, %r7, %p1;
+; CHECK-FP16-NEXT: cvt.u32.u16 %r10, %rs1;
+; CHECK-FP16-NEXT: shl.b32 %r11, %r10, 16;
+; CHECK-FP16-NEXT: ex2.approx.f32 %r12, %r11;
+; CHECK-FP16-NEXT: shr.u32 %r13, %r12, 16;
+; CHECK-FP16-NEXT: and.b32 %r14, %r13, 1;
+; CHECK-FP16-NEXT: add.s32 %r15, %r14, %r12;
+; CHECK-FP16-NEXT: add.s32 %r16, %r15, 32767;
+; CHECK-FP16-NEXT: setp.nan.f32 %p2, %r12, %r12;
+; CHECK-FP16-NEXT: or.b32 %r17, %r12, 4194304;
+; CHECK-FP16-NEXT: selp.b32 %r18, %r17, %r16, %p2;
+; CHECK-FP16-NEXT: prmt.b32 %r19, %r18, %r9, 0x7632U;
+; CHECK-FP16-NEXT: st.param.b32 [func_retval0], %r19;
; CHECK-FP16-NEXT: ret;
;
; CHECK-BF16-LABEL: exp2_bf16_test_v(
diff --git a/llvm/test/CodeGen/NVPTX/float-to-arbitrary-fp.ll b/llvm/test/CodeGen/NVPTX/float-to-arbitrary-fp.ll
index 16ac8c8634db5..585e6021d1a04 100644
--- a/llvm/test/CodeGen/NVPTX/float-to-arbitrary-fp.ll
+++ b/llvm/test/CodeGen/NVPTX/float-to-arbitrary-fp.ll
@@ -72,15 +72,15 @@ define i8 @to_f8e5m2_dynamic(float %x) {
; CHECK-NEXT: and.b32 %r37, %r36, 128;
; CHECK-NEXT: or.b32 %r38, %r37, %r35;
; CHECK-NEXT: or.b32 %r39, %r38, %r34;
-; CHECK-NEXT: bfe.u32 %r40, %r9, 21, 1;
-; CHECK-NEXT: and.b32 %r41, %r8, 1048575;
-; CHECK-NEXT: setp.ne.b32 %p6, %r41, 0;
-; CHECK-NEXT: selp.b32 %r42, 1, 0, %p6;
-; CHECK-NEXT: or.b32 %r43, %r42, %r40;
-; CHECK-NEXT: shr.u32 %r44, %r8, 20;
-; CHECK-NEXT: and.b32 %r45, %r44, %r43;
-; CHECK-NEXT: bfe.u32 %r46, %r8, 21, 2;
-; CHECK-NEXT: add.s32 %r47, %r46, %r45;
+; CHECK-NEXT: shr.u32 %r40, %r9, 21;
+; CHECK-NEXT: and.b32 %r41, %r40, 1;
+; CHECK-NEXT: and.b32 %r42, %r8, 1048575;
+; CHECK-NEXT: setp.ne.b32 %p6, %r42, 0;
+; CHECK-NEXT: selp.b32 %r43, 1, 0, %p6;
+; CHECK-NEXT: or.b32 %r44, %r43, %r41;
+; CHECK-NEXT: shr.u32 %r45, %r8, 20;
+; CHECK-NEXT: and.b32 %r46, %r45, %r44;
+; CHECK-NEXT: add.s32 %r47, %r40, %r46;
; CHECK-NEXT: setp.gt.s32 %p7, %r47, 3;
; CHECK-NEXT: selp.b32 %r48, 0, %r47, %p7;
; CHECK-NEXT: selp.b32 %r49, 1, 0, %p7;
@@ -129,45 +129,45 @@ define i8 @to_f8e5m2_round_towardzero(float %x) {
; CHECK-NEXT: add.s32 %r5, %r3, -2139095040;
; CHECK-NEXT: setp.lt.u32 %p2, %r5, -2139095039;
; CHECK-NEXT: selp.f32 %r6, %r1, %r4, %p2;
-; CHECK-NEXT: bfe.u32 %r7, %r6, 21, 2;
-; CHECK-NEXT: setp.gt.s32 %p3, %r7, 3;
-; CHECK-NEXT: selp.b32 %r8, 0, %r7, %p3;
-; CHECK-NEXT: and.b32 %r9, %r2, 2139095040;
-; CHECK-NEXT: selp.b32 %r10, %r9, %r3, %p1;
-; CHECK-NEXT: shr.u32 %r11, %r10, 23;
-; CHECK-NEXT: selp.b32 %r12, -25, 0, %p1;
-; CHECK-NEXT: add.s32 %r13, %r11, %r12;
-; CHECK-NEXT: add.s32 %r14, %r13, -126;
-; CHECK-NEXT: selp.b32 %r15, 0, %r14, %p2;
-; CHECK-NEXT: selp.b32 %r16, 1, 0, %p3;
-; CHECK-NEXT: add.s32 %r17, %r15, %r16;
-; CHECK-NEXT: add.s32 %r18, %r17, 14;
-; CHECK-NEXT: shl.b32 %r19, %r18, 2;
-; CHECK-NEXT: shr.u32 %r20, %r1, 24;
-; CHECK-NEXT: and.b32 %r21, %r20, 128;
-; CHECK-NEXT: or.b32 %r22, %r21, %r19;
-; CHECK-NEXT: or.b32 %r23, %r22, %r8;
-; CHECK-NEXT: and.b32 %r24, %r6, 8388607;
-; CHECK-NEXT: or.b32 %r25, %r24, 8388608;
-; CHECK-NEXT: sub.s32 %r26, 8, %r15;
+; CHECK-NEXT: and.b32 %r7, %r6, 8388607;
+; CHECK-NEXT: shr.u32 %r8, %r7, 21;
+; CHECK-NEXT: setp.gt.s32 %p3, %r8, 3;
+; CHECK-NEXT: selp.b32 %r9, 0, %r8, %p3;
+; CHECK-NEXT: and.b32 %r10, %r2, 2139095040;
+; CHECK-NEXT: selp.b32 %r11, %r10, %r3, %p1;
+; CHECK-NEXT: shr.u32 %r12, %r11, 23;
+; CHECK-NEXT: selp.b32 %r13, -25, 0, %p1;
+; CHECK-NEXT: add.s32 %r14, %r12, %r13;
+; CHECK-NEXT: add.s32 %r15, %r14, -126;
+; CHECK-NEXT: selp.b32 %r16, 0, %r15, %p2;
+; CHECK-NEXT: selp.b32 %r17, 1, 0, %p3;
+; CHECK-NEXT: add.s32 %r18, %r16, %r17;
+; CHECK-NEXT: add.s32 %r19, %r18, 14;
+; CHECK-NEXT: shl.b32 %r20, %r19, 2;
+; CHECK-NEXT: shr.u32 %r21, %r1, 24;
+; CHECK-NEXT: and.b32 %r22, %r21, 128;
+; CHECK-NEXT: or.b32 %r23, %r22, %r20;
+; CHECK-NEXT: or.b32 %r24, %r23, %r9;
+; CHECK-NEXT: or.b32 %r25, %r7, 8388608;
+; CHECK-NEXT: sub.s32 %r26, 8, %r16;
; CHECK-NEXT: min.u32 %r27, %r26, 31;
; CHECK-NEXT: shr.u32 %r28, %r25, %r27;
; CHECK-NEXT: setp.gt.s32 %p4, %r28, 3;
; CHECK-NEXT: selp.b32 %r29, 0, %r28, %p4;
; CHECK-NEXT: selp.b32 %r30, 4, 0, %p4;
-; CHECK-NEXT: or.b32 %r31, %r21, %r30;
+; CHECK-NEXT: or.b32 %r31, %r22, %r30;
; CHECK-NEXT: or.b32 %r32, %r31, %r29;
-; CHECK-NEXT: setp.lt.s32 %p5, %r18, 1;
-; CHECK-NEXT: selp.b32 %r33, %r32, %r23, %p5;
-; CHECK-NEXT: setp.gt.s32 %p6, %r8, 3;
-; CHECK-NEXT: or.b32 %r34, %r21, 124;
+; CHECK-NEXT: setp.lt.s32 %p5, %r19, 1;
+; CHECK-NEXT: selp.b32 %r33, %r32, %r24, %p5;
+; CHECK-NEXT: setp.gt.s32 %p6, %r9, 3;
+; CHECK-NEXT: or.b32 %r34, %r22, 124;
; CHECK-NEXT: selp.b32 %r35, %r34, %r33, %p6;
-; CHECK-NEXT: setp.eq.b32 %p7, %r18, 30;
+; CHECK-NEXT: setp.eq.b32 %p7, %r19, 30;
; CHECK-NEXT: selp.b32 %r36, %r35, %r33, %p7;
-; CHECK-NEXT: setp.gt.s32 %p8, %r18, 30;
+; CHECK-NEXT: setp.gt.s32 %p8, %r19, 30;
; CHECK-NEXT: selp.b32 %r37, %r34, %r36, %p8;
; CHECK-NEXT: setp.eq.f32 %p9, %r1, 0f00000000;
-; CHECK-NEXT: selp.b32 %r38, %r21, %r37, %p9;
+; CHECK-NEXT: selp.b32 %r38, %r22, %r37, %p9;
; CHECK-NEXT: abs.f32 %r39, %r1;
; CHECK-NEXT: setp.eq.f32 %p10, %r39, 0f7F800000;
; CHECK-NEXT: selp.b32 %r40, %r34, %r38, %p10;
@@ -231,7 +231,7 @@ define <4 x i4> @to_f4e2m1fn_v4f32(<4 x float> %x) {
; CHECK-NEXT: and.b32 %r40, %r39, 8;
; CHECK-NEXT: or.b32 %r41, %r40, %r38;
; CHECK-NEXT: or.b32 %r42, %r41, %r37;
-; CHECK-NEXT: bfe.u32 %r43, %r11, 22, 1;
+; CHECK-NEXT: shr.u32 %r43, %r12, 22;
; CHECK-NEXT: and.b32 %r44, %r11, 2097151;
; CHECK-NEXT: setp.ne.b32 %p6, %r44, 0;
; CHECK-NEXT: selp.b32 %r45, 1, 0, %p6;
@@ -292,7 +292,7 @@ define <4 x i4> @to_f4e2m1fn_v4f32(<4 x float> %x) {
; CHECK-NEXT: and.b32 %r92, %r91, 8;
; CHECK-NEXT: or.b32 %r93, %r92, %r90;
; CHECK-NEXT: or.b32 %r94, %r93, %r89;
-; CHECK-NEXT: bfe.u32 %r95, %r64, 22, 1;
+; CHECK-NEXT: shr.u32 %r95, %r65, 22;
; CHECK-NEXT: and.b32 %r96, %r64, 2097151;
; CHECK-NEXT: setp.ne.b32 %p15, %r96, 0;
; CHECK-NEXT: selp.b32 %r97, 1, 0, %p15;
@@ -354,7 +354,7 @@ define <4 x i4> @to_f4e2m1fn_v4f32(<4 x float> %x) {
; CHECK-NEXT: and.b32 %r145, %r144, 8;
; CHECK-NEXT: or.b32 %r146, %r145, %r143;
; CHECK-NEXT: or.b32 %r147, %r146, %r142;
-; CHECK-NEXT: bfe.u32 %r148, %r117, 22, 1;
+; CHECK-NEXT: shr.u32 %r148, %r118, 22;
; CHECK-NEXT: and.b32 %r149, %r117, 2097151;
; CHECK-NEXT: setp.ne.b32 %p24, %r149, 0;
; CHECK-NEXT: selp.b32 %r150, 1, 0, %p24;
@@ -415,7 +415,7 @@ define <4 x i4> @to_f4e2m1fn_v4f32(<4 x float> %x) {
; CHECK-NEXT: and.b32 %r197, %r196, 8;
; CHECK-NEXT: or.b32 %r198, %r197, %r195;
; CHECK-NEXT: or.b32 %r199, %r198, %r194;
-; CHECK-NEXT: bfe.u32 %r200, %r169, 22, 1;
+; CHECK-NEXT: shr.u32 %r200, %r170, 22;
; CHECK-NEXT: and.b32 %r201, %r169, 2097151;
; CHECK-NEXT: setp.ne.b32 %p33, %r201, 0;
; CHECK-NEXT: selp.b32 %r202, 1, 0, %p33;
@@ -497,15 +497,15 @@ define <2 x i8> @to_f8e5m2_v2f32(<2 x float> %x) {
; CHECK-NEXT: and.b32 %r38, %r37, 128;
; CHECK-NEXT: or.b32 %r39, %r38, %r36;
; CHECK-NEXT: or.b32 %r40, %r39, %r35;
-; CHECK-NEXT: bfe.u32 %r41, %r10, 21, 1;
-; CHECK-NEXT: and.b32 %r42, %r9, 1048575;
-; CHECK-NEXT: setp.ne.b32 %p6, %r42, 0;
-; CHECK-NEXT: selp.b32 %r43, 1, 0, %p6;
-; CHECK-NEXT: or.b32 %r44, %r43, %r41;
-; CHECK-NEXT: shr.u32 %r45, %r9, 20;
-; CHECK-NEXT: and.b32 %r46, %r45, %r44;
-; CHECK-NEXT: bfe.u32 %r47, %r9, 21, 2;
-; CHECK-NEXT: add.s32 %r48, %r47, %r46;
+; CHECK-NEXT: shr.u32 %r41, %r10, 21;
+; CHECK-NEXT: and.b32 %r42, %r41, 1;
+; CHECK-NEXT: and.b32 %r43, %r9, 1048575;
+; CHECK-NEXT: setp.ne.b32 %p6, %r43, 0;
+; CHECK-NEXT: selp.b32 %r44, 1, 0, %p6;
+; CHECK-NEXT: or.b32 %r45, %r44, %r42;
+; CHECK-NEXT: shr.u32 %r46, %r9, 20;
+; CHECK-NEXT: and.b32 %r47, %r46, %r45;
+; CHECK-NEXT: add.s32 %r48, %r41, %r47;
; CHECK-NEXT: setp.gt.s32 %p7, %r48, 3;
; CHECK-NEXT: selp.b32 %r49, 0, %r48, %p7;
; CHECK-NEXT: selp.b32 %r50, 1, 0, %p7;
@@ -573,15 +573,15 @@ define <2 x i8> @to_f8e5m2_v2f32(<2 x float> %x) {
; CHECK-NEXT: and.b32 %r99, %r98, 128;
; CHECK-NEXT: or.b32 %r100, %r99, %r97;
; CHECK-NEXT: or.b32 %r101, %r100, %r96;
-; CHECK-NEXT: bfe.u32 %r102, %r72, 21, 1;
-; CHECK-NEXT: and.b32 %r103, %r71, 1048575;
-; CHECK-NEXT: setp.ne.b32 %p20, %r103, 0;
-; CHECK-NEXT: selp.b32 %r104, 1, 0, %p20;
-; CHECK-NEXT: or.b32 %r105, %r104, %r102;
-; CHECK-NEXT: shr.u32 %r106, %r71, 20;
-; CHECK-NEXT: and.b32 %r107, %r106, %r105;
-; CHECK-NEXT: bfe.u32 %r108, %r71, 21, 2;
-; CHECK-NEXT: add.s32 %r109, %r108, %r107;
+; CHECK-NEXT: shr.u32 %r102, %r72, 21;
+; CHECK-NEXT: and.b32 %r103, %r102, 1;
+; CHECK-NEXT: and.b32 %r104, %r71, 1048575;
+; CHECK-NEXT: setp.ne.b32 %p20, %r104, 0;
+; CHECK-NEXT: selp.b32 %r105, 1, 0, %p20;
+; CHECK-NEXT: or.b32 %r106, %r105, %r103;
+; CHECK-NEXT: shr.u32 %r107, %r71, 20;
+; CHECK-NEXT: and.b32 %r108, %r107, %r106;
+; CHECK-NEXT: add.s32 %r109, %r102, %r108;
; CHECK-NEXT: setp.gt.s32 %p21, %r109, 3;
; CHECK-NEXT: selp.b32 %r110, 0, %r109, %p21;
; CHECK-NEXT: selp.b32 %r111, 1, 0, %p21;
@@ -666,15 +666,15 @@ define <2 x i8> @to_f8e4m3fn_v2f32(<2 x float> %x) {
; CHECK-NEXT: and.b32 %r38, %r37, 128;
; CHECK-NEXT: or.b32 %r39, %r38, %r36;
; CHECK-NEXT: or.b32 %r40, %r39, %r35;
-; CHECK-NEXT: bfe.u32 %r41, %r10, 20, 1;
-; CHECK-NEXT: and.b32 %r42, %r9, 524287;
-; CHECK-NEXT: setp.ne.b32 %p6, %r42, 0;
-; CHECK-NEXT: selp.b32 %r43, 1, 0, %p6;
-; CHECK-NEXT: or.b32 %r44, %r43, %r41;
-; CHECK-NEXT: shr.u32 %r45, %r9, 19;
-; CHECK-NEXT: and.b32 %r46, %r45, %r44;
-; CHECK-NEXT: bfe.u32 %r47, %r9, 20, 3;
-; CHECK-NEXT: add.s32 %r48, %r47, %r46;
+; CHECK-NEXT: shr.u32 %r41, %r10, 20;
+; CHECK-NEXT: and.b32 %r42, %r41, 1;
+; CHECK-NEXT: and.b32 %r43, %r9, 524287;
+; CHECK-NEXT: setp.ne.b32 %p6, %r43, 0;
+; CHECK-NEXT: selp.b32 %r44, 1, 0, %p6;
+; CHECK-NEXT: or.b32 %r45, %r44, %r42;
+; CHECK-NEXT: shr.u32 %r46, %r9, 19;
+; CHECK-NEXT: and.b32 %r47, %r46, %r45;
+; CHECK-NEXT: add.s32 %r48, %r41, %r47;
; CHECK-NEXT: setp.gt.s32 %p7, %r48, 7;
; CHECK-NEXT: selp.b32 %r49, 0, %r48, %p7;
; CHECK-NEXT: selp.b32 %r50, 1, 0, %p7;
@@ -732,15 +732,15 @@ define <2 x i8> @to_f8e4m3fn_v2f32(<2 x float> %x) {
; CHECK-NEXT: and.b32 %r93, %r92, 128;
; CHECK-NEXT: or.b32 %r94, %r93, %r91;
; CHECK-NEXT: or.b32 %r95, %r94, %r90;
-; CHECK-NEXT: bfe.u32 %r96, %r66, 20, 1;
-; CHECK-NEXT: and.b32 %r97, %r65, 524287;
-; CHECK-NEXT: setp.ne.b32 %p16, %r97, 0;
-; CHECK-NEXT: selp.b32 %r98, 1, 0, %p16;
-; CHECK-NEXT: or.b32 %r99, %r98, %r96;
-; CHECK-NEXT: shr.u32 %r100, %r65, 19;
-; CHECK-NEXT: and.b32 %r101, %r100, %r99;
-; CHECK-NEXT: bfe.u32 %r102, %r65, 20, 3;
-; CHECK-NEXT: add.s32 %r103, %r102, %r101;
+; CHECK-NEXT: shr.u32 %r96, %r66, 20;
+; CHECK-NEXT: and.b32 %r97, %r96, 1;
+; CHECK-NEXT: and.b32 %r98, %r65, 524287;
+; CHECK-NEXT: setp.ne.b32 %p16, %r98, 0;
+; CHECK-NEXT: selp.b32 %r99, 1, 0, %p16;
+; CHECK-NEXT: or.b32 %r100, %r99, %r97;
+; CHECK-NEXT: shr.u32 %r101, %r65, 19;
+; CHECK-NEXT: and.b32 %r102, %r101, %r100;
+; CHECK-NEXT: add.s32 %r103, %r96, %r102;
; CHECK-NEXT: setp.gt.s32 %p17, %r103, 7;
; CHECK-NEXT: selp.b32 %r104, 0, %r103, %p17;
; CHECK-NEXT: selp.b32 %r105, 1, 0, %p17;
@@ -815,15 +815,15 @@ define <3 x i8> @to_f8e4m3fn_v3f32(<3 x float> %x) {
; CHECK-NEXT: and.b32 %r39, %r38, 128;
; CHECK-NEXT: or.b32 %r40, %r39, %r37;
; CHECK-NEXT: or.b32 %r41, %r40, %r36;
-; CHECK-NEXT: bfe.u32 %r42, %r11, 20, 1;
-; CHECK-NEXT: and.b32 %r43, %r10, 524287;
-; CHECK-NEXT: setp.ne.b32 %p6, %r43, 0;
-; CHECK-NEXT: selp.b32 %r44, 1, 0, %p6;
-; CHECK-NEXT: or.b32 %r45, %r44, %r42;
-; CHECK-NEXT: shr.u32 %r46, %r10, 19;
-; CHECK-NEXT: and.b32 %r47, %r46, %r45;
-; CHECK-NEXT: bfe.u32 %r48, %r10, 20, 3;
-; CHECK-NEXT: add.s32 %r49, %r48, %r47;
+; CHECK-NEXT: shr.u32 %r42, %r11, 20;
+; CHECK-NEXT: and.b32 %r43, %r42, 1;
+; CHECK-NEXT: and.b32 %r44, %r10, 524287;
+; CHECK-NEXT: setp.ne.b32 %p6, %r44, 0;
+; CHECK-NEXT: selp.b32 %r45, 1, 0, %p6;
+; CHECK-NEXT: or.b32 %r46, %r45, %r43;
+; CHECK-NEXT: shr.u32 %r47, %r10, 19;
+; CHECK-NEXT: and.b32 %r48, %r47, %r46;
+; CHECK-NEXT: add.s32 %r49, %r42, %r48;
; CHECK-NEXT: setp.gt.s32 %p7, %r49, 7;
; CHECK-NEXT: selp.b32 %r50, 0, %r49, %p7;
; CHECK-NEXT: selp.b32 %r51, 1, 0, %p7;
@@ -880,15 +880,15 @@ define <3 x i8> @to_f8e4m3fn_v3f32(<3 x float> %x) {
; CHECK-NEXT: and.b32 %r94, %r93, 128;
; CHECK-NEXT: or.b32 %r95, %r94, %r92;
; CHECK-NEXT: or.b32 %r96, %r95, %r91;
-; CHECK-NEXT: bfe.u32 %r97, %r67, 20, 1;
-; CHECK-NEXT: and.b32 %r98, %r66, 524287;
-; CHECK-NEXT: setp.ne.b32 %p16, %r98, 0;
-; CHECK-NEXT: selp.b32 %r99, 1, 0, %p16;
-; CHECK-NEXT: or.b32 %r100, %r99, %r97;
-; CHECK-NEXT: shr.u32 %r101, %r66, 19;
-; CHECK-NEXT: and.b32 %r102, %r101, %r100;
-; CHECK-NEXT: bfe.u32 %r103, %r66, 20, 3;
-; CHECK-NEXT: add.s32 %r104, %r103, %r102;
+; CHECK-NEXT: shr.u32 %r97, %r67, 20;
+; CHECK-NEXT: and.b32 %r98, %r97, 1;
+; CHECK-NEXT: and.b32 %r99, %r66, 524287;
+; CHECK-NEXT: setp.ne.b32 %p16, %r99, 0;
+; CHECK-NEXT: selp.b32 %r100, 1, 0, %p16;
+; CHECK-NEXT: or.b32 %r101, %r100, %r98;
+; CHECK-NEXT: shr.u32 %r102, %r66, 19;
+; CHECK-NEXT: and.b32 %r103, %r102, %r101;
+; CHECK-NEXT: add.s32 %r104, %r97, %r103;
; CHECK-NEXT: setp.gt.s32 %p17, %r104, 7;
; CHECK-NEXT: selp.b32 %r105, 0, %r104, %p17;
; CHECK-NEXT: selp.b32 %r106, 1, 0, %p17;
@@ -945,15 +945,15 @@ define <3 x i8> @to_f8e4m3fn_v3f32(<3 x float> %x) {
; CHECK-NEXT: and.b32 %r149, %r148, 128;
; CHECK-NEXT: or.b32 %r150, %r149, %r147;
; CHECK-NEXT: or.b32 %r151, %r150, %r146;
-; CHECK-NEXT: bfe.u32 %r152, %r122, 20, 1;
-; CHECK-NEXT: and.b32 %r153, %r121, 524287;
-; CHECK-NEXT: setp.ne.b32 %p26, %r153, 0;
-; CHECK-NEXT: selp.b32 %r154, 1, 0, %p26;
-; CHECK-NEXT: or.b32 %r155, %r154, %r152;
-; CHECK-NEXT: shr.u32 %r156, %r121, 19;
-; CHECK-NEXT: and.b32 %r157, %r156, %r155;
-; CHECK-NEXT: bfe.u32 %r158, %r121, 20, 3;
-; CHECK-NEXT: add.s32 %r159, %r158, %r157;
+; CHECK-NEXT: shr.u32 %r152, %r122, 20;
+; CHECK-NEXT: and.b32 %r153, %r152, 1;
+; CHECK-NEXT: and.b32 %r154, %r121, 524287;
+; CHECK-NEXT: setp.ne.b32 %p26, %r154, 0;
+; CHECK-NEXT: selp.b32 %r155, 1, 0, %p26;
+; CHECK-NEXT: or.b32 %r156, %r155, %r153;
+; CHECK-NEXT: shr.u32 %r157, %r121, 19;
+; CHECK-NEXT: and.b32 %r158, %r157, %r156;
+; CHECK-NEXT: add.s32 %r159, %r152, %r158;
; CHECK-NEXT: setp.gt.s32 %p27, %r159, 7;
; CHECK-NEXT: selp.b32 %r160, 0, %r159, %p27;
; CHECK-NEXT: selp.b32 %r161, 1, 0, %p27;
@@ -1029,15 +1029,15 @@ define <4 x i8> @to_f8e4m3fn_v4f32(<4 x float> %x) {
; CHECK-NEXT: and.b32 %r40, %r39, 128;
; CHECK-NEXT: or.b32 %r41, %r40, %r38;
; CHECK-NEXT: or.b32 %r42, %r41, %r37;
-; CHECK-NEXT: bfe.u32 %r43, %r12, 20, 1;
-; CHECK-NEXT: and.b32 %r44, %r11, 524287;
-; CHECK-NEXT: setp.ne.b32 %p6, %r44, 0;
-; CHECK-NEXT: selp.b32 %r45, 1, 0, %p6;
-; CHECK-NEXT: or.b32 %r46, %r45, %r43;
-; CHECK-NEXT: shr.u32 %r47, %r11, 19;
-; CHECK-NEXT: and.b32 %r48, %r47, %r46;
-; CHECK-NEXT: bfe.u32 %r49, %r11, 20, 3;
-; CHECK-NEXT: add.s32 %r50, %r49, %r48;
+; CHECK-NEXT: shr.u32 %r43, %r12, 20;
+; CHECK-NEXT: and.b32 %r44, %r43, 1;
+; CHECK-NEXT: and.b32 %r45, %r11, 524287;
+; CHECK-NEXT: setp.ne.b32 %p6, %r45, 0;
+; CHECK-NEXT: selp.b32 %r46, 1, 0, %p6;
+; CHECK-NEXT: or.b32 %r47, %r46, %r44;
+; CHECK-NEXT: shr.u32 %r48, %r11, 19;
+; CHECK-NEXT: and.b32 %r49, %r48, %r47;
+; CHECK-NEXT: add.s32 %r50, %r43, %r49;
; CHECK-NEXT: setp.gt.s32 %p7, %r50, 7;
; CHECK-NEXT: selp.b32 %r51, 0, %r50, %p7;
; CHECK-NEXT: selp.b32 %r52, 1, 0, %p7;
@@ -1094,15 +1094,15 @@ define <4 x i8> @to_f8e4m3fn_v4f32(<4 x float> %x) {
; CHECK-NEXT: and.b32 %r95, %r94, 128;
; CHECK-NEXT: or.b32 %r96, %r95, %r93;
; CHECK-NEXT: or.b32 %r97, %r96, %r92;
-; CHECK-NEXT: bfe.u32 %r98, %r68, 20, 1;
-; CHECK-NEXT: and.b32 %r99, %r67, 524287;
-; CHECK-NEXT: setp.ne.b32 %p16, %r99, 0;
-; CHECK-NEXT: selp.b32 %r100, 1, 0, %p16;
-; CHECK-NEXT: or.b32 %r101, %r100, %r98;
-; CHECK-NEXT: shr.u32 %r102, %r67, 19;
-; CHECK-NEXT: and.b32 %r103, %r102, %r101;
-; CHECK-NEXT: bfe.u32 %r104, %r67, 20, 3;
-; CHECK-NEXT: add.s32 %r105, %r104, %r103;
+; CHECK-NEXT: shr.u32 %r98, %r68, 20;
+; CHECK-NEXT: and.b32 %r99, %r98, 1;
+; CHECK-NEXT: and.b32 %r100, %r67, 524287;
+; CHECK-NEXT: setp.ne.b32 %p16, %r100, 0;
+; CHECK-NEXT: selp.b32 %r101, 1, 0, %p16;
+; CHECK-NEXT: or.b32 %r102, %r101, %r99;
+; CHECK-NEXT: shr.u32 %r103, %r67, 19;
+; CHECK-NEXT: and.b32 %r104, %r103, %r102;
+; CHECK-NEXT: add.s32 %r105, %r98, %r104;
; CHECK-NEXT: setp.gt.s32 %p17, %r105, 7;
; CHECK-NEXT: selp.b32 %r106, 0, %r105, %p17;
; CHECK-NEXT: selp.b32 %r107, 1, 0, %p17;
@@ -1160,15 +1160,15 @@ define <4 x i8> @to_f8e4m3fn_v4f32(<4 x float> %x) {
; CHECK-NEXT: and.b32 %r151, %r150, 128;
; CHECK-NEXT: or.b32 %r152, %r151, %r149;
; CHECK-NEXT: or.b32 %r153, %r152, %r148;
-; CHECK-NEXT: bfe.u32 %r154, %r124, 20, 1;
-; CHECK-NEXT: and.b32 %r155, %r123, 524287;
-; CHECK-NEXT: setp.ne.b32 %p26, %r155, 0;
-; CHECK-NEXT: selp.b32 %r156, 1, 0, %p26;
-; CHECK-NEXT: or.b32 %r157, %r156, %r154;
-; CHECK-NEXT: shr.u32 %r158, %r123, 19;
-; CHECK-NEXT: and.b32 %r159, %r158, %r157;
-; CHECK-NEXT: bfe.u32 %r160, %r123, 20, 3;
-; CHECK-NEXT: add.s32 %r161, %r160, %r159;
+; CHECK-NEXT: shr.u32 %r154, %r124, 20;
+; CHECK-NEXT: and.b32 %r155, %r154, 1;
+; CHECK-NEXT: and.b32 %r156, %r123, 524287;
+; CHECK-NEXT: setp.ne.b32 %p26, %r156, 0;
+; CHECK-NEXT: selp.b32 %r157, 1, 0, %p26;
+; CHECK-NEXT: or.b32 %r158, %r157, %r155;
+; CHECK-NEXT: shr.u32 %r159, %r123, 19;
+; CHECK-NEXT: and.b32 %r160, %r159, %r158;
+; CHECK-NEXT: add.s32 %r161, %r154, %r160;
; CHECK-NEXT: setp.gt.s32 %p27, %r161, 7;
; CHECK-NEXT: selp.b32 %r162, 0, %r161, %p27;
; CHECK-NEXT: selp.b32 %r163, 1, 0, %p27;
@@ -1225,15 +1225,15 @@ define <4 x i8> @to_f8e4m3fn_v4f32(<4 x float> %x) {
; CHECK-NEXT: and.b32 %r206, %r205, 128;
; CHECK-NEXT: or.b32 %r207, %r206, %r204;
; CHECK-NEXT: or.b32 %r208, %r207, %r203;
-; CHECK-NEXT: bfe.u32 %r209, %r179, 20, 1;
-; CHECK-NEXT: and.b32 %r210, %r178, 524287;
-; CHECK-NEXT: setp.ne.b32 %p36, %r210, 0;
-; CHECK-NEXT: selp.b32 %r211, 1, 0, %p36;
-; CHECK-NEXT: or.b32 %r212, %r211, %r209;
-; CHECK-NEXT: shr.u32 %r213, %r178, 19;
-; CHECK-NEXT: and.b32 %r214, %r213, %r212;
-; CHECK-NEXT: bfe.u32 %r215, %r178, 20, 3;
-; CHECK-NEXT: add.s32 %r216, %r215, %r214;
+; CHECK-NEXT: shr.u32 %r209, %r179, 20;
+; CHECK-NEXT: and.b32 %r210, %r209, 1;
+; CHECK-NEXT: and.b32 %r211, %r178, 524287;
+; CHECK-NEXT: setp.ne.b32 %p36, %r211, 0;
+; CHECK-NEXT: selp.b32 %r212, 1, 0, %p36;
+; CHECK-NEXT: or.b32 %r213, %r212, %r210;
+; CHECK-NEXT: shr.u32 %r214, %r178, 19;
+; CHECK-NEXT: and.b32 %r215, %r214, %r213;
+; CHECK-NEXT: add.s32 %r216, %r209, %r215;
; CHECK-NEXT: setp.gt.s32 %p37, %r216, 7;
; CHECK-NEXT: selp.b32 %r217, 0, %r216, %p37;
; CHECK-NEXT: selp.b32 %r218, 1, 0, %p37;
@@ -1308,15 +1308,15 @@ define <4 x i8> @to_f8e5m2_v4f32(<4 x float> %x) {
; CHECK-NEXT: and.b32 %r40, %r39, 128;
; CHECK-NEXT: or.b32 %r41, %r40, %r38;
; CHECK-NEXT: or.b32 %r42, %r41, %r37;
-; CHECK-NEXT: bfe.u32 %r43, %r12, 21, 1;
-; CHECK-NEXT: and.b32 %r44, %r11, 1048575;
-; CHECK-NEXT: setp.ne.b32 %p6, %r44, 0;
-; CHECK-NEXT: selp.b32 %r45, 1, 0, %p6;
-; CHECK-NEXT: or.b32 %r46, %r45, %r43;
-; CHECK-NEXT: shr.u32 %r47, %r11, 20;
-; CHECK-NEXT: and.b32 %r48, %r47, %r46;
-; CHECK-NEXT: bfe.u32 %r49, %r11, 21, 2;
-; CHECK-NEXT: add.s32 %r50, %r49, %r48;
+; CHECK-NEXT: shr.u32 %r43, %r12, 21;
+; CHECK-NEXT: and.b32 %r44, %r43, 1;
+; CHECK-NEXT: and.b32 %r45, %r11, 1048575;
+; CHECK-NEXT: setp.ne.b32 %p6, %r45, 0;
+; CHECK-NEXT: selp.b32 %r46, 1, 0, %p6;
+; CHECK-NEXT: or.b32 %r47, %r46, %r44;
+; CHECK-NEXT: shr.u32 %r48, %r11, 20;
+; CHECK-NEXT: and.b32 %r49, %r48, %r47;
+; CHECK-NEXT: add.s32 %r50, %r43, %r49;
; CHECK-NEXT: setp.gt.s32 %p7, %r50, 3;
; CHECK-NEXT: selp.b32 %r51, 0, %r50, %p7;
; CHECK-NEXT: selp.b32 %r52, 1, 0, %p7;
@@ -1383,15 +1383,15 @@ define <4 x i8> @to_f8e5m2_v4f32(<4 x float> %x) {
; CHECK-NEXT: and.b32 %r101, %r100, 128;
; CHECK-NEXT: or.b32 %r102, %r101, %r99;
; CHECK-NEXT: or.b32 %r103, %r102, %r98;
-; CHECK-NEXT: bfe.u32 %r104, %r74, 21, 1;
-; CHECK-NEXT: and.b32 %r105, %r73, 1048575;
-; CHECK-NEXT: setp.ne.b32 %p20, %r105, 0;
-; CHECK-NEXT: selp.b32 %r106, 1, 0, %p20;
-; CHECK-NEXT: or.b32 %r107, %r106, %r104;
-; CHECK-NEXT: shr.u32 %r108, %r73, 20;
-; CHECK-NEXT: and.b32 %r109, %r108, %r107;
-; CHECK-NEXT: bfe.u32 %r110, %r73, 21, 2;
-; CHECK-NEXT: add.s32 %r111, %r110, %r109;
+; CHECK-NEXT: shr.u32 %r104, %r74, 21;
+; CHECK-NEXT: and.b32 %r105, %r104, 1;
+; CHECK-NEXT: and.b32 %r106, %r73, 1048575;
+; CHECK-NEXT: setp.ne.b32 %p20, %r106, 0;
+; CHECK-NEXT: selp.b32 %r107, 1, 0, %p20;
+; CHECK-NEXT: or.b32 %r108, %r107, %r105;
+; CHECK-NEXT: shr.u32 %r109, %r73, 20;
+; CHECK-NEXT: and.b32 %r110, %r109, %r108;
+; CHECK-NEXT: add.s32 %r111, %r104, %r110;
; CHECK-NEXT: setp.gt.s32 %p21, %r111, 3;
; CHECK-NEXT: selp.b32 %r112, 0, %r111, %p21;
; CHECK-NEXT: selp.b32 %r113, 1, 0, %p21;
@@ -1459,15 +1459,15 @@ define <4 x i8> @to_f8e5m2_v4f32(<4 x float> %x) {
; CHECK-NEXT: and.b32 %r163, %r162, 128;
; CHECK-NEXT: or.b32 %r164, %r163, %r161;
; CHECK-NEXT: or.b32 %r165, %r164, %r160;
-; CHECK-NEXT: bfe.u32 %r166, %r136, 21, 1;
-; CHECK-NEXT: and.b32 %r167, %r135, 1048575;
-; CHECK-NEXT: setp.ne.b32 %p34, %r167, 0;
-; CHECK-NEXT: selp.b32 %r168, 1, 0, %p34;
-; CHECK-NEXT: or.b32 %r169, %r168, %r166;
-; CHECK-NEXT: shr.u32 %r170, %r135, 20;
-; CHECK-NEXT: and.b32 %r171, %r170, %r169;
-; CHECK-NEXT: bfe.u32 %r172, %r135, 21, 2;
-; CHECK-NEXT: add.s32 %r173, %r172, %r171;
+; CHECK-NEXT: shr.u32 %r166, %r136, 21;
+; CHECK-NEXT: and.b32 %r167, %r166, 1;
+; CHECK-NEXT: and.b32 %r168, %r135, 1048575;
+; CHECK-NEXT: setp.ne.b32 %p34, %r168, 0;
+; CHECK-NEXT: selp.b32 %r169, 1, 0, %p34;
+; CHECK-NEXT: or.b32 %r170, %r169, %r167;
+; CHECK-NEXT: shr.u32 %r171, %r135, 20;
+; CHECK-NEXT: and.b32 %r172, %r171, %r170;
+; CHECK-NEXT: add.s32 %r173, %r166, %r172;
; CHECK-NEXT: setp.gt.s32 %p35, %r173, 3;
; CHECK-NEXT: selp.b32 %r174, 0, %r173, %p35;
; CHECK-NEXT: selp.b32 %r175, 1, 0, %p35;
@@ -1534,15 +1534,15 @@ define <4 x i8> @to_f8e5m2_v4f32(<4 x float> %x) {
; CHECK-NEXT: and.b32 %r224, %r223, 128;
; CHECK-NEXT: or.b32 %r225, %r224, %r222;
; CHECK-NEXT: or.b32 %r226, %r225, %r221;
-; CHECK-NEXT: bfe.u32 %r227, %r197, 21, 1;
-; CHECK-NEXT: and.b32 %r228, %r196, 1048575;
-; CHECK-NEXT: setp.ne.b32 %p48, %r228, 0;
-; CHECK-NEXT: selp.b32 %r229, 1, 0, %p48;
-; CHECK-NEXT: or.b32 %r230, %r229, %r227;
-; CHECK-NEXT: shr.u32 %r231, %r196, 20;
-; CHECK-NEXT: and.b32 %r232, %r231, %r230;
-; CHECK-NEXT: bfe.u32 %r233, %r196, 21, 2;
-; CHECK-NEXT: add.s32 %r234, %r233, %r232;
+; CHECK-NEXT: shr.u32 %r227, %r197, 21;
+; CHECK-NEXT: and.b32 %r228, %r227, 1;
+; CHECK-NEXT: and.b32 %r229, %r196, 1048575;
+; CHECK-NEXT: setp.ne.b32 %p48, %r229, 0;
+; CHECK-NEXT: selp.b32 %r230, 1, 0, %p48;
+; CHECK-NEXT: or.b32 %r231, %r230, %r228;
+; CHECK-NEXT: shr.u32 %r232, %r196, 20;
+; CHECK-NEXT: and.b32 %r233, %r232, %r231;
+; CHECK-NEXT: add.s32 %r234, %r227, %r233;
; CHECK-NEXT: setp.gt.s32 %p49, %r234, 3;
; CHECK-NEXT: selp.b32 %r235, 0, %r234, %p49;
; CHECK-NEXT: selp.b32 %r236, 1, 0, %p49;
@@ -1848,20 +1848,21 @@ define i8 @to_f8e5m2_from_bf16(bfloat %x) {
; CHECK: {
; CHECK-NEXT: .reg .pred %p<17>;
; CHECK-NEXT: .reg .b16 %rs<60>;
-; CHECK-NEXT: .reg .b32 %r<24>;
+; CHECK-NEXT: .reg .b32 %r<26>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
; CHECK-NEXT: ld.param.b16 %rs1, [to_f8e5m2_from_bf16_param_0];
; CHECK-NEXT: cvt.u32.u16 %r1, %rs1;
; CHECK-NEXT: shl.b32 %r2, %r1, 16;
; CHECK-NEXT: mul.rn.f32 %r3, %r2, 0f44000000;
-; CHECK-NEXT: bfe.u32 %r4, %r3, 16, 1;
-; CHECK-NEXT: add.s32 %r5, %r4, %r3;
-; CHECK-NEXT: add.s32 %r6, %r5, 32767;
+; CHECK-NEXT: shr.u32 %r4, %r3, 16;
+; CHECK-NEXT: and.b32 %r5, %r4, 1;
+; CHECK-NEXT: add.s32 %r6, %r5, %r3;
+; CHECK-NEXT: add.s32 %r7, %r6, 32767;
; CHECK-NEXT: setp.nan.f32 %p1, %r3, %r3;
-; CHECK-NEXT: or.b32 %r7, %r3, 4194304;
-; CHECK-NEXT: selp.b32 %r8, %r7, %r6, %p1;
-; CHECK-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs2}, %r8; }
+; CHECK-NEXT: or.b32 %r8, %r3, 4194304;
+; CHECK-NEXT: selp.b32 %r9, %r8, %r7, %p1;
+; CHECK-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs2}, %r9; }
; CHECK-NEXT: and.b16 %rs3, %rs1, 32767;
; CHECK-NEXT: setp.lt.u16 %p2, %rs3, 128;
; CHECK-NEXT: selp.b16 %rs4, %rs2, %rs1, %p2;
@@ -1875,28 +1876,28 @@ define i8 @to_f8e5m2_from_bf16(bfloat %x) {
; CHECK-NEXT: and.b16 %rs11, %rs2, 32640;
; CHECK-NEXT: selp.b16 %rs12, %rs11, %rs3, %p2;
; CHECK-NEXT: shr.u16 %rs13, %rs12, 7;
-; CHECK-NEXT: cvt.u32.u16 %r9, %rs13;
-; CHECK-NEXT: selp.b32 %r10, -9, 0, %p2;
-; CHECK-NEXT: add.s32 %r11, %r9, %r10;
-; CHECK-NEXT: add.s32 %r12, %r11, -126;
-; CHECK-NEXT: selp.b32 %r13, 0, %r12, %p3;
-; CHECK-NEXT: cvt.u16.u32 %rs14, %r13;
+; CHECK-NEXT: cvt.u32.u16 %r10, %rs13;
+; CHECK-NEXT: selp.b32 %r11, -9, 0, %p2;
+; CHECK-NEXT: add.s32 %r12, %r10, %r11;
+; CHECK-NEXT: add.s32 %r13, %r12, -126;
+; CHECK-NEXT: selp.b32 %r14, 0, %r13, %p3;
+; CHECK-NEXT: cvt.u16.u32 %rs14, %r14;
; CHECK-NEXT: sub.s16 %rs15, -8, %rs14;
; CHECK-NEXT: min.u16 %rs16, %rs15, 15;
-; CHECK-NEXT: cvt.u32.u16 %r14, %rs16;
-; CHECK-NEXT: shr.u16 %rs17, %rs10, %r14;
+; CHECK-NEXT: cvt.u32.u16 %r15, %rs16;
+; CHECK-NEXT: shr.u16 %rs17, %rs10, %r15;
; CHECK-NEXT: and.b16 %rs18, %rs17, 1;
; CHECK-NEXT: max.u16 %rs19, %rs16, 1;
; CHECK-NEXT: add.s16 %rs20, %rs19, -1;
-; CHECK-NEXT: cvt.u32.u16 %r15, %rs20;
+; CHECK-NEXT: cvt.u32.u16 %r16, %rs20;
; CHECK-NEXT: mov.b16 %rs21, 1;
-; CHECK-NEXT: shl.b16 %rs22, %rs21, %r15;
+; CHECK-NEXT: shl.b16 %rs22, %rs21, %r16;
; CHECK-NEXT: add.s16 %rs23, %rs22, -1;
; CHECK-NEXT: and.b16 %rs24, %rs10, %rs23;
; CHECK-NEXT: setp.ne.b16 %p4, %rs24, 0;
; CHECK-NEXT: selp.b16 %rs25, 1, 0, %p4;
; CHECK-NEXT: or.b16 %rs26, %rs25, %rs18;
-; CHECK-NEXT: shr.u16 %rs27, %rs10, %r15;
+; CHECK-NEXT: shr.u16 %rs27, %rs10, %r16;
; CHECK-NEXT: and.b16 %rs28, %rs27, %rs26;
; CHECK-NEXT: setp.ne.b16 %p5, %rs16, 0;
; CHECK-NEXT: selp.b16 %rs29, %rs28, 0, %p5;
@@ -1936,20 +1937,21 @@ define i8 @to_f8e5m2_from_bf16(bfloat %x) {
; CHECK-NEXT: selp.b16 %rs56, %rs53, %rs55, %p12;
; CHECK-NEXT: setp.eq.f32 %p13, %r2, 0f00000000;
; CHECK-NEXT: selp.b16 %rs57, %rs34, %rs56, %p13;
-; CHECK-NEXT: abs.f32 %r16, %r2;
-; CHECK-NEXT: bfe.u32 %r17, %r16, 16, 1;
-; CHECK-NEXT: or.b32 %r18, %r17, %r16;
-; CHECK-NEXT: add.s32 %r19, %r18, 32767;
-; CHECK-NEXT: setp.nan.f32 %p14, %r16, %r16;
-; CHECK-NEXT: or.b32 %r20, %r16, 4194304;
-; CHECK-NEXT: selp.b32 %r21, %r20, %r19, %p14;
-; CHECK-NEXT: and.b32 %r22, %r21, 2147418112;
-; CHECK-NEXT: setp.eq.f32 %p15, %r22, 0f7F800000;
+; CHECK-NEXT: abs.f32 %r17, %r2;
+; CHECK-NEXT: shr.u32 %r18, %r17, 16;
+; CHECK-NEXT: and.b32 %r19, %r18, 1;
+; CHECK-NEXT: or.b32 %r20, %r19, %r17;
+; CHECK-NEXT: add.s32 %r21, %r20, 32767;
+; CHECK-NEXT: setp.nan.f32 %p14, %r17, %r17;
+; CHECK-NEXT: or.b32 %r22, %r17, 4194304;
+; CHECK-NEXT: selp.b32 %r23, %r22, %r21, %p14;
+; CHECK-NEXT: and.b32 %r24, %r23, 2147418112;
+; CHECK-NEXT: setp.eq.f32 %p15, %r24, 0f7F800000;
; CHECK-NEXT: selp.b16 %rs58, %rs53, %rs57, %p15;
; CHECK-NEXT: setp.nan.f32 %p16, %r2, %r2;
; CHECK-NEXT: selp.b16 %rs59, 126, %rs58, %p16;
-; CHECK-NEXT: cvt.u32.u16 %r23, %rs59;
-; CHECK-NEXT: st.param.b32 [func_retval0], %r23;
+; CHECK-NEXT: cvt.u32.u16 %r25, %rs59;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r25;
; CHECK-NEXT: ret;
%r = call i8 @llvm.convert.to.arbitrary.fp.i8.bf16(bfloat %x, metadata !"Float8E5M2", metadata !"round.tonearest", i1 false)
ret i8 %r
@@ -2012,7 +2014,7 @@ define i8 @to_f8e5m2_from_f64(double %x) {
; CHECK-NEXT: and.b64 %rd34, %rd33, 128;
; CHECK-NEXT: or.b64 %rd35, %rd34, %rd32;
; CHECK-NEXT: or.b64 %rd36, %rd35, %rd31;
-; CHECK-NEXT: bfe.u64 %rd37, %rd8, 50, 2;
+; CHECK-NEXT: shr.u64 %rd37, %rd9, 50;
; CHECK-NEXT: and.b64 %rd38, %rd37, 1;
; CHECK-NEXT: and.b64 %rd39, %rd8, 562949953421311;
; CHECK-NEXT: setp.ne.b64 %p6, %rd39, 0;
diff --git a/llvm/test/CodeGen/NVPTX/fma-relu-contract.ll b/llvm/test/CodeGen/NVPTX/fma-relu-contract.ll
index 99ce704d9da6c..93fe01a7d7f88 100644
--- a/llvm/test/CodeGen/NVPTX/fma-relu-contract.ll
+++ b/llvm/test/CodeGen/NVPTX/fma-relu-contract.ll
@@ -261,7 +261,7 @@ define bfloat @fma_bf16_expanded_unsafe_with_nans(bfloat %a, bfloat %b, bfloat %
; CHECK-SM70: {
; CHECK-SM70-NEXT: .reg .pred %p<3>;
; CHECK-SM70-NEXT: .reg .b16 %rs<3>;
-; CHECK-SM70-NEXT: .reg .b32 %r<14>;
+; CHECK-SM70-NEXT: .reg .b32 %r<15>;
; CHECK-SM70-EMPTY:
; CHECK-SM70-NEXT: // %bb.0:
; CHECK-SM70-NEXT: ld.param.b16 %r1, [fma_bf16_expanded_unsafe_with_nans_param_2];
@@ -271,15 +271,16 @@ define bfloat @fma_bf16_expanded_unsafe_with_nans(bfloat %a, bfloat %b, bfloat %
; CHECK-SM70-NEXT: ld.param.b16 %r5, [fma_bf16_expanded_unsafe_with_nans_param_0];
; CHECK-SM70-NEXT: shl.b32 %r6, %r5, 16;
; CHECK-SM70-NEXT: fma.rn.f32 %r7, %r6, %r4, %r2;
-; CHECK-SM70-NEXT: bfe.u32 %r8, %r7, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r9, %r8, %r7;
-; CHECK-SM70-NEXT: add.s32 %r10, %r9, 32767;
+; CHECK-SM70-NEXT: shr.u32 %r8, %r7, 16;
+; CHECK-SM70-NEXT: and.b32 %r9, %r8, 1;
+; CHECK-SM70-NEXT: add.s32 %r10, %r9, %r7;
+; CHECK-SM70-NEXT: add.s32 %r11, %r10, 32767;
; CHECK-SM70-NEXT: setp.nan.f32 %p1, %r7, %r7;
-; CHECK-SM70-NEXT: or.b32 %r11, %r7, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r12, %r11, %r10, %p1;
-; CHECK-SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs1}, %r12; }
-; CHECK-SM70-NEXT: and.b32 %r13, %r12, -65536;
-; CHECK-SM70-NEXT: setp.gt.f32 %p2, %r13, 0f00000000;
+; CHECK-SM70-NEXT: or.b32 %r12, %r7, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r13, %r12, %r11, %p1;
+; CHECK-SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs1}, %r13; }
+; CHECK-SM70-NEXT: and.b32 %r14, %r13, -65536;
+; CHECK-SM70-NEXT: setp.gt.f32 %p2, %r14, 0f00000000;
; CHECK-SM70-NEXT: selp.b16 %rs2, %rs1, 0x0000, %p2;
; CHECK-SM70-NEXT: st.param.b16 [func_retval0], %rs2;
; CHECK-SM70-NEXT: ret;
@@ -319,7 +320,7 @@ define bfloat @fma_bf16_expanded_no_nans(bfloat %a, bfloat %b, bfloat %c) {
; CHECK-SM70: {
; CHECK-SM70-NEXT: .reg .pred %p<3>;
; CHECK-SM70-NEXT: .reg .b16 %rs<3>;
-; CHECK-SM70-NEXT: .reg .b32 %r<14>;
+; CHECK-SM70-NEXT: .reg .b32 %r<15>;
; CHECK-SM70-EMPTY:
; CHECK-SM70-NEXT: // %bb.0:
; CHECK-SM70-NEXT: ld.param.b16 %r1, [fma_bf16_expanded_no_nans_param_2];
@@ -329,15 +330,16 @@ define bfloat @fma_bf16_expanded_no_nans(bfloat %a, bfloat %b, bfloat %c) {
; CHECK-SM70-NEXT: ld.param.b16 %r5, [fma_bf16_expanded_no_nans_param_0];
; CHECK-SM70-NEXT: shl.b32 %r6, %r5, 16;
; CHECK-SM70-NEXT: fma.rn.f32 %r7, %r6, %r4, %r2;
-; CHECK-SM70-NEXT: bfe.u32 %r8, %r7, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r9, %r8, %r7;
-; CHECK-SM70-NEXT: add.s32 %r10, %r9, 32767;
+; CHECK-SM70-NEXT: shr.u32 %r8, %r7, 16;
+; CHECK-SM70-NEXT: and.b32 %r9, %r8, 1;
+; CHECK-SM70-NEXT: add.s32 %r10, %r9, %r7;
+; CHECK-SM70-NEXT: add.s32 %r11, %r10, 32767;
; CHECK-SM70-NEXT: setp.nan.f32 %p1, %r7, %r7;
-; CHECK-SM70-NEXT: or.b32 %r11, %r7, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r12, %r11, %r10, %p1;
-; CHECK-SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs1}, %r12; }
-; CHECK-SM70-NEXT: and.b32 %r13, %r12, -65536;
-; CHECK-SM70-NEXT: setp.gt.f32 %p2, %r13, 0f00000000;
+; CHECK-SM70-NEXT: or.b32 %r12, %r7, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r13, %r12, %r11, %p1;
+; CHECK-SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs1}, %r13; }
+; CHECK-SM70-NEXT: and.b32 %r14, %r13, -65536;
+; CHECK-SM70-NEXT: setp.gt.f32 %p2, %r14, 0f00000000;
; CHECK-SM70-NEXT: selp.b16 %rs2, %rs1, 0x0000, %p2;
; CHECK-SM70-NEXT: st.param.b16 [func_retval0], %rs2;
; CHECK-SM70-NEXT: ret;
@@ -397,7 +399,7 @@ define bfloat @fma_bf16_expanded_no_nans_multiple_uses_of_fma(bfloat %a, bfloat
; CHECK-SM70: {
; CHECK-SM70-NEXT: .reg .pred %p<5>;
; CHECK-SM70-NEXT: .reg .b16 %rs<3>;
-; CHECK-SM70-NEXT: .reg .b32 %r<30>;
+; CHECK-SM70-NEXT: .reg .b32 %r<33>;
; CHECK-SM70-EMPTY:
; CHECK-SM70-NEXT: // %bb.0:
; CHECK-SM70-NEXT: ld.param.b16 %r1, [fma_bf16_expanded_no_nans_multiple_uses_of_fma_param_2];
@@ -407,35 +409,38 @@ define bfloat @fma_bf16_expanded_no_nans_multiple_uses_of_fma(bfloat %a, bfloat
; CHECK-SM70-NEXT: ld.param.b16 %r5, [fma_bf16_expanded_no_nans_multiple_uses_of_fma_param_0];
; CHECK-SM70-NEXT: shl.b32 %r6, %r5, 16;
; CHECK-SM70-NEXT: fma.rn.f32 %r7, %r6, %r4, %r2;
-; CHECK-SM70-NEXT: bfe.u32 %r8, %r7, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r9, %r8, %r7;
-; CHECK-SM70-NEXT: add.s32 %r10, %r9, 32767;
+; CHECK-SM70-NEXT: shr.u32 %r8, %r7, 16;
+; CHECK-SM70-NEXT: and.b32 %r9, %r8, 1;
+; CHECK-SM70-NEXT: add.s32 %r10, %r9, %r7;
+; CHECK-SM70-NEXT: add.s32 %r11, %r10, 32767;
; CHECK-SM70-NEXT: setp.nan.f32 %p1, %r7, %r7;
-; CHECK-SM70-NEXT: or.b32 %r11, %r7, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r12, %r11, %r10, %p1;
-; CHECK-SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs1}, %r12; }
-; CHECK-SM70-NEXT: and.b32 %r13, %r12, -65536;
-; CHECK-SM70-NEXT: setp.gt.f32 %p2, %r13, 0f00000000;
+; CHECK-SM70-NEXT: or.b32 %r12, %r7, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r13, %r12, %r11, %p1;
+; CHECK-SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs1}, %r13; }
+; CHECK-SM70-NEXT: and.b32 %r14, %r13, -65536;
+; CHECK-SM70-NEXT: setp.gt.f32 %p2, %r14, 0f00000000;
; CHECK-SM70-NEXT: selp.b16 %rs2, %rs1, 0x0000, %p2;
-; CHECK-SM70-NEXT: add.f32 %r14, %r13, 0f40E00000;
-; CHECK-SM70-NEXT: bfe.u32 %r15, %r14, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r16, %r15, %r14;
-; CHECK-SM70-NEXT: add.s32 %r17, %r16, 32767;
-; CHECK-SM70-NEXT: setp.nan.f32 %p3, %r14, %r14;
-; CHECK-SM70-NEXT: or.b32 %r18, %r14, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r19, %r18, %r17, %p3;
-; CHECK-SM70-NEXT: cvt.u32.u16 %r20, %rs2;
-; CHECK-SM70-NEXT: shl.b32 %r21, %r20, 16;
-; CHECK-SM70-NEXT: and.b32 %r22, %r19, -65536;
-; CHECK-SM70-NEXT: add.f32 %r23, %r21, %r22;
-; CHECK-SM70-NEXT: bfe.u32 %r24, %r23, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r25, %r24, %r23;
-; CHECK-SM70-NEXT: add.s32 %r26, %r25, 32767;
-; CHECK-SM70-NEXT: setp.nan.f32 %p4, %r23, %r23;
-; CHECK-SM70-NEXT: or.b32 %r27, %r23, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r28, %r27, %r26, %p4;
-; CHECK-SM70-NEXT: shr.u32 %r29, %r28, 16;
-; CHECK-SM70-NEXT: st.param.b16 [func_retval0], %r29;
+; CHECK-SM70-NEXT: add.f32 %r15, %r14, 0f40E00000;
+; CHECK-SM70-NEXT: shr.u32 %r16, %r15, 16;
+; CHECK-SM70-NEXT: and.b32 %r17, %r16, 1;
+; CHECK-SM70-NEXT: add.s32 %r18, %r17, %r15;
+; CHECK-SM70-NEXT: add.s32 %r19, %r18, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p3, %r15, %r15;
+; CHECK-SM70-NEXT: or.b32 %r20, %r15, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r21, %r20, %r19, %p3;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r22, %rs2;
+; CHECK-SM70-NEXT: shl.b32 %r23, %r22, 16;
+; CHECK-SM70-NEXT: and.b32 %r24, %r21, -65536;
+; CHECK-SM70-NEXT: add.f32 %r25, %r23, %r24;
+; CHECK-SM70-NEXT: shr.u32 %r26, %r25, 16;
+; CHECK-SM70-NEXT: and.b32 %r27, %r26, 1;
+; CHECK-SM70-NEXT: add.s32 %r28, %r27, %r25;
+; CHECK-SM70-NEXT: add.s32 %r29, %r28, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p4, %r25, %r25;
+; CHECK-SM70-NEXT: or.b32 %r30, %r25, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r31, %r30, %r29, %p4;
+; CHECK-SM70-NEXT: shr.u32 %r32, %r31, 16;
+; CHECK-SM70-NEXT: st.param.b16 [func_retval0], %r32;
; CHECK-SM70-NEXT: ret;
%1 = fmul contract nnan bfloat %a, %b
%2 = fadd contract nnan bfloat %1, %c
@@ -474,7 +479,7 @@ define bfloat @fma_bf16_expanded_maxnum_no_nans(bfloat %a, bfloat %b, bfloat %c)
; CHECK-SM70-LABEL: fma_bf16_expanded_maxnum_no_nans(
; CHECK-SM70: {
; CHECK-SM70-NEXT: .reg .pred %p<3>;
-; CHECK-SM70-NEXT: .reg .b32 %r<21>;
+; CHECK-SM70-NEXT: .reg .b32 %r<23>;
; CHECK-SM70-EMPTY:
; CHECK-SM70-NEXT: // %bb.0:
; CHECK-SM70-NEXT: ld.param.b16 %r1, [fma_bf16_expanded_maxnum_no_nans_param_2];
@@ -484,22 +489,24 @@ define bfloat @fma_bf16_expanded_maxnum_no_nans(bfloat %a, bfloat %b, bfloat %c)
; CHECK-SM70-NEXT: ld.param.b16 %r5, [fma_bf16_expanded_maxnum_no_nans_param_0];
; CHECK-SM70-NEXT: shl.b32 %r6, %r5, 16;
; CHECK-SM70-NEXT: fma.rn.f32 %r7, %r6, %r4, %r2;
-; CHECK-SM70-NEXT: bfe.u32 %r8, %r7, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r9, %r8, %r7;
-; CHECK-SM70-NEXT: add.s32 %r10, %r9, 32767;
+; CHECK-SM70-NEXT: shr.u32 %r8, %r7, 16;
+; CHECK-SM70-NEXT: and.b32 %r9, %r8, 1;
+; CHECK-SM70-NEXT: add.s32 %r10, %r9, %r7;
+; CHECK-SM70-NEXT: add.s32 %r11, %r10, 32767;
; CHECK-SM70-NEXT: setp.nan.f32 %p1, %r7, %r7;
-; CHECK-SM70-NEXT: or.b32 %r11, %r7, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r12, %r11, %r10, %p1;
-; CHECK-SM70-NEXT: and.b32 %r13, %r12, -65536;
-; CHECK-SM70-NEXT: max.f32 %r14, %r13, 0f00000000;
-; CHECK-SM70-NEXT: bfe.u32 %r15, %r14, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r16, %r15, %r14;
-; CHECK-SM70-NEXT: add.s32 %r17, %r16, 32767;
-; CHECK-SM70-NEXT: setp.nan.f32 %p2, %r14, %r14;
-; CHECK-SM70-NEXT: or.b32 %r18, %r14, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r19, %r18, %r17, %p2;
-; CHECK-SM70-NEXT: shr.u32 %r20, %r19, 16;
-; CHECK-SM70-NEXT: st.param.b16 [func_retval0], %r20;
+; CHECK-SM70-NEXT: or.b32 %r12, %r7, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r13, %r12, %r11, %p1;
+; CHECK-SM70-NEXT: and.b32 %r14, %r13, -65536;
+; CHECK-SM70-NEXT: max.f32 %r15, %r14, 0f00000000;
+; CHECK-SM70-NEXT: shr.u32 %r16, %r15, 16;
+; CHECK-SM70-NEXT: and.b32 %r17, %r16, 1;
+; CHECK-SM70-NEXT: add.s32 %r18, %r17, %r15;
+; CHECK-SM70-NEXT: add.s32 %r19, %r18, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p2, %r15, %r15;
+; CHECK-SM70-NEXT: or.b32 %r20, %r15, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r21, %r20, %r19, %p2;
+; CHECK-SM70-NEXT: shr.u32 %r22, %r21, 16;
+; CHECK-SM70-NEXT: st.param.b16 [func_retval0], %r22;
; CHECK-SM70-NEXT: ret;
%1 = fmul contract nnan bfloat %a, %b
%2 = fadd contract nnan bfloat %1, %c
@@ -762,7 +769,7 @@ define <2 x bfloat> @fma_bf16x2_expanded_unsafe_with_nans(<2 x bfloat> %a, <2 x
; CHECK-SM70: {
; CHECK-SM70-NEXT: .reg .pred %p<5>;
; CHECK-SM70-NEXT: .reg .b16 %rs<11>;
-; CHECK-SM70-NEXT: .reg .b32 %r<27>;
+; CHECK-SM70-NEXT: .reg .b32 %r<29>;
; CHECK-SM70-EMPTY:
; CHECK-SM70-NEXT: // %bb.0:
; CHECK-SM70-NEXT: ld.param.v2.b16 {%rs1, %rs2}, [fma_bf16x2_expanded_unsafe_with_nans_param_0];
@@ -775,31 +782,33 @@ define <2 x bfloat> @fma_bf16x2_expanded_unsafe_with_nans(<2 x bfloat> %a, <2 x
; CHECK-SM70-NEXT: cvt.u32.u16 %r5, %rs1;
; CHECK-SM70-NEXT: shl.b32 %r6, %r5, 16;
; CHECK-SM70-NEXT: fma.rn.f32 %r7, %r6, %r4, %r2;
-; CHECK-SM70-NEXT: bfe.u32 %r8, %r7, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r9, %r8, %r7;
-; CHECK-SM70-NEXT: add.s32 %r10, %r9, 32767;
+; CHECK-SM70-NEXT: shr.u32 %r8, %r7, 16;
+; CHECK-SM70-NEXT: and.b32 %r9, %r8, 1;
+; CHECK-SM70-NEXT: add.s32 %r10, %r9, %r7;
+; CHECK-SM70-NEXT: add.s32 %r11, %r10, 32767;
; CHECK-SM70-NEXT: setp.nan.f32 %p1, %r7, %r7;
-; CHECK-SM70-NEXT: or.b32 %r11, %r7, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r12, %r11, %r10, %p1;
-; CHECK-SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs7}, %r12; }
-; CHECK-SM70-NEXT: cvt.u32.u16 %r13, %rs6;
-; CHECK-SM70-NEXT: shl.b32 %r14, %r13, 16;
-; CHECK-SM70-NEXT: cvt.u32.u16 %r15, %rs4;
-; CHECK-SM70-NEXT: shl.b32 %r16, %r15, 16;
-; CHECK-SM70-NEXT: cvt.u32.u16 %r17, %rs2;
-; CHECK-SM70-NEXT: shl.b32 %r18, %r17, 16;
-; CHECK-SM70-NEXT: fma.rn.f32 %r19, %r18, %r16, %r14;
-; CHECK-SM70-NEXT: bfe.u32 %r20, %r19, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r21, %r20, %r19;
-; CHECK-SM70-NEXT: add.s32 %r22, %r21, 32767;
-; CHECK-SM70-NEXT: setp.nan.f32 %p2, %r19, %r19;
-; CHECK-SM70-NEXT: or.b32 %r23, %r19, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r24, %r23, %r22, %p2;
-; CHECK-SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs8}, %r24; }
-; CHECK-SM70-NEXT: and.b32 %r25, %r12, -65536;
-; CHECK-SM70-NEXT: setp.gt.f32 %p3, %r25, 0f00000000;
-; CHECK-SM70-NEXT: and.b32 %r26, %r24, -65536;
-; CHECK-SM70-NEXT: setp.gt.f32 %p4, %r26, 0f00000000;
+; CHECK-SM70-NEXT: or.b32 %r12, %r7, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r13, %r12, %r11, %p1;
+; CHECK-SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs7}, %r13; }
+; CHECK-SM70-NEXT: cvt.u32.u16 %r14, %rs6;
+; CHECK-SM70-NEXT: shl.b32 %r15, %r14, 16;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r16, %rs4;
+; CHECK-SM70-NEXT: shl.b32 %r17, %r16, 16;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r18, %rs2;
+; CHECK-SM70-NEXT: shl.b32 %r19, %r18, 16;
+; CHECK-SM70-NEXT: fma.rn.f32 %r20, %r19, %r17, %r15;
+; CHECK-SM70-NEXT: shr.u32 %r21, %r20, 16;
+; CHECK-SM70-NEXT: and.b32 %r22, %r21, 1;
+; CHECK-SM70-NEXT: add.s32 %r23, %r22, %r20;
+; CHECK-SM70-NEXT: add.s32 %r24, %r23, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p2, %r20, %r20;
+; CHECK-SM70-NEXT: or.b32 %r25, %r20, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r26, %r25, %r24, %p2;
+; CHECK-SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs8}, %r26; }
+; CHECK-SM70-NEXT: and.b32 %r27, %r13, -65536;
+; CHECK-SM70-NEXT: setp.gt.f32 %p3, %r27, 0f00000000;
+; CHECK-SM70-NEXT: and.b32 %r28, %r26, -65536;
+; CHECK-SM70-NEXT: setp.gt.f32 %p4, %r28, 0f00000000;
; CHECK-SM70-NEXT: selp.b16 %rs9, %rs8, 0x0000, %p4;
; CHECK-SM70-NEXT: selp.b16 %rs10, %rs7, 0x0000, %p3;
; CHECK-SM70-NEXT: st.param.v2.b16 [func_retval0], {%rs10, %rs9};
@@ -840,7 +849,7 @@ define <2 x bfloat> @fma_bf16x2_expanded_no_nans(<2 x bfloat> %a, <2 x bfloat> %
; CHECK-SM70: {
; CHECK-SM70-NEXT: .reg .pred %p<5>;
; CHECK-SM70-NEXT: .reg .b16 %rs<11>;
-; CHECK-SM70-NEXT: .reg .b32 %r<27>;
+; CHECK-SM70-NEXT: .reg .b32 %r<29>;
; CHECK-SM70-EMPTY:
; CHECK-SM70-NEXT: // %bb.0:
; CHECK-SM70-NEXT: ld.param.v2.b16 {%rs1, %rs2}, [fma_bf16x2_expanded_no_nans_param_0];
@@ -853,31 +862,33 @@ define <2 x bfloat> @fma_bf16x2_expanded_no_nans(<2 x bfloat> %a, <2 x bfloat> %
; CHECK-SM70-NEXT: cvt.u32.u16 %r5, %rs1;
; CHECK-SM70-NEXT: shl.b32 %r6, %r5, 16;
; CHECK-SM70-NEXT: fma.rn.f32 %r7, %r6, %r4, %r2;
-; CHECK-SM70-NEXT: bfe.u32 %r8, %r7, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r9, %r8, %r7;
-; CHECK-SM70-NEXT: add.s32 %r10, %r9, 32767;
+; CHECK-SM70-NEXT: shr.u32 %r8, %r7, 16;
+; CHECK-SM70-NEXT: and.b32 %r9, %r8, 1;
+; CHECK-SM70-NEXT: add.s32 %r10, %r9, %r7;
+; CHECK-SM70-NEXT: add.s32 %r11, %r10, 32767;
; CHECK-SM70-NEXT: setp.nan.f32 %p1, %r7, %r7;
-; CHECK-SM70-NEXT: or.b32 %r11, %r7, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r12, %r11, %r10, %p1;
-; CHECK-SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs7}, %r12; }
-; CHECK-SM70-NEXT: cvt.u32.u16 %r13, %rs6;
-; CHECK-SM70-NEXT: shl.b32 %r14, %r13, 16;
-; CHECK-SM70-NEXT: cvt.u32.u16 %r15, %rs4;
-; CHECK-SM70-NEXT: shl.b32 %r16, %r15, 16;
-; CHECK-SM70-NEXT: cvt.u32.u16 %r17, %rs2;
-; CHECK-SM70-NEXT: shl.b32 %r18, %r17, 16;
-; CHECK-SM70-NEXT: fma.rn.f32 %r19, %r18, %r16, %r14;
-; CHECK-SM70-NEXT: bfe.u32 %r20, %r19, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r21, %r20, %r19;
-; CHECK-SM70-NEXT: add.s32 %r22, %r21, 32767;
-; CHECK-SM70-NEXT: setp.nan.f32 %p2, %r19, %r19;
-; CHECK-SM70-NEXT: or.b32 %r23, %r19, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r24, %r23, %r22, %p2;
-; CHECK-SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs8}, %r24; }
-; CHECK-SM70-NEXT: and.b32 %r25, %r12, -65536;
-; CHECK-SM70-NEXT: setp.gt.f32 %p3, %r25, 0f00000000;
-; CHECK-SM70-NEXT: and.b32 %r26, %r24, -65536;
-; CHECK-SM70-NEXT: setp.gt.f32 %p4, %r26, 0f00000000;
+; CHECK-SM70-NEXT: or.b32 %r12, %r7, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r13, %r12, %r11, %p1;
+; CHECK-SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs7}, %r13; }
+; CHECK-SM70-NEXT: cvt.u32.u16 %r14, %rs6;
+; CHECK-SM70-NEXT: shl.b32 %r15, %r14, 16;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r16, %rs4;
+; CHECK-SM70-NEXT: shl.b32 %r17, %r16, 16;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r18, %rs2;
+; CHECK-SM70-NEXT: shl.b32 %r19, %r18, 16;
+; CHECK-SM70-NEXT: fma.rn.f32 %r20, %r19, %r17, %r15;
+; CHECK-SM70-NEXT: shr.u32 %r21, %r20, 16;
+; CHECK-SM70-NEXT: and.b32 %r22, %r21, 1;
+; CHECK-SM70-NEXT: add.s32 %r23, %r22, %r20;
+; CHECK-SM70-NEXT: add.s32 %r24, %r23, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p2, %r20, %r20;
+; CHECK-SM70-NEXT: or.b32 %r25, %r20, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r26, %r25, %r24, %p2;
+; CHECK-SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs8}, %r26; }
+; CHECK-SM70-NEXT: and.b32 %r27, %r13, -65536;
+; CHECK-SM70-NEXT: setp.gt.f32 %p3, %r27, 0f00000000;
+; CHECK-SM70-NEXT: and.b32 %r28, %r26, -65536;
+; CHECK-SM70-NEXT: setp.gt.f32 %p4, %r28, 0f00000000;
; CHECK-SM70-NEXT: selp.b16 %rs9, %rs8, 0x0000, %p4;
; CHECK-SM70-NEXT: selp.b16 %rs10, %rs7, 0x0000, %p3;
; CHECK-SM70-NEXT: st.param.v2.b16 [func_retval0], {%rs10, %rs9};
@@ -949,7 +960,7 @@ define <2 x bfloat> @fma_bf16x2_expanded_no_nans_multiple_uses_of_fma(<2 x bfloa
; CHECK-SM70: {
; CHECK-SM70-NEXT: .reg .pred %p<9>;
; CHECK-SM70-NEXT: .reg .b16 %rs<11>;
-; CHECK-SM70-NEXT: .reg .b32 %r<58>;
+; CHECK-SM70-NEXT: .reg .b32 %r<64>;
; CHECK-SM70-EMPTY:
; CHECK-SM70-NEXT: // %bb.0:
; CHECK-SM70-NEXT: ld.param.v2.b16 {%rs1, %rs2}, [fma_bf16x2_expanded_no_nans_multiple_uses_of_fma_param_0];
@@ -962,69 +973,75 @@ define <2 x bfloat> @fma_bf16x2_expanded_no_nans_multiple_uses_of_fma(<2 x bfloa
; CHECK-SM70-NEXT: cvt.u32.u16 %r5, %rs2;
; CHECK-SM70-NEXT: shl.b32 %r6, %r5, 16;
; CHECK-SM70-NEXT: fma.rn.f32 %r7, %r6, %r4, %r2;
-; CHECK-SM70-NEXT: bfe.u32 %r8, %r7, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r9, %r8, %r7;
-; CHECK-SM70-NEXT: add.s32 %r10, %r9, 32767;
+; CHECK-SM70-NEXT: shr.u32 %r8, %r7, 16;
+; CHECK-SM70-NEXT: and.b32 %r9, %r8, 1;
+; CHECK-SM70-NEXT: add.s32 %r10, %r9, %r7;
+; CHECK-SM70-NEXT: add.s32 %r11, %r10, 32767;
; CHECK-SM70-NEXT: setp.nan.f32 %p1, %r7, %r7;
-; CHECK-SM70-NEXT: or.b32 %r11, %r7, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r12, %r11, %r10, %p1;
-; CHECK-SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs7}, %r12; }
-; CHECK-SM70-NEXT: cvt.u32.u16 %r13, %rs5;
-; CHECK-SM70-NEXT: shl.b32 %r14, %r13, 16;
-; CHECK-SM70-NEXT: cvt.u32.u16 %r15, %rs3;
-; CHECK-SM70-NEXT: shl.b32 %r16, %r15, 16;
-; CHECK-SM70-NEXT: cvt.u32.u16 %r17, %rs1;
-; CHECK-SM70-NEXT: shl.b32 %r18, %r17, 16;
-; CHECK-SM70-NEXT: fma.rn.f32 %r19, %r18, %r16, %r14;
-; CHECK-SM70-NEXT: bfe.u32 %r20, %r19, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r21, %r20, %r19;
-; CHECK-SM70-NEXT: add.s32 %r22, %r21, 32767;
-; CHECK-SM70-NEXT: setp.nan.f32 %p2, %r19, %r19;
-; CHECK-SM70-NEXT: or.b32 %r23, %r19, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r24, %r23, %r22, %p2;
-; CHECK-SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs8}, %r24; }
-; CHECK-SM70-NEXT: and.b32 %r25, %r12, -65536;
-; CHECK-SM70-NEXT: setp.gt.f32 %p3, %r25, 0f00000000;
-; CHECK-SM70-NEXT: and.b32 %r26, %r24, -65536;
-; CHECK-SM70-NEXT: setp.gt.f32 %p4, %r26, 0f00000000;
+; CHECK-SM70-NEXT: or.b32 %r12, %r7, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r13, %r12, %r11, %p1;
+; CHECK-SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs7}, %r13; }
+; CHECK-SM70-NEXT: cvt.u32.u16 %r14, %rs5;
+; CHECK-SM70-NEXT: shl.b32 %r15, %r14, 16;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r16, %rs3;
+; CHECK-SM70-NEXT: shl.b32 %r17, %r16, 16;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r18, %rs1;
+; CHECK-SM70-NEXT: shl.b32 %r19, %r18, 16;
+; CHECK-SM70-NEXT: fma.rn.f32 %r20, %r19, %r17, %r15;
+; CHECK-SM70-NEXT: shr.u32 %r21, %r20, 16;
+; CHECK-SM70-NEXT: and.b32 %r22, %r21, 1;
+; CHECK-SM70-NEXT: add.s32 %r23, %r22, %r20;
+; CHECK-SM70-NEXT: add.s32 %r24, %r23, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p2, %r20, %r20;
+; CHECK-SM70-NEXT: or.b32 %r25, %r20, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r26, %r25, %r24, %p2;
+; CHECK-SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs8}, %r26; }
+; CHECK-SM70-NEXT: and.b32 %r27, %r13, -65536;
+; CHECK-SM70-NEXT: setp.gt.f32 %p3, %r27, 0f00000000;
+; CHECK-SM70-NEXT: and.b32 %r28, %r26, -65536;
+; CHECK-SM70-NEXT: setp.gt.f32 %p4, %r28, 0f00000000;
; CHECK-SM70-NEXT: selp.b16 %rs9, %rs8, 0x0000, %p4;
; CHECK-SM70-NEXT: selp.b16 %rs10, %rs7, 0x0000, %p3;
-; CHECK-SM70-NEXT: add.f32 %r27, %r26, 0f40E00000;
-; CHECK-SM70-NEXT: bfe.u32 %r28, %r27, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r29, %r28, %r27;
-; CHECK-SM70-NEXT: add.s32 %r30, %r29, 32767;
-; CHECK-SM70-NEXT: setp.nan.f32 %p5, %r27, %r27;
-; CHECK-SM70-NEXT: or.b32 %r31, %r27, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r32, %r31, %r30, %p5;
-; CHECK-SM70-NEXT: add.f32 %r33, %r25, 0f40E00000;
-; CHECK-SM70-NEXT: bfe.u32 %r34, %r33, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r35, %r34, %r33;
-; CHECK-SM70-NEXT: add.s32 %r36, %r35, 32767;
-; CHECK-SM70-NEXT: setp.nan.f32 %p6, %r33, %r33;
-; CHECK-SM70-NEXT: or.b32 %r37, %r33, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r38, %r37, %r36, %p6;
-; CHECK-SM70-NEXT: cvt.u32.u16 %r39, %rs10;
-; CHECK-SM70-NEXT: shl.b32 %r40, %r39, 16;
-; CHECK-SM70-NEXT: and.b32 %r41, %r38, -65536;
-; CHECK-SM70-NEXT: add.f32 %r42, %r40, %r41;
-; CHECK-SM70-NEXT: bfe.u32 %r43, %r42, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r44, %r43, %r42;
-; CHECK-SM70-NEXT: add.s32 %r45, %r44, 32767;
-; CHECK-SM70-NEXT: setp.nan.f32 %p7, %r42, %r42;
-; CHECK-SM70-NEXT: or.b32 %r46, %r42, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r47, %r46, %r45, %p7;
-; CHECK-SM70-NEXT: cvt.u32.u16 %r48, %rs9;
-; CHECK-SM70-NEXT: shl.b32 %r49, %r48, 16;
-; CHECK-SM70-NEXT: and.b32 %r50, %r32, -65536;
-; CHECK-SM70-NEXT: add.f32 %r51, %r49, %r50;
-; CHECK-SM70-NEXT: bfe.u32 %r52, %r51, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r53, %r52, %r51;
-; CHECK-SM70-NEXT: add.s32 %r54, %r53, 32767;
-; CHECK-SM70-NEXT: setp.nan.f32 %p8, %r51, %r51;
-; CHECK-SM70-NEXT: or.b32 %r55, %r51, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r56, %r55, %r54, %p8;
-; CHECK-SM70-NEXT: prmt.b32 %r57, %r56, %r47, 0x7632U;
-; CHECK-SM70-NEXT: st.param.b32 [func_retval0], %r57;
+; CHECK-SM70-NEXT: add.f32 %r29, %r28, 0f40E00000;
+; CHECK-SM70-NEXT: shr.u32 %r30, %r29, 16;
+; CHECK-SM70-NEXT: and.b32 %r31, %r30, 1;
+; CHECK-SM70-NEXT: add.s32 %r32, %r31, %r29;
+; CHECK-SM70-NEXT: add.s32 %r33, %r32, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p5, %r29, %r29;
+; CHECK-SM70-NEXT: or.b32 %r34, %r29, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r35, %r34, %r33, %p5;
+; CHECK-SM70-NEXT: add.f32 %r36, %r27, 0f40E00000;
+; CHECK-SM70-NEXT: shr.u32 %r37, %r36, 16;
+; CHECK-SM70-NEXT: and.b32 %r38, %r37, 1;
+; CHECK-SM70-NEXT: add.s32 %r39, %r38, %r36;
+; CHECK-SM70-NEXT: add.s32 %r40, %r39, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p6, %r36, %r36;
+; CHECK-SM70-NEXT: or.b32 %r41, %r36, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r42, %r41, %r40, %p6;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r43, %rs10;
+; CHECK-SM70-NEXT: shl.b32 %r44, %r43, 16;
+; CHECK-SM70-NEXT: and.b32 %r45, %r42, -65536;
+; CHECK-SM70-NEXT: add.f32 %r46, %r44, %r45;
+; CHECK-SM70-NEXT: shr.u32 %r47, %r46, 16;
+; CHECK-SM70-NEXT: and.b32 %r48, %r47, 1;
+; CHECK-SM70-NEXT: add.s32 %r49, %r48, %r46;
+; CHECK-SM70-NEXT: add.s32 %r50, %r49, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p7, %r46, %r46;
+; CHECK-SM70-NEXT: or.b32 %r51, %r46, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r52, %r51, %r50, %p7;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r53, %rs9;
+; CHECK-SM70-NEXT: shl.b32 %r54, %r53, 16;
+; CHECK-SM70-NEXT: and.b32 %r55, %r35, -65536;
+; CHECK-SM70-NEXT: add.f32 %r56, %r54, %r55;
+; CHECK-SM70-NEXT: shr.u32 %r57, %r56, 16;
+; CHECK-SM70-NEXT: and.b32 %r58, %r57, 1;
+; CHECK-SM70-NEXT: add.s32 %r59, %r58, %r56;
+; CHECK-SM70-NEXT: add.s32 %r60, %r59, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p8, %r56, %r56;
+; CHECK-SM70-NEXT: or.b32 %r61, %r56, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r62, %r61, %r60, %p8;
+; CHECK-SM70-NEXT: prmt.b32 %r63, %r62, %r52, 0x7632U;
+; CHECK-SM70-NEXT: st.param.b32 [func_retval0], %r63;
; CHECK-SM70-NEXT: ret;
%1 = fmul contract nnan <2 x bfloat> %a, %b
%2 = fadd contract nnan <2 x bfloat> %1, %c
@@ -1064,7 +1081,7 @@ define <2 x bfloat> @fma_bf16x2_expanded_maxnum_no_nans(<2 x bfloat> %a, <2 x bf
; CHECK-SM70: {
; CHECK-SM70-NEXT: .reg .pred %p<5>;
; CHECK-SM70-NEXT: .reg .b16 %rs<7>;
-; CHECK-SM70-NEXT: .reg .b32 %r<40>;
+; CHECK-SM70-NEXT: .reg .b32 %r<44>;
; CHECK-SM70-EMPTY:
; CHECK-SM70-NEXT: // %bb.0:
; CHECK-SM70-NEXT: ld.param.v2.b16 {%rs1, %rs2}, [fma_bf16x2_expanded_maxnum_no_nans_param_0];
@@ -1077,43 +1094,47 @@ define <2 x bfloat> @fma_bf16x2_expanded_maxnum_no_nans(<2 x bfloat> %a, <2 x bf
; CHECK-SM70-NEXT: cvt.u32.u16 %r5, %rs1;
; CHECK-SM70-NEXT: shl.b32 %r6, %r5, 16;
; CHECK-SM70-NEXT: fma.rn.f32 %r7, %r6, %r4, %r2;
-; CHECK-SM70-NEXT: bfe.u32 %r8, %r7, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r9, %r8, %r7;
-; CHECK-SM70-NEXT: add.s32 %r10, %r9, 32767;
+; CHECK-SM70-NEXT: shr.u32 %r8, %r7, 16;
+; CHECK-SM70-NEXT: and.b32 %r9, %r8, 1;
+; CHECK-SM70-NEXT: add.s32 %r10, %r9, %r7;
+; CHECK-SM70-NEXT: add.s32 %r11, %r10, 32767;
; CHECK-SM70-NEXT: setp.nan.f32 %p1, %r7, %r7;
-; CHECK-SM70-NEXT: or.b32 %r11, %r7, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r12, %r11, %r10, %p1;
-; CHECK-SM70-NEXT: cvt.u32.u16 %r13, %rs6;
-; CHECK-SM70-NEXT: shl.b32 %r14, %r13, 16;
-; CHECK-SM70-NEXT: cvt.u32.u16 %r15, %rs4;
-; CHECK-SM70-NEXT: shl.b32 %r16, %r15, 16;
-; CHECK-SM70-NEXT: cvt.u32.u16 %r17, %rs2;
-; CHECK-SM70-NEXT: shl.b32 %r18, %r17, 16;
-; CHECK-SM70-NEXT: fma.rn.f32 %r19, %r18, %r16, %r14;
-; CHECK-SM70-NEXT: bfe.u32 %r20, %r19, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r21, %r20, %r19;
-; CHECK-SM70-NEXT: add.s32 %r22, %r21, 32767;
-; CHECK-SM70-NEXT: setp.nan.f32 %p2, %r19, %r19;
-; CHECK-SM70-NEXT: or.b32 %r23, %r19, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r24, %r23, %r22, %p2;
-; CHECK-SM70-NEXT: and.b32 %r25, %r24, -65536;
-; CHECK-SM70-NEXT: max.f32 %r26, %r25, 0f00000000;
-; CHECK-SM70-NEXT: bfe.u32 %r27, %r26, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r28, %r27, %r26;
-; CHECK-SM70-NEXT: add.s32 %r29, %r28, 32767;
-; CHECK-SM70-NEXT: setp.nan.f32 %p3, %r26, %r26;
-; CHECK-SM70-NEXT: or.b32 %r30, %r26, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r31, %r30, %r29, %p3;
-; CHECK-SM70-NEXT: and.b32 %r32, %r12, -65536;
-; CHECK-SM70-NEXT: max.f32 %r33, %r32, 0f00000000;
-; CHECK-SM70-NEXT: bfe.u32 %r34, %r33, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r35, %r34, %r33;
-; CHECK-SM70-NEXT: add.s32 %r36, %r35, 32767;
-; CHECK-SM70-NEXT: setp.nan.f32 %p4, %r33, %r33;
-; CHECK-SM70-NEXT: or.b32 %r37, %r33, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r38, %r37, %r36, %p4;
-; CHECK-SM70-NEXT: prmt.b32 %r39, %r38, %r31, 0x7632U;
-; CHECK-SM70-NEXT: st.param.b32 [func_retval0], %r39;
+; CHECK-SM70-NEXT: or.b32 %r12, %r7, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r13, %r12, %r11, %p1;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r14, %rs6;
+; CHECK-SM70-NEXT: shl.b32 %r15, %r14, 16;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r16, %rs4;
+; CHECK-SM70-NEXT: shl.b32 %r17, %r16, 16;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r18, %rs2;
+; CHECK-SM70-NEXT: shl.b32 %r19, %r18, 16;
+; CHECK-SM70-NEXT: fma.rn.f32 %r20, %r19, %r17, %r15;
+; CHECK-SM70-NEXT: shr.u32 %r21, %r20, 16;
+; CHECK-SM70-NEXT: and.b32 %r22, %r21, 1;
+; CHECK-SM70-NEXT: add.s32 %r23, %r22, %r20;
+; CHECK-SM70-NEXT: add.s32 %r24, %r23, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p2, %r20, %r20;
+; CHECK-SM70-NEXT: or.b32 %r25, %r20, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r26, %r25, %r24, %p2;
+; CHECK-SM70-NEXT: and.b32 %r27, %r26, -65536;
+; CHECK-SM70-NEXT: max.f32 %r28, %r27, 0f00000000;
+; CHECK-SM70-NEXT: shr.u32 %r29, %r28, 16;
+; CHECK-SM70-NEXT: and.b32 %r30, %r29, 1;
+; CHECK-SM70-NEXT: add.s32 %r31, %r30, %r28;
+; CHECK-SM70-NEXT: add.s32 %r32, %r31, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p3, %r28, %r28;
+; CHECK-SM70-NEXT: or.b32 %r33, %r28, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r34, %r33, %r32, %p3;
+; CHECK-SM70-NEXT: and.b32 %r35, %r13, -65536;
+; CHECK-SM70-NEXT: max.f32 %r36, %r35, 0f00000000;
+; CHECK-SM70-NEXT: shr.u32 %r37, %r36, 16;
+; CHECK-SM70-NEXT: and.b32 %r38, %r37, 1;
+; CHECK-SM70-NEXT: add.s32 %r39, %r38, %r36;
+; CHECK-SM70-NEXT: add.s32 %r40, %r39, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p4, %r36, %r36;
+; CHECK-SM70-NEXT: or.b32 %r41, %r36, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r42, %r41, %r40, %p4;
+; CHECK-SM70-NEXT: prmt.b32 %r43, %r42, %r34, 0x7632U;
+; CHECK-SM70-NEXT: st.param.b32 [func_retval0], %r43;
; CHECK-SM70-NEXT: ret;
%1 = fmul contract nnan <2 x bfloat> %a, %b
%2 = fadd contract nnan <2 x bfloat> %1, %c
diff --git a/llvm/test/CodeGen/NVPTX/fma-relu-fma-intrinsic.ll b/llvm/test/CodeGen/NVPTX/fma-relu-fma-intrinsic.ll
index 6d983ba6bf0ff..041490509d129 100644
--- a/llvm/test/CodeGen/NVPTX/fma-relu-fma-intrinsic.ll
+++ b/llvm/test/CodeGen/NVPTX/fma-relu-fma-intrinsic.ll
@@ -183,7 +183,7 @@ define bfloat @fma_bf16_no_nans(bfloat %a, bfloat %b, bfloat %c) {
; CHECK-SM70: {
; CHECK-SM70-NEXT: .reg .pred %p<3>;
; CHECK-SM70-NEXT: .reg .b16 %rs<3>;
-; CHECK-SM70-NEXT: .reg .b32 %r<14>;
+; CHECK-SM70-NEXT: .reg .b32 %r<15>;
; CHECK-SM70-EMPTY:
; CHECK-SM70-NEXT: // %bb.0:
; CHECK-SM70-NEXT: ld.param.b16 %r1, [fma_bf16_no_nans_param_2];
@@ -193,15 +193,16 @@ define bfloat @fma_bf16_no_nans(bfloat %a, bfloat %b, bfloat %c) {
; CHECK-SM70-NEXT: ld.param.b16 %r5, [fma_bf16_no_nans_param_0];
; CHECK-SM70-NEXT: shl.b32 %r6, %r5, 16;
; CHECK-SM70-NEXT: fma.rn.f32 %r7, %r6, %r4, %r2;
-; CHECK-SM70-NEXT: bfe.u32 %r8, %r7, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r9, %r8, %r7;
-; CHECK-SM70-NEXT: add.s32 %r10, %r9, 32767;
+; CHECK-SM70-NEXT: shr.u32 %r8, %r7, 16;
+; CHECK-SM70-NEXT: and.b32 %r9, %r8, 1;
+; CHECK-SM70-NEXT: add.s32 %r10, %r9, %r7;
+; CHECK-SM70-NEXT: add.s32 %r11, %r10, 32767;
; CHECK-SM70-NEXT: setp.nan.f32 %p1, %r7, %r7;
-; CHECK-SM70-NEXT: or.b32 %r11, %r7, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r12, %r11, %r10, %p1;
-; CHECK-SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs1}, %r12; }
-; CHECK-SM70-NEXT: and.b32 %r13, %r12, -65536;
-; CHECK-SM70-NEXT: setp.gt.f32 %p2, %r13, 0f00000000;
+; CHECK-SM70-NEXT: or.b32 %r12, %r7, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r13, %r12, %r11, %p1;
+; CHECK-SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs1}, %r13; }
+; CHECK-SM70-NEXT: and.b32 %r14, %r13, -65536;
+; CHECK-SM70-NEXT: setp.gt.f32 %p2, %r14, 0f00000000;
; CHECK-SM70-NEXT: selp.b16 %rs2, %rs1, 0x0000, %p2;
; CHECK-SM70-NEXT: st.param.b16 [func_retval0], %rs2;
; CHECK-SM70-NEXT: ret;
@@ -253,7 +254,7 @@ define bfloat @fma_bf16_no_nans_multiple_uses_of_fma(bfloat %a, bfloat %b, bfloa
; CHECK-SM70-LABEL: fma_bf16_no_nans_multiple_uses_of_fma(
; CHECK-SM70: {
; CHECK-SM70-NEXT: .reg .pred %p<4>;
-; CHECK-SM70-NEXT: .reg .b32 %r<28>;
+; CHECK-SM70-NEXT: .reg .b32 %r<31>;
; CHECK-SM70-EMPTY:
; CHECK-SM70-NEXT: // %bb.0:
; CHECK-SM70-NEXT: ld.param.b16 %r1, [fma_bf16_no_nans_multiple_uses_of_fma_param_2];
@@ -263,30 +264,33 @@ define bfloat @fma_bf16_no_nans_multiple_uses_of_fma(bfloat %a, bfloat %b, bfloa
; CHECK-SM70-NEXT: ld.param.b16 %r5, [fma_bf16_no_nans_multiple_uses_of_fma_param_0];
; CHECK-SM70-NEXT: shl.b32 %r6, %r5, 16;
; CHECK-SM70-NEXT: fma.rn.f32 %r7, %r6, %r4, %r2;
-; CHECK-SM70-NEXT: bfe.u32 %r8, %r7, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r9, %r8, %r7;
-; CHECK-SM70-NEXT: add.s32 %r10, %r9, 32767;
+; CHECK-SM70-NEXT: shr.u32 %r8, %r7, 16;
+; CHECK-SM70-NEXT: and.b32 %r9, %r8, 1;
+; CHECK-SM70-NEXT: add.s32 %r10, %r9, %r7;
+; CHECK-SM70-NEXT: add.s32 %r11, %r10, 32767;
; CHECK-SM70-NEXT: setp.nan.f32 %p1, %r7, %r7;
-; CHECK-SM70-NEXT: or.b32 %r11, %r7, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r12, %r11, %r10, %p1;
-; CHECK-SM70-NEXT: and.b32 %r13, %r12, -65536;
-; CHECK-SM70-NEXT: add.f32 %r14, %r13, 0f40E00000;
-; CHECK-SM70-NEXT: bfe.u32 %r15, %r14, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r16, %r15, %r14;
-; CHECK-SM70-NEXT: add.s32 %r17, %r16, 32767;
-; CHECK-SM70-NEXT: setp.nan.f32 %p2, %r14, %r14;
-; CHECK-SM70-NEXT: or.b32 %r18, %r14, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r19, %r18, %r17, %p2;
-; CHECK-SM70-NEXT: and.b32 %r20, %r19, -65536;
-; CHECK-SM70-NEXT: add.f32 %r21, %r20, %r13;
-; CHECK-SM70-NEXT: bfe.u32 %r22, %r21, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r23, %r22, %r21;
-; CHECK-SM70-NEXT: add.s32 %r24, %r23, 32767;
-; CHECK-SM70-NEXT: setp.nan.f32 %p3, %r21, %r21;
-; CHECK-SM70-NEXT: or.b32 %r25, %r21, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r26, %r25, %r24, %p3;
-; CHECK-SM70-NEXT: shr.u32 %r27, %r26, 16;
-; CHECK-SM70-NEXT: st.param.b16 [func_retval0], %r27;
+; CHECK-SM70-NEXT: or.b32 %r12, %r7, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r13, %r12, %r11, %p1;
+; CHECK-SM70-NEXT: and.b32 %r14, %r13, -65536;
+; CHECK-SM70-NEXT: add.f32 %r15, %r14, 0f40E00000;
+; CHECK-SM70-NEXT: shr.u32 %r16, %r15, 16;
+; CHECK-SM70-NEXT: and.b32 %r17, %r16, 1;
+; CHECK-SM70-NEXT: add.s32 %r18, %r17, %r15;
+; CHECK-SM70-NEXT: add.s32 %r19, %r18, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p2, %r15, %r15;
+; CHECK-SM70-NEXT: or.b32 %r20, %r15, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r21, %r20, %r19, %p2;
+; CHECK-SM70-NEXT: and.b32 %r22, %r21, -65536;
+; CHECK-SM70-NEXT: add.f32 %r23, %r22, %r14;
+; CHECK-SM70-NEXT: shr.u32 %r24, %r23, 16;
+; CHECK-SM70-NEXT: and.b32 %r25, %r24, 1;
+; CHECK-SM70-NEXT: add.s32 %r26, %r25, %r23;
+; CHECK-SM70-NEXT: add.s32 %r27, %r26, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p3, %r23, %r23;
+; CHECK-SM70-NEXT: or.b32 %r28, %r23, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r29, %r28, %r27, %p3;
+; CHECK-SM70-NEXT: shr.u32 %r30, %r29, 16;
+; CHECK-SM70-NEXT: st.param.b16 [func_retval0], %r30;
; CHECK-SM70-NEXT: ret;
%1 = call bfloat @llvm.fma.bf16(bfloat %a, bfloat %b, bfloat %c)
%2 = fcmp ogt bfloat %1, 0.0
@@ -324,7 +328,7 @@ define bfloat @fma_bf16_maxnum_no_nans(bfloat %a, bfloat %b, bfloat %c) {
; CHECK-SM70-LABEL: fma_bf16_maxnum_no_nans(
; CHECK-SM70: {
; CHECK-SM70-NEXT: .reg .pred %p<3>;
-; CHECK-SM70-NEXT: .reg .b32 %r<21>;
+; CHECK-SM70-NEXT: .reg .b32 %r<23>;
; CHECK-SM70-EMPTY:
; CHECK-SM70-NEXT: // %bb.0:
; CHECK-SM70-NEXT: ld.param.b16 %r1, [fma_bf16_maxnum_no_nans_param_2];
@@ -334,22 +338,24 @@ define bfloat @fma_bf16_maxnum_no_nans(bfloat %a, bfloat %b, bfloat %c) {
; CHECK-SM70-NEXT: ld.param.b16 %r5, [fma_bf16_maxnum_no_nans_param_0];
; CHECK-SM70-NEXT: shl.b32 %r6, %r5, 16;
; CHECK-SM70-NEXT: fma.rn.f32 %r7, %r6, %r4, %r2;
-; CHECK-SM70-NEXT: bfe.u32 %r8, %r7, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r9, %r8, %r7;
-; CHECK-SM70-NEXT: add.s32 %r10, %r9, 32767;
+; CHECK-SM70-NEXT: shr.u32 %r8, %r7, 16;
+; CHECK-SM70-NEXT: and.b32 %r9, %r8, 1;
+; CHECK-SM70-NEXT: add.s32 %r10, %r9, %r7;
+; CHECK-SM70-NEXT: add.s32 %r11, %r10, 32767;
; CHECK-SM70-NEXT: setp.nan.f32 %p1, %r7, %r7;
-; CHECK-SM70-NEXT: or.b32 %r11, %r7, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r12, %r11, %r10, %p1;
-; CHECK-SM70-NEXT: and.b32 %r13, %r12, -65536;
-; CHECK-SM70-NEXT: max.f32 %r14, %r13, 0f00000000;
-; CHECK-SM70-NEXT: bfe.u32 %r15, %r14, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r16, %r15, %r14;
-; CHECK-SM70-NEXT: add.s32 %r17, %r16, 32767;
-; CHECK-SM70-NEXT: setp.nan.f32 %p2, %r14, %r14;
-; CHECK-SM70-NEXT: or.b32 %r18, %r14, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r19, %r18, %r17, %p2;
-; CHECK-SM70-NEXT: shr.u32 %r20, %r19, 16;
-; CHECK-SM70-NEXT: st.param.b16 [func_retval0], %r20;
+; CHECK-SM70-NEXT: or.b32 %r12, %r7, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r13, %r12, %r11, %p1;
+; CHECK-SM70-NEXT: and.b32 %r14, %r13, -65536;
+; CHECK-SM70-NEXT: max.f32 %r15, %r14, 0f00000000;
+; CHECK-SM70-NEXT: shr.u32 %r16, %r15, 16;
+; CHECK-SM70-NEXT: and.b32 %r17, %r16, 1;
+; CHECK-SM70-NEXT: add.s32 %r18, %r17, %r15;
+; CHECK-SM70-NEXT: add.s32 %r19, %r18, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p2, %r15, %r15;
+; CHECK-SM70-NEXT: or.b32 %r20, %r15, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r21, %r20, %r19, %p2;
+; CHECK-SM70-NEXT: shr.u32 %r22, %r21, 16;
+; CHECK-SM70-NEXT: st.param.b16 [func_retval0], %r22;
; CHECK-SM70-NEXT: ret;
%1 = call nnan bfloat @llvm.fma.bf16(bfloat %a, bfloat %b, bfloat %c)
%2 = call nsz bfloat @llvm.maxnum.bf16(bfloat %1, bfloat 0.0)
@@ -538,7 +544,7 @@ define <2 x bfloat> @fma_bf16x2_no_nans(<2 x bfloat> %a, <2 x bfloat> %b, <2 x b
; CHECK-SM70: {
; CHECK-SM70-NEXT: .reg .pred %p<5>;
; CHECK-SM70-NEXT: .reg .b16 %rs<11>;
-; CHECK-SM70-NEXT: .reg .b32 %r<27>;
+; CHECK-SM70-NEXT: .reg .b32 %r<29>;
; CHECK-SM70-EMPTY:
; CHECK-SM70-NEXT: // %bb.0:
; CHECK-SM70-NEXT: ld.param.v2.b16 {%rs1, %rs2}, [fma_bf16x2_no_nans_param_0];
@@ -551,31 +557,33 @@ define <2 x bfloat> @fma_bf16x2_no_nans(<2 x bfloat> %a, <2 x bfloat> %b, <2 x b
; CHECK-SM70-NEXT: cvt.u32.u16 %r5, %rs1;
; CHECK-SM70-NEXT: shl.b32 %r6, %r5, 16;
; CHECK-SM70-NEXT: fma.rn.f32 %r7, %r6, %r4, %r2;
-; CHECK-SM70-NEXT: bfe.u32 %r8, %r7, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r9, %r8, %r7;
-; CHECK-SM70-NEXT: add.s32 %r10, %r9, 32767;
+; CHECK-SM70-NEXT: shr.u32 %r8, %r7, 16;
+; CHECK-SM70-NEXT: and.b32 %r9, %r8, 1;
+; CHECK-SM70-NEXT: add.s32 %r10, %r9, %r7;
+; CHECK-SM70-NEXT: add.s32 %r11, %r10, 32767;
; CHECK-SM70-NEXT: setp.nan.f32 %p1, %r7, %r7;
-; CHECK-SM70-NEXT: or.b32 %r11, %r7, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r12, %r11, %r10, %p1;
-; CHECK-SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs7}, %r12; }
-; CHECK-SM70-NEXT: cvt.u32.u16 %r13, %rs4;
-; CHECK-SM70-NEXT: shl.b32 %r14, %r13, 16;
-; CHECK-SM70-NEXT: cvt.u32.u16 %r15, %rs6;
-; CHECK-SM70-NEXT: shl.b32 %r16, %r15, 16;
-; CHECK-SM70-NEXT: cvt.u32.u16 %r17, %rs2;
-; CHECK-SM70-NEXT: shl.b32 %r18, %r17, 16;
-; CHECK-SM70-NEXT: fma.rn.f32 %r19, %r18, %r16, %r14;
-; CHECK-SM70-NEXT: bfe.u32 %r20, %r19, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r21, %r20, %r19;
-; CHECK-SM70-NEXT: add.s32 %r22, %r21, 32767;
-; CHECK-SM70-NEXT: setp.nan.f32 %p2, %r19, %r19;
-; CHECK-SM70-NEXT: or.b32 %r23, %r19, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r24, %r23, %r22, %p2;
-; CHECK-SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs8}, %r24; }
-; CHECK-SM70-NEXT: and.b32 %r25, %r12, -65536;
-; CHECK-SM70-NEXT: setp.gt.f32 %p3, %r25, 0f00000000;
-; CHECK-SM70-NEXT: and.b32 %r26, %r24, -65536;
-; CHECK-SM70-NEXT: setp.gt.f32 %p4, %r26, 0f00000000;
+; CHECK-SM70-NEXT: or.b32 %r12, %r7, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r13, %r12, %r11, %p1;
+; CHECK-SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs7}, %r13; }
+; CHECK-SM70-NEXT: cvt.u32.u16 %r14, %rs4;
+; CHECK-SM70-NEXT: shl.b32 %r15, %r14, 16;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r16, %rs6;
+; CHECK-SM70-NEXT: shl.b32 %r17, %r16, 16;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r18, %rs2;
+; CHECK-SM70-NEXT: shl.b32 %r19, %r18, 16;
+; CHECK-SM70-NEXT: fma.rn.f32 %r20, %r19, %r17, %r15;
+; CHECK-SM70-NEXT: shr.u32 %r21, %r20, 16;
+; CHECK-SM70-NEXT: and.b32 %r22, %r21, 1;
+; CHECK-SM70-NEXT: add.s32 %r23, %r22, %r20;
+; CHECK-SM70-NEXT: add.s32 %r24, %r23, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p2, %r20, %r20;
+; CHECK-SM70-NEXT: or.b32 %r25, %r20, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r26, %r25, %r24, %p2;
+; CHECK-SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs8}, %r26; }
+; CHECK-SM70-NEXT: and.b32 %r27, %r13, -65536;
+; CHECK-SM70-NEXT: setp.gt.f32 %p3, %r27, 0f00000000;
+; CHECK-SM70-NEXT: and.b32 %r28, %r26, -65536;
+; CHECK-SM70-NEXT: setp.gt.f32 %p4, %r28, 0f00000000;
; CHECK-SM70-NEXT: selp.b16 %rs9, %rs8, 0x0000, %p4;
; CHECK-SM70-NEXT: selp.b16 %rs10, %rs7, 0x0000, %p3;
; CHECK-SM70-NEXT: st.param.v2.b16 [func_retval0], {%rs10, %rs9};
@@ -637,7 +645,7 @@ define <2 x bfloat> @fma_bf16x2_no_nans_multiple_uses_of_fma(<2 x bfloat> %a, <2
; CHECK-SM70: {
; CHECK-SM70-NEXT: .reg .pred %p<7>;
; CHECK-SM70-NEXT: .reg .b16 %rs<7>;
-; CHECK-SM70-NEXT: .reg .b32 %r<54>;
+; CHECK-SM70-NEXT: .reg .b32 %r<60>;
; CHECK-SM70-EMPTY:
; CHECK-SM70-NEXT: // %bb.0:
; CHECK-SM70-NEXT: ld.param.v2.b16 {%rs1, %rs2}, [fma_bf16x2_no_nans_multiple_uses_of_fma_param_0];
@@ -650,59 +658,65 @@ define <2 x bfloat> @fma_bf16x2_no_nans_multiple_uses_of_fma(<2 x bfloat> %a, <2
; CHECK-SM70-NEXT: cvt.u32.u16 %r5, %rs2;
; CHECK-SM70-NEXT: shl.b32 %r6, %r5, 16;
; CHECK-SM70-NEXT: fma.rn.f32 %r7, %r6, %r4, %r2;
-; CHECK-SM70-NEXT: bfe.u32 %r8, %r7, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r9, %r8, %r7;
-; CHECK-SM70-NEXT: add.s32 %r10, %r9, 32767;
+; CHECK-SM70-NEXT: shr.u32 %r8, %r7, 16;
+; CHECK-SM70-NEXT: and.b32 %r9, %r8, 1;
+; CHECK-SM70-NEXT: add.s32 %r10, %r9, %r7;
+; CHECK-SM70-NEXT: add.s32 %r11, %r10, 32767;
; CHECK-SM70-NEXT: setp.nan.f32 %p1, %r7, %r7;
-; CHECK-SM70-NEXT: or.b32 %r11, %r7, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r12, %r11, %r10, %p1;
-; CHECK-SM70-NEXT: cvt.u32.u16 %r13, %rs3;
-; CHECK-SM70-NEXT: shl.b32 %r14, %r13, 16;
-; CHECK-SM70-NEXT: cvt.u32.u16 %r15, %rs5;
-; CHECK-SM70-NEXT: shl.b32 %r16, %r15, 16;
-; CHECK-SM70-NEXT: cvt.u32.u16 %r17, %rs1;
-; CHECK-SM70-NEXT: shl.b32 %r18, %r17, 16;
-; CHECK-SM70-NEXT: fma.rn.f32 %r19, %r18, %r16, %r14;
-; CHECK-SM70-NEXT: bfe.u32 %r20, %r19, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r21, %r20, %r19;
-; CHECK-SM70-NEXT: add.s32 %r22, %r21, 32767;
-; CHECK-SM70-NEXT: setp.nan.f32 %p2, %r19, %r19;
-; CHECK-SM70-NEXT: or.b32 %r23, %r19, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r24, %r23, %r22, %p2;
-; CHECK-SM70-NEXT: and.b32 %r25, %r24, -65536;
-; CHECK-SM70-NEXT: add.f32 %r26, %r25, 0f40E00000;
-; CHECK-SM70-NEXT: bfe.u32 %r27, %r26, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r28, %r27, %r26;
-; CHECK-SM70-NEXT: add.s32 %r29, %r28, 32767;
-; CHECK-SM70-NEXT: setp.nan.f32 %p3, %r26, %r26;
-; CHECK-SM70-NEXT: or.b32 %r30, %r26, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r31, %r30, %r29, %p3;
-; CHECK-SM70-NEXT: and.b32 %r32, %r12, -65536;
-; CHECK-SM70-NEXT: add.f32 %r33, %r32, 0f40E00000;
-; CHECK-SM70-NEXT: bfe.u32 %r34, %r33, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r35, %r34, %r33;
-; CHECK-SM70-NEXT: add.s32 %r36, %r35, 32767;
-; CHECK-SM70-NEXT: setp.nan.f32 %p4, %r33, %r33;
-; CHECK-SM70-NEXT: or.b32 %r37, %r33, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r38, %r37, %r36, %p4;
-; CHECK-SM70-NEXT: and.b32 %r39, %r38, -65536;
-; CHECK-SM70-NEXT: add.f32 %r40, %r39, %r32;
-; CHECK-SM70-NEXT: bfe.u32 %r41, %r40, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r42, %r41, %r40;
-; CHECK-SM70-NEXT: add.s32 %r43, %r42, 32767;
-; CHECK-SM70-NEXT: setp.nan.f32 %p5, %r40, %r40;
-; CHECK-SM70-NEXT: or.b32 %r44, %r40, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r45, %r44, %r43, %p5;
-; CHECK-SM70-NEXT: and.b32 %r46, %r31, -65536;
-; CHECK-SM70-NEXT: add.f32 %r47, %r46, %r25;
-; CHECK-SM70-NEXT: bfe.u32 %r48, %r47, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r49, %r48, %r47;
-; CHECK-SM70-NEXT: add.s32 %r50, %r49, 32767;
-; CHECK-SM70-NEXT: setp.nan.f32 %p6, %r47, %r47;
-; CHECK-SM70-NEXT: or.b32 %r51, %r47, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r52, %r51, %r50, %p6;
-; CHECK-SM70-NEXT: prmt.b32 %r53, %r52, %r45, 0x7632U;
-; CHECK-SM70-NEXT: st.param.b32 [func_retval0], %r53;
+; CHECK-SM70-NEXT: or.b32 %r12, %r7, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r13, %r12, %r11, %p1;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r14, %rs3;
+; CHECK-SM70-NEXT: shl.b32 %r15, %r14, 16;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r16, %rs5;
+; CHECK-SM70-NEXT: shl.b32 %r17, %r16, 16;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r18, %rs1;
+; CHECK-SM70-NEXT: shl.b32 %r19, %r18, 16;
+; CHECK-SM70-NEXT: fma.rn.f32 %r20, %r19, %r17, %r15;
+; CHECK-SM70-NEXT: shr.u32 %r21, %r20, 16;
+; CHECK-SM70-NEXT: and.b32 %r22, %r21, 1;
+; CHECK-SM70-NEXT: add.s32 %r23, %r22, %r20;
+; CHECK-SM70-NEXT: add.s32 %r24, %r23, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p2, %r20, %r20;
+; CHECK-SM70-NEXT: or.b32 %r25, %r20, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r26, %r25, %r24, %p2;
+; CHECK-SM70-NEXT: and.b32 %r27, %r26, -65536;
+; CHECK-SM70-NEXT: add.f32 %r28, %r27, 0f40E00000;
+; CHECK-SM70-NEXT: shr.u32 %r29, %r28, 16;
+; CHECK-SM70-NEXT: and.b32 %r30, %r29, 1;
+; CHECK-SM70-NEXT: add.s32 %r31, %r30, %r28;
+; CHECK-SM70-NEXT: add.s32 %r32, %r31, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p3, %r28, %r28;
+; CHECK-SM70-NEXT: or.b32 %r33, %r28, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r34, %r33, %r32, %p3;
+; CHECK-SM70-NEXT: and.b32 %r35, %r13, -65536;
+; CHECK-SM70-NEXT: add.f32 %r36, %r35, 0f40E00000;
+; CHECK-SM70-NEXT: shr.u32 %r37, %r36, 16;
+; CHECK-SM70-NEXT: and.b32 %r38, %r37, 1;
+; CHECK-SM70-NEXT: add.s32 %r39, %r38, %r36;
+; CHECK-SM70-NEXT: add.s32 %r40, %r39, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p4, %r36, %r36;
+; CHECK-SM70-NEXT: or.b32 %r41, %r36, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r42, %r41, %r40, %p4;
+; CHECK-SM70-NEXT: and.b32 %r43, %r42, -65536;
+; CHECK-SM70-NEXT: add.f32 %r44, %r43, %r35;
+; CHECK-SM70-NEXT: shr.u32 %r45, %r44, 16;
+; CHECK-SM70-NEXT: and.b32 %r46, %r45, 1;
+; CHECK-SM70-NEXT: add.s32 %r47, %r46, %r44;
+; CHECK-SM70-NEXT: add.s32 %r48, %r47, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p5, %r44, %r44;
+; CHECK-SM70-NEXT: or.b32 %r49, %r44, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r50, %r49, %r48, %p5;
+; CHECK-SM70-NEXT: and.b32 %r51, %r34, -65536;
+; CHECK-SM70-NEXT: add.f32 %r52, %r51, %r27;
+; CHECK-SM70-NEXT: shr.u32 %r53, %r52, 16;
+; CHECK-SM70-NEXT: and.b32 %r54, %r53, 1;
+; CHECK-SM70-NEXT: add.s32 %r55, %r54, %r52;
+; CHECK-SM70-NEXT: add.s32 %r56, %r55, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p6, %r52, %r52;
+; CHECK-SM70-NEXT: or.b32 %r57, %r52, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r58, %r57, %r56, %p6;
+; CHECK-SM70-NEXT: prmt.b32 %r59, %r58, %r50, 0x7632U;
+; CHECK-SM70-NEXT: st.param.b32 [func_retval0], %r59;
; CHECK-SM70-NEXT: ret;
%1 = call <2 x bfloat> @llvm.fma.bf16x2(<2 x bfloat> %a, <2 x bfloat> %b, <2 x bfloat> %c)
%2 = fcmp ogt <2 x bfloat> %1, <bfloat 0.0, bfloat 0.0>
@@ -741,7 +755,7 @@ define <2 x bfloat> @fma_bf16x2_maxnum_no_nans(<2 x bfloat> %a, <2 x bfloat> %b,
; CHECK-SM70: {
; CHECK-SM70-NEXT: .reg .pred %p<5>;
; CHECK-SM70-NEXT: .reg .b16 %rs<7>;
-; CHECK-SM70-NEXT: .reg .b32 %r<40>;
+; CHECK-SM70-NEXT: .reg .b32 %r<44>;
; CHECK-SM70-EMPTY:
; CHECK-SM70-NEXT: // %bb.0:
; CHECK-SM70-NEXT: ld.param.v2.b16 {%rs1, %rs2}, [fma_bf16x2_maxnum_no_nans_param_0];
@@ -754,43 +768,47 @@ define <2 x bfloat> @fma_bf16x2_maxnum_no_nans(<2 x bfloat> %a, <2 x bfloat> %b,
; CHECK-SM70-NEXT: cvt.u32.u16 %r5, %rs1;
; CHECK-SM70-NEXT: shl.b32 %r6, %r5, 16;
; CHECK-SM70-NEXT: fma.rn.f32 %r7, %r6, %r4, %r2;
-; CHECK-SM70-NEXT: bfe.u32 %r8, %r7, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r9, %r8, %r7;
-; CHECK-SM70-NEXT: add.s32 %r10, %r9, 32767;
+; CHECK-SM70-NEXT: shr.u32 %r8, %r7, 16;
+; CHECK-SM70-NEXT: and.b32 %r9, %r8, 1;
+; CHECK-SM70-NEXT: add.s32 %r10, %r9, %r7;
+; CHECK-SM70-NEXT: add.s32 %r11, %r10, 32767;
; CHECK-SM70-NEXT: setp.nan.f32 %p1, %r7, %r7;
-; CHECK-SM70-NEXT: or.b32 %r11, %r7, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r12, %r11, %r10, %p1;
-; CHECK-SM70-NEXT: cvt.u32.u16 %r13, %rs4;
-; CHECK-SM70-NEXT: shl.b32 %r14, %r13, 16;
-; CHECK-SM70-NEXT: cvt.u32.u16 %r15, %rs6;
-; CHECK-SM70-NEXT: shl.b32 %r16, %r15, 16;
-; CHECK-SM70-NEXT: cvt.u32.u16 %r17, %rs2;
-; CHECK-SM70-NEXT: shl.b32 %r18, %r17, 16;
-; CHECK-SM70-NEXT: fma.rn.f32 %r19, %r18, %r16, %r14;
-; CHECK-SM70-NEXT: bfe.u32 %r20, %r19, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r21, %r20, %r19;
-; CHECK-SM70-NEXT: add.s32 %r22, %r21, 32767;
-; CHECK-SM70-NEXT: setp.nan.f32 %p2, %r19, %r19;
-; CHECK-SM70-NEXT: or.b32 %r23, %r19, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r24, %r23, %r22, %p2;
-; CHECK-SM70-NEXT: and.b32 %r25, %r24, -65536;
-; CHECK-SM70-NEXT: max.f32 %r26, %r25, 0f00000000;
-; CHECK-SM70-NEXT: bfe.u32 %r27, %r26, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r28, %r27, %r26;
-; CHECK-SM70-NEXT: add.s32 %r29, %r28, 32767;
-; CHECK-SM70-NEXT: setp.nan.f32 %p3, %r26, %r26;
-; CHECK-SM70-NEXT: or.b32 %r30, %r26, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r31, %r30, %r29, %p3;
-; CHECK-SM70-NEXT: and.b32 %r32, %r12, -65536;
-; CHECK-SM70-NEXT: max.f32 %r33, %r32, 0f00000000;
-; CHECK-SM70-NEXT: bfe.u32 %r34, %r33, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r35, %r34, %r33;
-; CHECK-SM70-NEXT: add.s32 %r36, %r35, 32767;
-; CHECK-SM70-NEXT: setp.nan.f32 %p4, %r33, %r33;
-; CHECK-SM70-NEXT: or.b32 %r37, %r33, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r38, %r37, %r36, %p4;
-; CHECK-SM70-NEXT: prmt.b32 %r39, %r38, %r31, 0x7632U;
-; CHECK-SM70-NEXT: st.param.b32 [func_retval0], %r39;
+; CHECK-SM70-NEXT: or.b32 %r12, %r7, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r13, %r12, %r11, %p1;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r14, %rs4;
+; CHECK-SM70-NEXT: shl.b32 %r15, %r14, 16;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r16, %rs6;
+; CHECK-SM70-NEXT: shl.b32 %r17, %r16, 16;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r18, %rs2;
+; CHECK-SM70-NEXT: shl.b32 %r19, %r18, 16;
+; CHECK-SM70-NEXT: fma.rn.f32 %r20, %r19, %r17, %r15;
+; CHECK-SM70-NEXT: shr.u32 %r21, %r20, 16;
+; CHECK-SM70-NEXT: and.b32 %r22, %r21, 1;
+; CHECK-SM70-NEXT: add.s32 %r23, %r22, %r20;
+; CHECK-SM70-NEXT: add.s32 %r24, %r23, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p2, %r20, %r20;
+; CHECK-SM70-NEXT: or.b32 %r25, %r20, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r26, %r25, %r24, %p2;
+; CHECK-SM70-NEXT: and.b32 %r27, %r26, -65536;
+; CHECK-SM70-NEXT: max.f32 %r28, %r27, 0f00000000;
+; CHECK-SM70-NEXT: shr.u32 %r29, %r28, 16;
+; CHECK-SM70-NEXT: and.b32 %r30, %r29, 1;
+; CHECK-SM70-NEXT: add.s32 %r31, %r30, %r28;
+; CHECK-SM70-NEXT: add.s32 %r32, %r31, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p3, %r28, %r28;
+; CHECK-SM70-NEXT: or.b32 %r33, %r28, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r34, %r33, %r32, %p3;
+; CHECK-SM70-NEXT: and.b32 %r35, %r13, -65536;
+; CHECK-SM70-NEXT: max.f32 %r36, %r35, 0f00000000;
+; CHECK-SM70-NEXT: shr.u32 %r37, %r36, 16;
+; CHECK-SM70-NEXT: and.b32 %r38, %r37, 1;
+; CHECK-SM70-NEXT: add.s32 %r39, %r38, %r36;
+; CHECK-SM70-NEXT: add.s32 %r40, %r39, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p4, %r36, %r36;
+; CHECK-SM70-NEXT: or.b32 %r41, %r36, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r42, %r41, %r40, %p4;
+; CHECK-SM70-NEXT: prmt.b32 %r43, %r42, %r34, 0x7632U;
+; CHECK-SM70-NEXT: st.param.b32 [func_retval0], %r43;
; CHECK-SM70-NEXT: ret;
%1 = call nnan <2 x bfloat> @llvm.fma.bf16x2(<2 x bfloat> %a, <2 x bfloat> %b, <2 x bfloat> %c)
%2 = call nsz <2 x bfloat> @llvm.maxnum.bf16x2(<2 x bfloat> %1, <2 x bfloat> <bfloat 0.0, bfloat 0.0>)
diff --git a/llvm/test/CodeGen/NVPTX/fma-relu-instruction-flag.ll b/llvm/test/CodeGen/NVPTX/fma-relu-instruction-flag.ll
index 090dd00f1b933..eb0b6826d521b 100644
--- a/llvm/test/CodeGen/NVPTX/fma-relu-instruction-flag.ll
+++ b/llvm/test/CodeGen/NVPTX/fma-relu-instruction-flag.ll
@@ -194,7 +194,7 @@ define bfloat @fma_bf16_expanded_no_nans(bfloat %a, bfloat %b, bfloat %c) {
; CHECK-SM70: {
; CHECK-SM70-NEXT: .reg .pred %p<3>;
; CHECK-SM70-NEXT: .reg .b16 %rs<3>;
-; CHECK-SM70-NEXT: .reg .b32 %r<14>;
+; CHECK-SM70-NEXT: .reg .b32 %r<15>;
; CHECK-SM70-EMPTY:
; CHECK-SM70-NEXT: // %bb.0:
; CHECK-SM70-NEXT: ld.param.b16 %r1, [fma_bf16_expanded_no_nans_param_2];
@@ -204,15 +204,16 @@ define bfloat @fma_bf16_expanded_no_nans(bfloat %a, bfloat %b, bfloat %c) {
; CHECK-SM70-NEXT: ld.param.b16 %r5, [fma_bf16_expanded_no_nans_param_0];
; CHECK-SM70-NEXT: shl.b32 %r6, %r5, 16;
; CHECK-SM70-NEXT: fma.rn.f32 %r7, %r6, %r4, %r2;
-; CHECK-SM70-NEXT: bfe.u32 %r8, %r7, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r9, %r8, %r7;
-; CHECK-SM70-NEXT: add.s32 %r10, %r9, 32767;
+; CHECK-SM70-NEXT: shr.u32 %r8, %r7, 16;
+; CHECK-SM70-NEXT: and.b32 %r9, %r8, 1;
+; CHECK-SM70-NEXT: add.s32 %r10, %r9, %r7;
+; CHECK-SM70-NEXT: add.s32 %r11, %r10, 32767;
; CHECK-SM70-NEXT: setp.nan.f32 %p1, %r7, %r7;
-; CHECK-SM70-NEXT: or.b32 %r11, %r7, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r12, %r11, %r10, %p1;
-; CHECK-SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs1}, %r12; }
-; CHECK-SM70-NEXT: and.b32 %r13, %r12, -65536;
-; CHECK-SM70-NEXT: setp.gt.f32 %p2, %r13, 0f00000000;
+; CHECK-SM70-NEXT: or.b32 %r12, %r7, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r13, %r12, %r11, %p1;
+; CHECK-SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs1}, %r13; }
+; CHECK-SM70-NEXT: and.b32 %r14, %r13, -65536;
+; CHECK-SM70-NEXT: setp.gt.f32 %p2, %r14, 0f00000000;
; CHECK-SM70-NEXT: selp.b16 %rs2, %rs1, 0x0000, %p2;
; CHECK-SM70-NEXT: st.param.b16 [func_retval0], %rs2;
; CHECK-SM70-NEXT: ret;
@@ -272,7 +273,7 @@ define bfloat @fma_bf16_expanded_no_nans_multiple_uses_of_fma(bfloat %a, bfloat
; CHECK-SM70: {
; CHECK-SM70-NEXT: .reg .pred %p<5>;
; CHECK-SM70-NEXT: .reg .b16 %rs<3>;
-; CHECK-SM70-NEXT: .reg .b32 %r<30>;
+; CHECK-SM70-NEXT: .reg .b32 %r<33>;
; CHECK-SM70-EMPTY:
; CHECK-SM70-NEXT: // %bb.0:
; CHECK-SM70-NEXT: ld.param.b16 %r1, [fma_bf16_expanded_no_nans_multiple_uses_of_fma_param_2];
@@ -282,35 +283,38 @@ define bfloat @fma_bf16_expanded_no_nans_multiple_uses_of_fma(bfloat %a, bfloat
; CHECK-SM70-NEXT: ld.param.b16 %r5, [fma_bf16_expanded_no_nans_multiple_uses_of_fma_param_0];
; CHECK-SM70-NEXT: shl.b32 %r6, %r5, 16;
; CHECK-SM70-NEXT: fma.rn.f32 %r7, %r6, %r4, %r2;
-; CHECK-SM70-NEXT: bfe.u32 %r8, %r7, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r9, %r8, %r7;
-; CHECK-SM70-NEXT: add.s32 %r10, %r9, 32767;
+; CHECK-SM70-NEXT: shr.u32 %r8, %r7, 16;
+; CHECK-SM70-NEXT: and.b32 %r9, %r8, 1;
+; CHECK-SM70-NEXT: add.s32 %r10, %r9, %r7;
+; CHECK-SM70-NEXT: add.s32 %r11, %r10, 32767;
; CHECK-SM70-NEXT: setp.nan.f32 %p1, %r7, %r7;
-; CHECK-SM70-NEXT: or.b32 %r11, %r7, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r12, %r11, %r10, %p1;
-; CHECK-SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs1}, %r12; }
-; CHECK-SM70-NEXT: and.b32 %r13, %r12, -65536;
-; CHECK-SM70-NEXT: setp.gt.f32 %p2, %r13, 0f00000000;
+; CHECK-SM70-NEXT: or.b32 %r12, %r7, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r13, %r12, %r11, %p1;
+; CHECK-SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs1}, %r13; }
+; CHECK-SM70-NEXT: and.b32 %r14, %r13, -65536;
+; CHECK-SM70-NEXT: setp.gt.f32 %p2, %r14, 0f00000000;
; CHECK-SM70-NEXT: selp.b16 %rs2, %rs1, 0x0000, %p2;
-; CHECK-SM70-NEXT: add.rn.f32 %r14, %r13, 0f40E00000;
-; CHECK-SM70-NEXT: bfe.u32 %r15, %r14, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r16, %r15, %r14;
-; CHECK-SM70-NEXT: add.s32 %r17, %r16, 32767;
-; CHECK-SM70-NEXT: setp.nan.f32 %p3, %r14, %r14;
-; CHECK-SM70-NEXT: or.b32 %r18, %r14, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r19, %r18, %r17, %p3;
-; CHECK-SM70-NEXT: cvt.u32.u16 %r20, %rs2;
-; CHECK-SM70-NEXT: shl.b32 %r21, %r20, 16;
-; CHECK-SM70-NEXT: and.b32 %r22, %r19, -65536;
-; CHECK-SM70-NEXT: add.rn.f32 %r23, %r21, %r22;
-; CHECK-SM70-NEXT: bfe.u32 %r24, %r23, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r25, %r24, %r23;
-; CHECK-SM70-NEXT: add.s32 %r26, %r25, 32767;
-; CHECK-SM70-NEXT: setp.nan.f32 %p4, %r23, %r23;
-; CHECK-SM70-NEXT: or.b32 %r27, %r23, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r28, %r27, %r26, %p4;
-; CHECK-SM70-NEXT: shr.u32 %r29, %r28, 16;
-; CHECK-SM70-NEXT: st.param.b16 [func_retval0], %r29;
+; CHECK-SM70-NEXT: add.rn.f32 %r15, %r14, 0f40E00000;
+; CHECK-SM70-NEXT: shr.u32 %r16, %r15, 16;
+; CHECK-SM70-NEXT: and.b32 %r17, %r16, 1;
+; CHECK-SM70-NEXT: add.s32 %r18, %r17, %r15;
+; CHECK-SM70-NEXT: add.s32 %r19, %r18, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p3, %r15, %r15;
+; CHECK-SM70-NEXT: or.b32 %r20, %r15, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r21, %r20, %r19, %p3;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r22, %rs2;
+; CHECK-SM70-NEXT: shl.b32 %r23, %r22, 16;
+; CHECK-SM70-NEXT: and.b32 %r24, %r21, -65536;
+; CHECK-SM70-NEXT: add.rn.f32 %r25, %r23, %r24;
+; CHECK-SM70-NEXT: shr.u32 %r26, %r25, 16;
+; CHECK-SM70-NEXT: and.b32 %r27, %r26, 1;
+; CHECK-SM70-NEXT: add.s32 %r28, %r27, %r25;
+; CHECK-SM70-NEXT: add.s32 %r29, %r28, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p4, %r25, %r25;
+; CHECK-SM70-NEXT: or.b32 %r30, %r25, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r31, %r30, %r29, %p4;
+; CHECK-SM70-NEXT: shr.u32 %r32, %r31, 16;
+; CHECK-SM70-NEXT: st.param.b16 [func_retval0], %r32;
; CHECK-SM70-NEXT: ret;
%1 = fmul fast bfloat %a, %b
%2 = fadd fast bfloat %1, %c
@@ -351,7 +355,7 @@ define bfloat @fma_bf16_expanded_maxnum_no_nans(bfloat %a, bfloat %b, bfloat %c)
; CHECK-SM70-LABEL: fma_bf16_expanded_maxnum_no_nans(
; CHECK-SM70: {
; CHECK-SM70-NEXT: .reg .pred %p<3>;
-; CHECK-SM70-NEXT: .reg .b32 %r<21>;
+; CHECK-SM70-NEXT: .reg .b32 %r<23>;
; CHECK-SM70-EMPTY:
; CHECK-SM70-NEXT: // %bb.0:
; CHECK-SM70-NEXT: ld.param.b16 %r1, [fma_bf16_expanded_maxnum_no_nans_param_2];
@@ -361,22 +365,24 @@ define bfloat @fma_bf16_expanded_maxnum_no_nans(bfloat %a, bfloat %b, bfloat %c)
; CHECK-SM70-NEXT: ld.param.b16 %r5, [fma_bf16_expanded_maxnum_no_nans_param_0];
; CHECK-SM70-NEXT: shl.b32 %r6, %r5, 16;
; CHECK-SM70-NEXT: fma.rn.f32 %r7, %r6, %r4, %r2;
-; CHECK-SM70-NEXT: bfe.u32 %r8, %r7, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r9, %r8, %r7;
-; CHECK-SM70-NEXT: add.s32 %r10, %r9, 32767;
+; CHECK-SM70-NEXT: shr.u32 %r8, %r7, 16;
+; CHECK-SM70-NEXT: and.b32 %r9, %r8, 1;
+; CHECK-SM70-NEXT: add.s32 %r10, %r9, %r7;
+; CHECK-SM70-NEXT: add.s32 %r11, %r10, 32767;
; CHECK-SM70-NEXT: setp.nan.f32 %p1, %r7, %r7;
-; CHECK-SM70-NEXT: or.b32 %r11, %r7, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r12, %r11, %r10, %p1;
-; CHECK-SM70-NEXT: and.b32 %r13, %r12, -65536;
-; CHECK-SM70-NEXT: max.f32 %r14, %r13, 0f00000000;
-; CHECK-SM70-NEXT: bfe.u32 %r15, %r14, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r16, %r15, %r14;
-; CHECK-SM70-NEXT: add.s32 %r17, %r16, 32767;
-; CHECK-SM70-NEXT: setp.nan.f32 %p2, %r14, %r14;
-; CHECK-SM70-NEXT: or.b32 %r18, %r14, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r19, %r18, %r17, %p2;
-; CHECK-SM70-NEXT: shr.u32 %r20, %r19, 16;
-; CHECK-SM70-NEXT: st.param.b16 [func_retval0], %r20;
+; CHECK-SM70-NEXT: or.b32 %r12, %r7, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r13, %r12, %r11, %p1;
+; CHECK-SM70-NEXT: and.b32 %r14, %r13, -65536;
+; CHECK-SM70-NEXT: max.f32 %r15, %r14, 0f00000000;
+; CHECK-SM70-NEXT: shr.u32 %r16, %r15, 16;
+; CHECK-SM70-NEXT: and.b32 %r17, %r16, 1;
+; CHECK-SM70-NEXT: add.s32 %r18, %r17, %r15;
+; CHECK-SM70-NEXT: add.s32 %r19, %r18, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p2, %r15, %r15;
+; CHECK-SM70-NEXT: or.b32 %r20, %r15, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r21, %r20, %r19, %p2;
+; CHECK-SM70-NEXT: shr.u32 %r22, %r21, 16;
+; CHECK-SM70-NEXT: st.param.b16 [func_retval0], %r22;
; CHECK-SM70-NEXT: ret;
%1 = fmul fast bfloat %a, %b
%2 = fadd fast bfloat %1, %c
@@ -585,7 +591,7 @@ define <2 x bfloat> @fma_bf16x2_expanded_no_nans(<2 x bfloat> %a, <2 x bfloat> %
; CHECK-SM70: {
; CHECK-SM70-NEXT: .reg .pred %p<5>;
; CHECK-SM70-NEXT: .reg .b16 %rs<11>;
-; CHECK-SM70-NEXT: .reg .b32 %r<27>;
+; CHECK-SM70-NEXT: .reg .b32 %r<29>;
; CHECK-SM70-EMPTY:
; CHECK-SM70-NEXT: // %bb.0:
; CHECK-SM70-NEXT: ld.param.v2.b16 {%rs1, %rs2}, [fma_bf16x2_expanded_no_nans_param_0];
@@ -598,31 +604,33 @@ define <2 x bfloat> @fma_bf16x2_expanded_no_nans(<2 x bfloat> %a, <2 x bfloat> %
; CHECK-SM70-NEXT: cvt.u32.u16 %r5, %rs1;
; CHECK-SM70-NEXT: shl.b32 %r6, %r5, 16;
; CHECK-SM70-NEXT: fma.rn.f32 %r7, %r6, %r4, %r2;
-; CHECK-SM70-NEXT: bfe.u32 %r8, %r7, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r9, %r8, %r7;
-; CHECK-SM70-NEXT: add.s32 %r10, %r9, 32767;
+; CHECK-SM70-NEXT: shr.u32 %r8, %r7, 16;
+; CHECK-SM70-NEXT: and.b32 %r9, %r8, 1;
+; CHECK-SM70-NEXT: add.s32 %r10, %r9, %r7;
+; CHECK-SM70-NEXT: add.s32 %r11, %r10, 32767;
; CHECK-SM70-NEXT: setp.nan.f32 %p1, %r7, %r7;
-; CHECK-SM70-NEXT: or.b32 %r11, %r7, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r12, %r11, %r10, %p1;
-; CHECK-SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs7}, %r12; }
-; CHECK-SM70-NEXT: cvt.u32.u16 %r13, %rs6;
-; CHECK-SM70-NEXT: shl.b32 %r14, %r13, 16;
-; CHECK-SM70-NEXT: cvt.u32.u16 %r15, %rs4;
-; CHECK-SM70-NEXT: shl.b32 %r16, %r15, 16;
-; CHECK-SM70-NEXT: cvt.u32.u16 %r17, %rs2;
-; CHECK-SM70-NEXT: shl.b32 %r18, %r17, 16;
-; CHECK-SM70-NEXT: fma.rn.f32 %r19, %r18, %r16, %r14;
-; CHECK-SM70-NEXT: bfe.u32 %r20, %r19, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r21, %r20, %r19;
-; CHECK-SM70-NEXT: add.s32 %r22, %r21, 32767;
-; CHECK-SM70-NEXT: setp.nan.f32 %p2, %r19, %r19;
-; CHECK-SM70-NEXT: or.b32 %r23, %r19, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r24, %r23, %r22, %p2;
-; CHECK-SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs8}, %r24; }
-; CHECK-SM70-NEXT: and.b32 %r25, %r12, -65536;
-; CHECK-SM70-NEXT: setp.gt.f32 %p3, %r25, 0f00000000;
-; CHECK-SM70-NEXT: and.b32 %r26, %r24, -65536;
-; CHECK-SM70-NEXT: setp.gt.f32 %p4, %r26, 0f00000000;
+; CHECK-SM70-NEXT: or.b32 %r12, %r7, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r13, %r12, %r11, %p1;
+; CHECK-SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs7}, %r13; }
+; CHECK-SM70-NEXT: cvt.u32.u16 %r14, %rs6;
+; CHECK-SM70-NEXT: shl.b32 %r15, %r14, 16;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r16, %rs4;
+; CHECK-SM70-NEXT: shl.b32 %r17, %r16, 16;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r18, %rs2;
+; CHECK-SM70-NEXT: shl.b32 %r19, %r18, 16;
+; CHECK-SM70-NEXT: fma.rn.f32 %r20, %r19, %r17, %r15;
+; CHECK-SM70-NEXT: shr.u32 %r21, %r20, 16;
+; CHECK-SM70-NEXT: and.b32 %r22, %r21, 1;
+; CHECK-SM70-NEXT: add.s32 %r23, %r22, %r20;
+; CHECK-SM70-NEXT: add.s32 %r24, %r23, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p2, %r20, %r20;
+; CHECK-SM70-NEXT: or.b32 %r25, %r20, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r26, %r25, %r24, %p2;
+; CHECK-SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs8}, %r26; }
+; CHECK-SM70-NEXT: and.b32 %r27, %r13, -65536;
+; CHECK-SM70-NEXT: setp.gt.f32 %p3, %r27, 0f00000000;
+; CHECK-SM70-NEXT: and.b32 %r28, %r26, -65536;
+; CHECK-SM70-NEXT: setp.gt.f32 %p4, %r28, 0f00000000;
; CHECK-SM70-NEXT: selp.b16 %rs9, %rs8, 0x0000, %p4;
; CHECK-SM70-NEXT: selp.b16 %rs10, %rs7, 0x0000, %p3;
; CHECK-SM70-NEXT: st.param.v2.b16 [func_retval0], {%rs10, %rs9};
@@ -694,7 +702,7 @@ define <2 x bfloat> @fma_bf16x2_expanded_no_nans_multiple_uses_of_fma(<2 x bfloa
; CHECK-SM70: {
; CHECK-SM70-NEXT: .reg .pred %p<9>;
; CHECK-SM70-NEXT: .reg .b16 %rs<11>;
-; CHECK-SM70-NEXT: .reg .b32 %r<58>;
+; CHECK-SM70-NEXT: .reg .b32 %r<64>;
; CHECK-SM70-EMPTY:
; CHECK-SM70-NEXT: // %bb.0:
; CHECK-SM70-NEXT: ld.param.v2.b16 {%rs1, %rs2}, [fma_bf16x2_expanded_no_nans_multiple_uses_of_fma_param_0];
@@ -707,69 +715,75 @@ define <2 x bfloat> @fma_bf16x2_expanded_no_nans_multiple_uses_of_fma(<2 x bfloa
; CHECK-SM70-NEXT: cvt.u32.u16 %r5, %rs2;
; CHECK-SM70-NEXT: shl.b32 %r6, %r5, 16;
; CHECK-SM70-NEXT: fma.rn.f32 %r7, %r6, %r4, %r2;
-; CHECK-SM70-NEXT: bfe.u32 %r8, %r7, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r9, %r8, %r7;
-; CHECK-SM70-NEXT: add.s32 %r10, %r9, 32767;
+; CHECK-SM70-NEXT: shr.u32 %r8, %r7, 16;
+; CHECK-SM70-NEXT: and.b32 %r9, %r8, 1;
+; CHECK-SM70-NEXT: add.s32 %r10, %r9, %r7;
+; CHECK-SM70-NEXT: add.s32 %r11, %r10, 32767;
; CHECK-SM70-NEXT: setp.nan.f32 %p1, %r7, %r7;
-; CHECK-SM70-NEXT: or.b32 %r11, %r7, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r12, %r11, %r10, %p1;
-; CHECK-SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs7}, %r12; }
-; CHECK-SM70-NEXT: cvt.u32.u16 %r13, %rs5;
-; CHECK-SM70-NEXT: shl.b32 %r14, %r13, 16;
-; CHECK-SM70-NEXT: cvt.u32.u16 %r15, %rs3;
-; CHECK-SM70-NEXT: shl.b32 %r16, %r15, 16;
-; CHECK-SM70-NEXT: cvt.u32.u16 %r17, %rs1;
-; CHECK-SM70-NEXT: shl.b32 %r18, %r17, 16;
-; CHECK-SM70-NEXT: fma.rn.f32 %r19, %r18, %r16, %r14;
-; CHECK-SM70-NEXT: bfe.u32 %r20, %r19, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r21, %r20, %r19;
-; CHECK-SM70-NEXT: add.s32 %r22, %r21, 32767;
-; CHECK-SM70-NEXT: setp.nan.f32 %p2, %r19, %r19;
-; CHECK-SM70-NEXT: or.b32 %r23, %r19, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r24, %r23, %r22, %p2;
-; CHECK-SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs8}, %r24; }
-; CHECK-SM70-NEXT: and.b32 %r25, %r12, -65536;
-; CHECK-SM70-NEXT: setp.gt.f32 %p3, %r25, 0f00000000;
-; CHECK-SM70-NEXT: and.b32 %r26, %r24, -65536;
-; CHECK-SM70-NEXT: setp.gt.f32 %p4, %r26, 0f00000000;
+; CHECK-SM70-NEXT: or.b32 %r12, %r7, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r13, %r12, %r11, %p1;
+; CHECK-SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs7}, %r13; }
+; CHECK-SM70-NEXT: cvt.u32.u16 %r14, %rs5;
+; CHECK-SM70-NEXT: shl.b32 %r15, %r14, 16;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r16, %rs3;
+; CHECK-SM70-NEXT: shl.b32 %r17, %r16, 16;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r18, %rs1;
+; CHECK-SM70-NEXT: shl.b32 %r19, %r18, 16;
+; CHECK-SM70-NEXT: fma.rn.f32 %r20, %r19, %r17, %r15;
+; CHECK-SM70-NEXT: shr.u32 %r21, %r20, 16;
+; CHECK-SM70-NEXT: and.b32 %r22, %r21, 1;
+; CHECK-SM70-NEXT: add.s32 %r23, %r22, %r20;
+; CHECK-SM70-NEXT: add.s32 %r24, %r23, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p2, %r20, %r20;
+; CHECK-SM70-NEXT: or.b32 %r25, %r20, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r26, %r25, %r24, %p2;
+; CHECK-SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs8}, %r26; }
+; CHECK-SM70-NEXT: and.b32 %r27, %r13, -65536;
+; CHECK-SM70-NEXT: setp.gt.f32 %p3, %r27, 0f00000000;
+; CHECK-SM70-NEXT: and.b32 %r28, %r26, -65536;
+; CHECK-SM70-NEXT: setp.gt.f32 %p4, %r28, 0f00000000;
; CHECK-SM70-NEXT: selp.b16 %rs9, %rs8, 0x0000, %p4;
; CHECK-SM70-NEXT: selp.b16 %rs10, %rs7, 0x0000, %p3;
-; CHECK-SM70-NEXT: add.rn.f32 %r27, %r26, 0f40E00000;
-; CHECK-SM70-NEXT: bfe.u32 %r28, %r27, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r29, %r28, %r27;
-; CHECK-SM70-NEXT: add.s32 %r30, %r29, 32767;
-; CHECK-SM70-NEXT: setp.nan.f32 %p5, %r27, %r27;
-; CHECK-SM70-NEXT: or.b32 %r31, %r27, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r32, %r31, %r30, %p5;
-; CHECK-SM70-NEXT: add.rn.f32 %r33, %r25, 0f40E00000;
-; CHECK-SM70-NEXT: bfe.u32 %r34, %r33, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r35, %r34, %r33;
-; CHECK-SM70-NEXT: add.s32 %r36, %r35, 32767;
-; CHECK-SM70-NEXT: setp.nan.f32 %p6, %r33, %r33;
-; CHECK-SM70-NEXT: or.b32 %r37, %r33, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r38, %r37, %r36, %p6;
-; CHECK-SM70-NEXT: cvt.u32.u16 %r39, %rs10;
-; CHECK-SM70-NEXT: shl.b32 %r40, %r39, 16;
-; CHECK-SM70-NEXT: and.b32 %r41, %r38, -65536;
-; CHECK-SM70-NEXT: add.rn.f32 %r42, %r40, %r41;
-; CHECK-SM70-NEXT: bfe.u32 %r43, %r42, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r44, %r43, %r42;
-; CHECK-SM70-NEXT: add.s32 %r45, %r44, 32767;
-; CHECK-SM70-NEXT: setp.nan.f32 %p7, %r42, %r42;
-; CHECK-SM70-NEXT: or.b32 %r46, %r42, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r47, %r46, %r45, %p7;
-; CHECK-SM70-NEXT: cvt.u32.u16 %r48, %rs9;
-; CHECK-SM70-NEXT: shl.b32 %r49, %r48, 16;
-; CHECK-SM70-NEXT: and.b32 %r50, %r32, -65536;
-; CHECK-SM70-NEXT: add.rn.f32 %r51, %r49, %r50;
-; CHECK-SM70-NEXT: bfe.u32 %r52, %r51, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r53, %r52, %r51;
-; CHECK-SM70-NEXT: add.s32 %r54, %r53, 32767;
-; CHECK-SM70-NEXT: setp.nan.f32 %p8, %r51, %r51;
-; CHECK-SM70-NEXT: or.b32 %r55, %r51, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r56, %r55, %r54, %p8;
-; CHECK-SM70-NEXT: prmt.b32 %r57, %r56, %r47, 0x7632U;
-; CHECK-SM70-NEXT: st.param.b32 [func_retval0], %r57;
+; CHECK-SM70-NEXT: add.rn.f32 %r29, %r28, 0f40E00000;
+; CHECK-SM70-NEXT: shr.u32 %r30, %r29, 16;
+; CHECK-SM70-NEXT: and.b32 %r31, %r30, 1;
+; CHECK-SM70-NEXT: add.s32 %r32, %r31, %r29;
+; CHECK-SM70-NEXT: add.s32 %r33, %r32, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p5, %r29, %r29;
+; CHECK-SM70-NEXT: or.b32 %r34, %r29, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r35, %r34, %r33, %p5;
+; CHECK-SM70-NEXT: add.rn.f32 %r36, %r27, 0f40E00000;
+; CHECK-SM70-NEXT: shr.u32 %r37, %r36, 16;
+; CHECK-SM70-NEXT: and.b32 %r38, %r37, 1;
+; CHECK-SM70-NEXT: add.s32 %r39, %r38, %r36;
+; CHECK-SM70-NEXT: add.s32 %r40, %r39, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p6, %r36, %r36;
+; CHECK-SM70-NEXT: or.b32 %r41, %r36, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r42, %r41, %r40, %p6;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r43, %rs10;
+; CHECK-SM70-NEXT: shl.b32 %r44, %r43, 16;
+; CHECK-SM70-NEXT: and.b32 %r45, %r42, -65536;
+; CHECK-SM70-NEXT: add.rn.f32 %r46, %r44, %r45;
+; CHECK-SM70-NEXT: shr.u32 %r47, %r46, 16;
+; CHECK-SM70-NEXT: and.b32 %r48, %r47, 1;
+; CHECK-SM70-NEXT: add.s32 %r49, %r48, %r46;
+; CHECK-SM70-NEXT: add.s32 %r50, %r49, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p7, %r46, %r46;
+; CHECK-SM70-NEXT: or.b32 %r51, %r46, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r52, %r51, %r50, %p7;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r53, %rs9;
+; CHECK-SM70-NEXT: shl.b32 %r54, %r53, 16;
+; CHECK-SM70-NEXT: and.b32 %r55, %r35, -65536;
+; CHECK-SM70-NEXT: add.rn.f32 %r56, %r54, %r55;
+; CHECK-SM70-NEXT: shr.u32 %r57, %r56, 16;
+; CHECK-SM70-NEXT: and.b32 %r58, %r57, 1;
+; CHECK-SM70-NEXT: add.s32 %r59, %r58, %r56;
+; CHECK-SM70-NEXT: add.s32 %r60, %r59, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p8, %r56, %r56;
+; CHECK-SM70-NEXT: or.b32 %r61, %r56, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r62, %r61, %r60, %p8;
+; CHECK-SM70-NEXT: prmt.b32 %r63, %r62, %r52, 0x7632U;
+; CHECK-SM70-NEXT: st.param.b32 [func_retval0], %r63;
; CHECK-SM70-NEXT: ret;
%1 = fmul fast <2 x bfloat> %a, %b
%2 = fadd fast <2 x bfloat> %1, %c
@@ -809,7 +823,7 @@ define <2 x bfloat> @fma_bf16x2_expanded_maxnum_no_nans(<2 x bfloat> %a, <2 x bf
; CHECK-SM70: {
; CHECK-SM70-NEXT: .reg .pred %p<5>;
; CHECK-SM70-NEXT: .reg .b16 %rs<7>;
-; CHECK-SM70-NEXT: .reg .b32 %r<40>;
+; CHECK-SM70-NEXT: .reg .b32 %r<44>;
; CHECK-SM70-EMPTY:
; CHECK-SM70-NEXT: // %bb.0:
; CHECK-SM70-NEXT: ld.param.v2.b16 {%rs1, %rs2}, [fma_bf16x2_expanded_maxnum_no_nans_param_0];
@@ -822,43 +836,47 @@ define <2 x bfloat> @fma_bf16x2_expanded_maxnum_no_nans(<2 x bfloat> %a, <2 x bf
; CHECK-SM70-NEXT: cvt.u32.u16 %r5, %rs1;
; CHECK-SM70-NEXT: shl.b32 %r6, %r5, 16;
; CHECK-SM70-NEXT: fma.rn.f32 %r7, %r6, %r4, %r2;
-; CHECK-SM70-NEXT: bfe.u32 %r8, %r7, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r9, %r8, %r7;
-; CHECK-SM70-NEXT: add.s32 %r10, %r9, 32767;
+; CHECK-SM70-NEXT: shr.u32 %r8, %r7, 16;
+; CHECK-SM70-NEXT: and.b32 %r9, %r8, 1;
+; CHECK-SM70-NEXT: add.s32 %r10, %r9, %r7;
+; CHECK-SM70-NEXT: add.s32 %r11, %r10, 32767;
; CHECK-SM70-NEXT: setp.nan.f32 %p1, %r7, %r7;
-; CHECK-SM70-NEXT: or.b32 %r11, %r7, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r12, %r11, %r10, %p1;
-; CHECK-SM70-NEXT: cvt.u32.u16 %r13, %rs6;
-; CHECK-SM70-NEXT: shl.b32 %r14, %r13, 16;
-; CHECK-SM70-NEXT: cvt.u32.u16 %r15, %rs4;
-; CHECK-SM70-NEXT: shl.b32 %r16, %r15, 16;
-; CHECK-SM70-NEXT: cvt.u32.u16 %r17, %rs2;
-; CHECK-SM70-NEXT: shl.b32 %r18, %r17, 16;
-; CHECK-SM70-NEXT: fma.rn.f32 %r19, %r18, %r16, %r14;
-; CHECK-SM70-NEXT: bfe.u32 %r20, %r19, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r21, %r20, %r19;
-; CHECK-SM70-NEXT: add.s32 %r22, %r21, 32767;
-; CHECK-SM70-NEXT: setp.nan.f32 %p2, %r19, %r19;
-; CHECK-SM70-NEXT: or.b32 %r23, %r19, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r24, %r23, %r22, %p2;
-; CHECK-SM70-NEXT: and.b32 %r25, %r24, -65536;
-; CHECK-SM70-NEXT: max.f32 %r26, %r25, 0f00000000;
-; CHECK-SM70-NEXT: bfe.u32 %r27, %r26, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r28, %r27, %r26;
-; CHECK-SM70-NEXT: add.s32 %r29, %r28, 32767;
-; CHECK-SM70-NEXT: setp.nan.f32 %p3, %r26, %r26;
-; CHECK-SM70-NEXT: or.b32 %r30, %r26, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r31, %r30, %r29, %p3;
-; CHECK-SM70-NEXT: and.b32 %r32, %r12, -65536;
-; CHECK-SM70-NEXT: max.f32 %r33, %r32, 0f00000000;
-; CHECK-SM70-NEXT: bfe.u32 %r34, %r33, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r35, %r34, %r33;
-; CHECK-SM70-NEXT: add.s32 %r36, %r35, 32767;
-; CHECK-SM70-NEXT: setp.nan.f32 %p4, %r33, %r33;
-; CHECK-SM70-NEXT: or.b32 %r37, %r33, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r38, %r37, %r36, %p4;
-; CHECK-SM70-NEXT: prmt.b32 %r39, %r38, %r31, 0x7632U;
-; CHECK-SM70-NEXT: st.param.b32 [func_retval0], %r39;
+; CHECK-SM70-NEXT: or.b32 %r12, %r7, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r13, %r12, %r11, %p1;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r14, %rs6;
+; CHECK-SM70-NEXT: shl.b32 %r15, %r14, 16;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r16, %rs4;
+; CHECK-SM70-NEXT: shl.b32 %r17, %r16, 16;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r18, %rs2;
+; CHECK-SM70-NEXT: shl.b32 %r19, %r18, 16;
+; CHECK-SM70-NEXT: fma.rn.f32 %r20, %r19, %r17, %r15;
+; CHECK-SM70-NEXT: shr.u32 %r21, %r20, 16;
+; CHECK-SM70-NEXT: and.b32 %r22, %r21, 1;
+; CHECK-SM70-NEXT: add.s32 %r23, %r22, %r20;
+; CHECK-SM70-NEXT: add.s32 %r24, %r23, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p2, %r20, %r20;
+; CHECK-SM70-NEXT: or.b32 %r25, %r20, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r26, %r25, %r24, %p2;
+; CHECK-SM70-NEXT: and.b32 %r27, %r26, -65536;
+; CHECK-SM70-NEXT: max.f32 %r28, %r27, 0f00000000;
+; CHECK-SM70-NEXT: shr.u32 %r29, %r28, 16;
+; CHECK-SM70-NEXT: and.b32 %r30, %r29, 1;
+; CHECK-SM70-NEXT: add.s32 %r31, %r30, %r28;
+; CHECK-SM70-NEXT: add.s32 %r32, %r31, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p3, %r28, %r28;
+; CHECK-SM70-NEXT: or.b32 %r33, %r28, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r34, %r33, %r32, %p3;
+; CHECK-SM70-NEXT: and.b32 %r35, %r13, -65536;
+; CHECK-SM70-NEXT: max.f32 %r36, %r35, 0f00000000;
+; CHECK-SM70-NEXT: shr.u32 %r37, %r36, 16;
+; CHECK-SM70-NEXT: and.b32 %r38, %r37, 1;
+; CHECK-SM70-NEXT: add.s32 %r39, %r38, %r36;
+; CHECK-SM70-NEXT: add.s32 %r40, %r39, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p4, %r36, %r36;
+; CHECK-SM70-NEXT: or.b32 %r41, %r36, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r42, %r41, %r40, %p4;
+; CHECK-SM70-NEXT: prmt.b32 %r43, %r42, %r34, 0x7632U;
+; CHECK-SM70-NEXT: st.param.b32 [func_retval0], %r43;
; CHECK-SM70-NEXT: ret;
%1 = fmul fast <2 x bfloat> %a, %b
%2 = fadd fast <2 x bfloat> %1, %c
@@ -1040,7 +1058,7 @@ define bfloat @fma_bf16_no_nans(bfloat %a, bfloat %b, bfloat %c) {
; CHECK-SM70: {
; CHECK-SM70-NEXT: .reg .pred %p<3>;
; CHECK-SM70-NEXT: .reg .b16 %rs<3>;
-; CHECK-SM70-NEXT: .reg .b32 %r<14>;
+; CHECK-SM70-NEXT: .reg .b32 %r<15>;
; CHECK-SM70-EMPTY:
; CHECK-SM70-NEXT: // %bb.0:
; CHECK-SM70-NEXT: ld.param.b16 %r1, [fma_bf16_no_nans_param_2];
@@ -1050,15 +1068,16 @@ define bfloat @fma_bf16_no_nans(bfloat %a, bfloat %b, bfloat %c) {
; CHECK-SM70-NEXT: ld.param.b16 %r5, [fma_bf16_no_nans_param_0];
; CHECK-SM70-NEXT: shl.b32 %r6, %r5, 16;
; CHECK-SM70-NEXT: fma.rn.f32 %r7, %r6, %r4, %r2;
-; CHECK-SM70-NEXT: bfe.u32 %r8, %r7, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r9, %r8, %r7;
-; CHECK-SM70-NEXT: add.s32 %r10, %r9, 32767;
+; CHECK-SM70-NEXT: shr.u32 %r8, %r7, 16;
+; CHECK-SM70-NEXT: and.b32 %r9, %r8, 1;
+; CHECK-SM70-NEXT: add.s32 %r10, %r9, %r7;
+; CHECK-SM70-NEXT: add.s32 %r11, %r10, 32767;
; CHECK-SM70-NEXT: setp.nan.f32 %p1, %r7, %r7;
-; CHECK-SM70-NEXT: or.b32 %r11, %r7, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r12, %r11, %r10, %p1;
-; CHECK-SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs1}, %r12; }
-; CHECK-SM70-NEXT: and.b32 %r13, %r12, -65536;
-; CHECK-SM70-NEXT: setp.gt.f32 %p2, %r13, 0f00000000;
+; CHECK-SM70-NEXT: or.b32 %r12, %r7, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r13, %r12, %r11, %p1;
+; CHECK-SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs1}, %r13; }
+; CHECK-SM70-NEXT: and.b32 %r14, %r13, -65536;
+; CHECK-SM70-NEXT: setp.gt.f32 %p2, %r14, 0f00000000;
; CHECK-SM70-NEXT: selp.b16 %rs2, %rs1, 0x0000, %p2;
; CHECK-SM70-NEXT: st.param.b16 [func_retval0], %rs2;
; CHECK-SM70-NEXT: ret;
@@ -1110,7 +1129,7 @@ define bfloat @fma_bf16_no_nans_multiple_uses_of_fma(bfloat %a, bfloat %b, bfloa
; CHECK-SM70-LABEL: fma_bf16_no_nans_multiple_uses_of_fma(
; CHECK-SM70: {
; CHECK-SM70-NEXT: .reg .pred %p<4>;
-; CHECK-SM70-NEXT: .reg .b32 %r<28>;
+; CHECK-SM70-NEXT: .reg .b32 %r<31>;
; CHECK-SM70-EMPTY:
; CHECK-SM70-NEXT: // %bb.0:
; CHECK-SM70-NEXT: ld.param.b16 %r1, [fma_bf16_no_nans_multiple_uses_of_fma_param_2];
@@ -1120,30 +1139,33 @@ define bfloat @fma_bf16_no_nans_multiple_uses_of_fma(bfloat %a, bfloat %b, bfloa
; CHECK-SM70-NEXT: ld.param.b16 %r5, [fma_bf16_no_nans_multiple_uses_of_fma_param_0];
; CHECK-SM70-NEXT: shl.b32 %r6, %r5, 16;
; CHECK-SM70-NEXT: fma.rn.f32 %r7, %r6, %r4, %r2;
-; CHECK-SM70-NEXT: bfe.u32 %r8, %r7, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r9, %r8, %r7;
-; CHECK-SM70-NEXT: add.s32 %r10, %r9, 32767;
+; CHECK-SM70-NEXT: shr.u32 %r8, %r7, 16;
+; CHECK-SM70-NEXT: and.b32 %r9, %r8, 1;
+; CHECK-SM70-NEXT: add.s32 %r10, %r9, %r7;
+; CHECK-SM70-NEXT: add.s32 %r11, %r10, 32767;
; CHECK-SM70-NEXT: setp.nan.f32 %p1, %r7, %r7;
-; CHECK-SM70-NEXT: or.b32 %r11, %r7, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r12, %r11, %r10, %p1;
-; CHECK-SM70-NEXT: and.b32 %r13, %r12, -65536;
-; CHECK-SM70-NEXT: add.rn.f32 %r14, %r13, 0f40E00000;
-; CHECK-SM70-NEXT: bfe.u32 %r15, %r14, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r16, %r15, %r14;
-; CHECK-SM70-NEXT: add.s32 %r17, %r16, 32767;
-; CHECK-SM70-NEXT: setp.nan.f32 %p2, %r14, %r14;
-; CHECK-SM70-NEXT: or.b32 %r18, %r14, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r19, %r18, %r17, %p2;
-; CHECK-SM70-NEXT: and.b32 %r20, %r19, -65536;
-; CHECK-SM70-NEXT: add.rn.f32 %r21, %r20, %r13;
-; CHECK-SM70-NEXT: bfe.u32 %r22, %r21, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r23, %r22, %r21;
-; CHECK-SM70-NEXT: add.s32 %r24, %r23, 32767;
-; CHECK-SM70-NEXT: setp.nan.f32 %p3, %r21, %r21;
-; CHECK-SM70-NEXT: or.b32 %r25, %r21, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r26, %r25, %r24, %p3;
-; CHECK-SM70-NEXT: shr.u32 %r27, %r26, 16;
-; CHECK-SM70-NEXT: st.param.b16 [func_retval0], %r27;
+; CHECK-SM70-NEXT: or.b32 %r12, %r7, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r13, %r12, %r11, %p1;
+; CHECK-SM70-NEXT: and.b32 %r14, %r13, -65536;
+; CHECK-SM70-NEXT: add.rn.f32 %r15, %r14, 0f40E00000;
+; CHECK-SM70-NEXT: shr.u32 %r16, %r15, 16;
+; CHECK-SM70-NEXT: and.b32 %r17, %r16, 1;
+; CHECK-SM70-NEXT: add.s32 %r18, %r17, %r15;
+; CHECK-SM70-NEXT: add.s32 %r19, %r18, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p2, %r15, %r15;
+; CHECK-SM70-NEXT: or.b32 %r20, %r15, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r21, %r20, %r19, %p2;
+; CHECK-SM70-NEXT: and.b32 %r22, %r21, -65536;
+; CHECK-SM70-NEXT: add.rn.f32 %r23, %r22, %r14;
+; CHECK-SM70-NEXT: shr.u32 %r24, %r23, 16;
+; CHECK-SM70-NEXT: and.b32 %r25, %r24, 1;
+; CHECK-SM70-NEXT: add.s32 %r26, %r25, %r23;
+; CHECK-SM70-NEXT: add.s32 %r27, %r26, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p3, %r23, %r23;
+; CHECK-SM70-NEXT: or.b32 %r28, %r23, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r29, %r28, %r27, %p3;
+; CHECK-SM70-NEXT: shr.u32 %r30, %r29, 16;
+; CHECK-SM70-NEXT: st.param.b16 [func_retval0], %r30;
; CHECK-SM70-NEXT: ret;
%1 = call nnan bfloat @llvm.fma.bf16(bfloat %a, bfloat %b, bfloat %c)
%2 = fcmp nsz ogt bfloat %1, 0.0
@@ -1181,7 +1203,7 @@ define bfloat @fma_bf16_maxnum_no_nans(bfloat %a, bfloat %b, bfloat %c) {
; CHECK-SM70-LABEL: fma_bf16_maxnum_no_nans(
; CHECK-SM70: {
; CHECK-SM70-NEXT: .reg .pred %p<3>;
-; CHECK-SM70-NEXT: .reg .b32 %r<21>;
+; CHECK-SM70-NEXT: .reg .b32 %r<23>;
; CHECK-SM70-EMPTY:
; CHECK-SM70-NEXT: // %bb.0:
; CHECK-SM70-NEXT: ld.param.b16 %r1, [fma_bf16_maxnum_no_nans_param_2];
@@ -1191,22 +1213,24 @@ define bfloat @fma_bf16_maxnum_no_nans(bfloat %a, bfloat %b, bfloat %c) {
; CHECK-SM70-NEXT: ld.param.b16 %r5, [fma_bf16_maxnum_no_nans_param_0];
; CHECK-SM70-NEXT: shl.b32 %r6, %r5, 16;
; CHECK-SM70-NEXT: fma.rn.f32 %r7, %r6, %r4, %r2;
-; CHECK-SM70-NEXT: bfe.u32 %r8, %r7, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r9, %r8, %r7;
-; CHECK-SM70-NEXT: add.s32 %r10, %r9, 32767;
+; CHECK-SM70-NEXT: shr.u32 %r8, %r7, 16;
+; CHECK-SM70-NEXT: and.b32 %r9, %r8, 1;
+; CHECK-SM70-NEXT: add.s32 %r10, %r9, %r7;
+; CHECK-SM70-NEXT: add.s32 %r11, %r10, 32767;
; CHECK-SM70-NEXT: setp.nan.f32 %p1, %r7, %r7;
-; CHECK-SM70-NEXT: or.b32 %r11, %r7, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r12, %r11, %r10, %p1;
-; CHECK-SM70-NEXT: and.b32 %r13, %r12, -65536;
-; CHECK-SM70-NEXT: max.f32 %r14, %r13, 0f00000000;
-; CHECK-SM70-NEXT: bfe.u32 %r15, %r14, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r16, %r15, %r14;
-; CHECK-SM70-NEXT: add.s32 %r17, %r16, 32767;
-; CHECK-SM70-NEXT: setp.nan.f32 %p2, %r14, %r14;
-; CHECK-SM70-NEXT: or.b32 %r18, %r14, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r19, %r18, %r17, %p2;
-; CHECK-SM70-NEXT: shr.u32 %r20, %r19, 16;
-; CHECK-SM70-NEXT: st.param.b16 [func_retval0], %r20;
+; CHECK-SM70-NEXT: or.b32 %r12, %r7, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r13, %r12, %r11, %p1;
+; CHECK-SM70-NEXT: and.b32 %r14, %r13, -65536;
+; CHECK-SM70-NEXT: max.f32 %r15, %r14, 0f00000000;
+; CHECK-SM70-NEXT: shr.u32 %r16, %r15, 16;
+; CHECK-SM70-NEXT: and.b32 %r17, %r16, 1;
+; CHECK-SM70-NEXT: add.s32 %r18, %r17, %r15;
+; CHECK-SM70-NEXT: add.s32 %r19, %r18, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p2, %r15, %r15;
+; CHECK-SM70-NEXT: or.b32 %r20, %r15, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r21, %r20, %r19, %p2;
+; CHECK-SM70-NEXT: shr.u32 %r22, %r21, 16;
+; CHECK-SM70-NEXT: st.param.b16 [func_retval0], %r22;
; CHECK-SM70-NEXT: ret;
%1 = call nnan bfloat @llvm.fma.bf16(bfloat %a, bfloat %b, bfloat %c)
%2 = call nsz bfloat @llvm.maxnum.bf16(bfloat %1, bfloat 0.0)
@@ -1399,7 +1423,7 @@ define <2 x bfloat> @fma_bf16x2_no_nans(<2 x bfloat> %a, <2 x bfloat> %b, <2 x b
; CHECK-SM70: {
; CHECK-SM70-NEXT: .reg .pred %p<5>;
; CHECK-SM70-NEXT: .reg .b16 %rs<11>;
-; CHECK-SM70-NEXT: .reg .b32 %r<27>;
+; CHECK-SM70-NEXT: .reg .b32 %r<29>;
; CHECK-SM70-EMPTY:
; CHECK-SM70-NEXT: // %bb.0:
; CHECK-SM70-NEXT: ld.param.v2.b16 {%rs1, %rs2}, [fma_bf16x2_no_nans_param_0];
@@ -1412,31 +1436,33 @@ define <2 x bfloat> @fma_bf16x2_no_nans(<2 x bfloat> %a, <2 x bfloat> %b, <2 x b
; CHECK-SM70-NEXT: cvt.u32.u16 %r5, %rs1;
; CHECK-SM70-NEXT: shl.b32 %r6, %r5, 16;
; CHECK-SM70-NEXT: fma.rn.f32 %r7, %r6, %r4, %r2;
-; CHECK-SM70-NEXT: bfe.u32 %r8, %r7, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r9, %r8, %r7;
-; CHECK-SM70-NEXT: add.s32 %r10, %r9, 32767;
+; CHECK-SM70-NEXT: shr.u32 %r8, %r7, 16;
+; CHECK-SM70-NEXT: and.b32 %r9, %r8, 1;
+; CHECK-SM70-NEXT: add.s32 %r10, %r9, %r7;
+; CHECK-SM70-NEXT: add.s32 %r11, %r10, 32767;
; CHECK-SM70-NEXT: setp.nan.f32 %p1, %r7, %r7;
-; CHECK-SM70-NEXT: or.b32 %r11, %r7, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r12, %r11, %r10, %p1;
-; CHECK-SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs7}, %r12; }
-; CHECK-SM70-NEXT: cvt.u32.u16 %r13, %rs4;
-; CHECK-SM70-NEXT: shl.b32 %r14, %r13, 16;
-; CHECK-SM70-NEXT: cvt.u32.u16 %r15, %rs6;
-; CHECK-SM70-NEXT: shl.b32 %r16, %r15, 16;
-; CHECK-SM70-NEXT: cvt.u32.u16 %r17, %rs2;
-; CHECK-SM70-NEXT: shl.b32 %r18, %r17, 16;
-; CHECK-SM70-NEXT: fma.rn.f32 %r19, %r18, %r16, %r14;
-; CHECK-SM70-NEXT: bfe.u32 %r20, %r19, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r21, %r20, %r19;
-; CHECK-SM70-NEXT: add.s32 %r22, %r21, 32767;
-; CHECK-SM70-NEXT: setp.nan.f32 %p2, %r19, %r19;
-; CHECK-SM70-NEXT: or.b32 %r23, %r19, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r24, %r23, %r22, %p2;
-; CHECK-SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs8}, %r24; }
-; CHECK-SM70-NEXT: and.b32 %r25, %r12, -65536;
-; CHECK-SM70-NEXT: setp.gt.f32 %p3, %r25, 0f00000000;
-; CHECK-SM70-NEXT: and.b32 %r26, %r24, -65536;
-; CHECK-SM70-NEXT: setp.gt.f32 %p4, %r26, 0f00000000;
+; CHECK-SM70-NEXT: or.b32 %r12, %r7, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r13, %r12, %r11, %p1;
+; CHECK-SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs7}, %r13; }
+; CHECK-SM70-NEXT: cvt.u32.u16 %r14, %rs4;
+; CHECK-SM70-NEXT: shl.b32 %r15, %r14, 16;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r16, %rs6;
+; CHECK-SM70-NEXT: shl.b32 %r17, %r16, 16;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r18, %rs2;
+; CHECK-SM70-NEXT: shl.b32 %r19, %r18, 16;
+; CHECK-SM70-NEXT: fma.rn.f32 %r20, %r19, %r17, %r15;
+; CHECK-SM70-NEXT: shr.u32 %r21, %r20, 16;
+; CHECK-SM70-NEXT: and.b32 %r22, %r21, 1;
+; CHECK-SM70-NEXT: add.s32 %r23, %r22, %r20;
+; CHECK-SM70-NEXT: add.s32 %r24, %r23, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p2, %r20, %r20;
+; CHECK-SM70-NEXT: or.b32 %r25, %r20, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r26, %r25, %r24, %p2;
+; CHECK-SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs8}, %r26; }
+; CHECK-SM70-NEXT: and.b32 %r27, %r13, -65536;
+; CHECK-SM70-NEXT: setp.gt.f32 %p3, %r27, 0f00000000;
+; CHECK-SM70-NEXT: and.b32 %r28, %r26, -65536;
+; CHECK-SM70-NEXT: setp.gt.f32 %p4, %r28, 0f00000000;
; CHECK-SM70-NEXT: selp.b16 %rs9, %rs8, 0x0000, %p4;
; CHECK-SM70-NEXT: selp.b16 %rs10, %rs7, 0x0000, %p3;
; CHECK-SM70-NEXT: st.param.v2.b16 [func_retval0], {%rs10, %rs9};
@@ -1498,7 +1524,7 @@ define <2 x bfloat> @fma_bf16x2_no_nans_multiple_uses_of_fma(<2 x bfloat> %a, <2
; CHECK-SM70: {
; CHECK-SM70-NEXT: .reg .pred %p<7>;
; CHECK-SM70-NEXT: .reg .b16 %rs<7>;
-; CHECK-SM70-NEXT: .reg .b32 %r<54>;
+; CHECK-SM70-NEXT: .reg .b32 %r<60>;
; CHECK-SM70-EMPTY:
; CHECK-SM70-NEXT: // %bb.0:
; CHECK-SM70-NEXT: ld.param.v2.b16 {%rs1, %rs2}, [fma_bf16x2_no_nans_multiple_uses_of_fma_param_0];
@@ -1511,59 +1537,65 @@ define <2 x bfloat> @fma_bf16x2_no_nans_multiple_uses_of_fma(<2 x bfloat> %a, <2
; CHECK-SM70-NEXT: cvt.u32.u16 %r5, %rs2;
; CHECK-SM70-NEXT: shl.b32 %r6, %r5, 16;
; CHECK-SM70-NEXT: fma.rn.f32 %r7, %r6, %r4, %r2;
-; CHECK-SM70-NEXT: bfe.u32 %r8, %r7, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r9, %r8, %r7;
-; CHECK-SM70-NEXT: add.s32 %r10, %r9, 32767;
+; CHECK-SM70-NEXT: shr.u32 %r8, %r7, 16;
+; CHECK-SM70-NEXT: and.b32 %r9, %r8, 1;
+; CHECK-SM70-NEXT: add.s32 %r10, %r9, %r7;
+; CHECK-SM70-NEXT: add.s32 %r11, %r10, 32767;
; CHECK-SM70-NEXT: setp.nan.f32 %p1, %r7, %r7;
-; CHECK-SM70-NEXT: or.b32 %r11, %r7, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r12, %r11, %r10, %p1;
-; CHECK-SM70-NEXT: cvt.u32.u16 %r13, %rs3;
-; CHECK-SM70-NEXT: shl.b32 %r14, %r13, 16;
-; CHECK-SM70-NEXT: cvt.u32.u16 %r15, %rs5;
-; CHECK-SM70-NEXT: shl.b32 %r16, %r15, 16;
-; CHECK-SM70-NEXT: cvt.u32.u16 %r17, %rs1;
-; CHECK-SM70-NEXT: shl.b32 %r18, %r17, 16;
-; CHECK-SM70-NEXT: fma.rn.f32 %r19, %r18, %r16, %r14;
-; CHECK-SM70-NEXT: bfe.u32 %r20, %r19, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r21, %r20, %r19;
-; CHECK-SM70-NEXT: add.s32 %r22, %r21, 32767;
-; CHECK-SM70-NEXT: setp.nan.f32 %p2, %r19, %r19;
-; CHECK-SM70-NEXT: or.b32 %r23, %r19, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r24, %r23, %r22, %p2;
-; CHECK-SM70-NEXT: and.b32 %r25, %r24, -65536;
-; CHECK-SM70-NEXT: add.rn.f32 %r26, %r25, 0f40E00000;
-; CHECK-SM70-NEXT: bfe.u32 %r27, %r26, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r28, %r27, %r26;
-; CHECK-SM70-NEXT: add.s32 %r29, %r28, 32767;
-; CHECK-SM70-NEXT: setp.nan.f32 %p3, %r26, %r26;
-; CHECK-SM70-NEXT: or.b32 %r30, %r26, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r31, %r30, %r29, %p3;
-; CHECK-SM70-NEXT: and.b32 %r32, %r12, -65536;
-; CHECK-SM70-NEXT: add.rn.f32 %r33, %r32, 0f40E00000;
-; CHECK-SM70-NEXT: bfe.u32 %r34, %r33, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r35, %r34, %r33;
-; CHECK-SM70-NEXT: add.s32 %r36, %r35, 32767;
-; CHECK-SM70-NEXT: setp.nan.f32 %p4, %r33, %r33;
-; CHECK-SM70-NEXT: or.b32 %r37, %r33, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r38, %r37, %r36, %p4;
-; CHECK-SM70-NEXT: and.b32 %r39, %r38, -65536;
-; CHECK-SM70-NEXT: add.rn.f32 %r40, %r39, %r32;
-; CHECK-SM70-NEXT: bfe.u32 %r41, %r40, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r42, %r41, %r40;
-; CHECK-SM70-NEXT: add.s32 %r43, %r42, 32767;
-; CHECK-SM70-NEXT: setp.nan.f32 %p5, %r40, %r40;
-; CHECK-SM70-NEXT: or.b32 %r44, %r40, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r45, %r44, %r43, %p5;
-; CHECK-SM70-NEXT: and.b32 %r46, %r31, -65536;
-; CHECK-SM70-NEXT: add.rn.f32 %r47, %r46, %r25;
-; CHECK-SM70-NEXT: bfe.u32 %r48, %r47, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r49, %r48, %r47;
-; CHECK-SM70-NEXT: add.s32 %r50, %r49, 32767;
-; CHECK-SM70-NEXT: setp.nan.f32 %p6, %r47, %r47;
-; CHECK-SM70-NEXT: or.b32 %r51, %r47, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r52, %r51, %r50, %p6;
-; CHECK-SM70-NEXT: prmt.b32 %r53, %r52, %r45, 0x7632U;
-; CHECK-SM70-NEXT: st.param.b32 [func_retval0], %r53;
+; CHECK-SM70-NEXT: or.b32 %r12, %r7, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r13, %r12, %r11, %p1;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r14, %rs3;
+; CHECK-SM70-NEXT: shl.b32 %r15, %r14, 16;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r16, %rs5;
+; CHECK-SM70-NEXT: shl.b32 %r17, %r16, 16;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r18, %rs1;
+; CHECK-SM70-NEXT: shl.b32 %r19, %r18, 16;
+; CHECK-SM70-NEXT: fma.rn.f32 %r20, %r19, %r17, %r15;
+; CHECK-SM70-NEXT: shr.u32 %r21, %r20, 16;
+; CHECK-SM70-NEXT: and.b32 %r22, %r21, 1;
+; CHECK-SM70-NEXT: add.s32 %r23, %r22, %r20;
+; CHECK-SM70-NEXT: add.s32 %r24, %r23, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p2, %r20, %r20;
+; CHECK-SM70-NEXT: or.b32 %r25, %r20, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r26, %r25, %r24, %p2;
+; CHECK-SM70-NEXT: and.b32 %r27, %r26, -65536;
+; CHECK-SM70-NEXT: add.rn.f32 %r28, %r27, 0f40E00000;
+; CHECK-SM70-NEXT: shr.u32 %r29, %r28, 16;
+; CHECK-SM70-NEXT: and.b32 %r30, %r29, 1;
+; CHECK-SM70-NEXT: add.s32 %r31, %r30, %r28;
+; CHECK-SM70-NEXT: add.s32 %r32, %r31, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p3, %r28, %r28;
+; CHECK-SM70-NEXT: or.b32 %r33, %r28, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r34, %r33, %r32, %p3;
+; CHECK-SM70-NEXT: and.b32 %r35, %r13, -65536;
+; CHECK-SM70-NEXT: add.rn.f32 %r36, %r35, 0f40E00000;
+; CHECK-SM70-NEXT: shr.u32 %r37, %r36, 16;
+; CHECK-SM70-NEXT: and.b32 %r38, %r37, 1;
+; CHECK-SM70-NEXT: add.s32 %r39, %r38, %r36;
+; CHECK-SM70-NEXT: add.s32 %r40, %r39, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p4, %r36, %r36;
+; CHECK-SM70-NEXT: or.b32 %r41, %r36, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r42, %r41, %r40, %p4;
+; CHECK-SM70-NEXT: and.b32 %r43, %r42, -65536;
+; CHECK-SM70-NEXT: add.rn.f32 %r44, %r43, %r35;
+; CHECK-SM70-NEXT: shr.u32 %r45, %r44, 16;
+; CHECK-SM70-NEXT: and.b32 %r46, %r45, 1;
+; CHECK-SM70-NEXT: add.s32 %r47, %r46, %r44;
+; CHECK-SM70-NEXT: add.s32 %r48, %r47, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p5, %r44, %r44;
+; CHECK-SM70-NEXT: or.b32 %r49, %r44, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r50, %r49, %r48, %p5;
+; CHECK-SM70-NEXT: and.b32 %r51, %r34, -65536;
+; CHECK-SM70-NEXT: add.rn.f32 %r52, %r51, %r27;
+; CHECK-SM70-NEXT: shr.u32 %r53, %r52, 16;
+; CHECK-SM70-NEXT: and.b32 %r54, %r53, 1;
+; CHECK-SM70-NEXT: add.s32 %r55, %r54, %r52;
+; CHECK-SM70-NEXT: add.s32 %r56, %r55, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p6, %r52, %r52;
+; CHECK-SM70-NEXT: or.b32 %r57, %r52, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r58, %r57, %r56, %p6;
+; CHECK-SM70-NEXT: prmt.b32 %r59, %r58, %r50, 0x7632U;
+; CHECK-SM70-NEXT: st.param.b32 [func_retval0], %r59;
; CHECK-SM70-NEXT: ret;
%1 = call nnan <2 x bfloat> @llvm.fma.bf16x2(<2 x bfloat> %a, <2 x bfloat> %b, <2 x bfloat> %c)
%2 = fcmp nsz ogt <2 x bfloat> %1, <bfloat 0.0, bfloat 0.0>
@@ -1602,7 +1634,7 @@ define <2 x bfloat> @fma_bf16x2_maxnum_no_nans(<2 x bfloat> %a, <2 x bfloat> %b,
; CHECK-SM70: {
; CHECK-SM70-NEXT: .reg .pred %p<5>;
; CHECK-SM70-NEXT: .reg .b16 %rs<7>;
-; CHECK-SM70-NEXT: .reg .b32 %r<40>;
+; CHECK-SM70-NEXT: .reg .b32 %r<44>;
; CHECK-SM70-EMPTY:
; CHECK-SM70-NEXT: // %bb.0:
; CHECK-SM70-NEXT: ld.param.v2.b16 {%rs1, %rs2}, [fma_bf16x2_maxnum_no_nans_param_0];
@@ -1615,43 +1647,47 @@ define <2 x bfloat> @fma_bf16x2_maxnum_no_nans(<2 x bfloat> %a, <2 x bfloat> %b,
; CHECK-SM70-NEXT: cvt.u32.u16 %r5, %rs1;
; CHECK-SM70-NEXT: shl.b32 %r6, %r5, 16;
; CHECK-SM70-NEXT: fma.rn.f32 %r7, %r6, %r4, %r2;
-; CHECK-SM70-NEXT: bfe.u32 %r8, %r7, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r9, %r8, %r7;
-; CHECK-SM70-NEXT: add.s32 %r10, %r9, 32767;
+; CHECK-SM70-NEXT: shr.u32 %r8, %r7, 16;
+; CHECK-SM70-NEXT: and.b32 %r9, %r8, 1;
+; CHECK-SM70-NEXT: add.s32 %r10, %r9, %r7;
+; CHECK-SM70-NEXT: add.s32 %r11, %r10, 32767;
; CHECK-SM70-NEXT: setp.nan.f32 %p1, %r7, %r7;
-; CHECK-SM70-NEXT: or.b32 %r11, %r7, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r12, %r11, %r10, %p1;
-; CHECK-SM70-NEXT: cvt.u32.u16 %r13, %rs4;
-; CHECK-SM70-NEXT: shl.b32 %r14, %r13, 16;
-; CHECK-SM70-NEXT: cvt.u32.u16 %r15, %rs6;
-; CHECK-SM70-NEXT: shl.b32 %r16, %r15, 16;
-; CHECK-SM70-NEXT: cvt.u32.u16 %r17, %rs2;
-; CHECK-SM70-NEXT: shl.b32 %r18, %r17, 16;
-; CHECK-SM70-NEXT: fma.rn.f32 %r19, %r18, %r16, %r14;
-; CHECK-SM70-NEXT: bfe.u32 %r20, %r19, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r21, %r20, %r19;
-; CHECK-SM70-NEXT: add.s32 %r22, %r21, 32767;
-; CHECK-SM70-NEXT: setp.nan.f32 %p2, %r19, %r19;
-; CHECK-SM70-NEXT: or.b32 %r23, %r19, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r24, %r23, %r22, %p2;
-; CHECK-SM70-NEXT: and.b32 %r25, %r24, -65536;
-; CHECK-SM70-NEXT: max.f32 %r26, %r25, 0f00000000;
-; CHECK-SM70-NEXT: bfe.u32 %r27, %r26, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r28, %r27, %r26;
-; CHECK-SM70-NEXT: add.s32 %r29, %r28, 32767;
-; CHECK-SM70-NEXT: setp.nan.f32 %p3, %r26, %r26;
-; CHECK-SM70-NEXT: or.b32 %r30, %r26, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r31, %r30, %r29, %p3;
-; CHECK-SM70-NEXT: and.b32 %r32, %r12, -65536;
-; CHECK-SM70-NEXT: max.f32 %r33, %r32, 0f00000000;
-; CHECK-SM70-NEXT: bfe.u32 %r34, %r33, 16, 1;
-; CHECK-SM70-NEXT: add.s32 %r35, %r34, %r33;
-; CHECK-SM70-NEXT: add.s32 %r36, %r35, 32767;
-; CHECK-SM70-NEXT: setp.nan.f32 %p4, %r33, %r33;
-; CHECK-SM70-NEXT: or.b32 %r37, %r33, 4194304;
-; CHECK-SM70-NEXT: selp.b32 %r38, %r37, %r36, %p4;
-; CHECK-SM70-NEXT: prmt.b32 %r39, %r38, %r31, 0x7632U;
-; CHECK-SM70-NEXT: st.param.b32 [func_retval0], %r39;
+; CHECK-SM70-NEXT: or.b32 %r12, %r7, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r13, %r12, %r11, %p1;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r14, %rs4;
+; CHECK-SM70-NEXT: shl.b32 %r15, %r14, 16;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r16, %rs6;
+; CHECK-SM70-NEXT: shl.b32 %r17, %r16, 16;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r18, %rs2;
+; CHECK-SM70-NEXT: shl.b32 %r19, %r18, 16;
+; CHECK-SM70-NEXT: fma.rn.f32 %r20, %r19, %r17, %r15;
+; CHECK-SM70-NEXT: shr.u32 %r21, %r20, 16;
+; CHECK-SM70-NEXT: and.b32 %r22, %r21, 1;
+; CHECK-SM70-NEXT: add.s32 %r23, %r22, %r20;
+; CHECK-SM70-NEXT: add.s32 %r24, %r23, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p2, %r20, %r20;
+; CHECK-SM70-NEXT: or.b32 %r25, %r20, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r26, %r25, %r24, %p2;
+; CHECK-SM70-NEXT: and.b32 %r27, %r26, -65536;
+; CHECK-SM70-NEXT: max.f32 %r28, %r27, 0f00000000;
+; CHECK-SM70-NEXT: shr.u32 %r29, %r28, 16;
+; CHECK-SM70-NEXT: and.b32 %r30, %r29, 1;
+; CHECK-SM70-NEXT: add.s32 %r31, %r30, %r28;
+; CHECK-SM70-NEXT: add.s32 %r32, %r31, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p3, %r28, %r28;
+; CHECK-SM70-NEXT: or.b32 %r33, %r28, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r34, %r33, %r32, %p3;
+; CHECK-SM70-NEXT: and.b32 %r35, %r13, -65536;
+; CHECK-SM70-NEXT: max.f32 %r36, %r35, 0f00000000;
+; CHECK-SM70-NEXT: shr.u32 %r37, %r36, 16;
+; CHECK-SM70-NEXT: and.b32 %r38, %r37, 1;
+; CHECK-SM70-NEXT: add.s32 %r39, %r38, %r36;
+; CHECK-SM70-NEXT: add.s32 %r40, %r39, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p4, %r36, %r36;
+; CHECK-SM70-NEXT: or.b32 %r41, %r36, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r42, %r41, %r40, %p4;
+; CHECK-SM70-NEXT: prmt.b32 %r43, %r42, %r34, 0x7632U;
+; CHECK-SM70-NEXT: st.param.b32 [func_retval0], %r43;
; CHECK-SM70-NEXT: ret;
%1 = call nnan <2 x bfloat> @llvm.fma.bf16x2(<2 x bfloat> %a, <2 x bfloat> %b, <2 x bfloat> %c)
%2 = call nsz <2 x bfloat> @llvm.maxnum.bf16x2(<2 x bfloat> %1, <2 x bfloat> <bfloat 0.0, bfloat 0.0>)
diff --git a/llvm/test/CodeGen/NVPTX/mulwide.ll b/llvm/test/CodeGen/NVPTX/mulwide.ll
index ce2e0f293342e..05d0b193dafdd 100644
--- a/llvm/test/CodeGen/NVPTX/mulwide.ll
+++ b/llvm/test/CodeGen/NVPTX/mulwide.ll
@@ -258,7 +258,7 @@ define i64 @mulwides7(i7 %a, i7 %b) {
; NOOPT-LABEL: mulwides7(
; NOOPT: {
; NOOPT-NEXT: .reg .b16 %rs<9>;
-; NOOPT-NEXT: .reg .b64 %rd<6>;
+; NOOPT-NEXT: .reg .b64 %rd<8>;
; NOOPT-EMPTY:
; NOOPT-NEXT: // %bb.0:
; NOOPT-NEXT: ld.param.b8 %rs3, [mulwides7_param_0+1];
@@ -271,10 +271,12 @@ define i64 @mulwides7(i7 %a, i7 %b) {
; NOOPT-NEXT: ld.param.b8 %rs7, [mulwides7_param_1+1];
; NOOPT-NEXT: shl.b16 %rs8, %rs7, 8;
; NOOPT-NEXT: or.b16 %rs2, %rs8, %rs6;
-; NOOPT-NEXT: bfe.s64 %rd3, %rd2, 0, 7;
-; NOOPT-NEXT: bfe.s64 %rd4, %rd1, 0, 7;
-; NOOPT-NEXT: mul.lo.s64 %rd5, %rd3, %rd4;
-; NOOPT-NEXT: st.param.b64 [func_retval0], %rd5;
+; NOOPT-NEXT: shl.b64 %rd3, %rd2, 57;
+; NOOPT-NEXT: shr.s64 %rd4, %rd3, 57;
+; NOOPT-NEXT: shl.b64 %rd5, %rd1, 57;
+; NOOPT-NEXT: shr.s64 %rd6, %rd5, 57;
+; NOOPT-NEXT: mul.lo.s64 %rd7, %rd4, %rd6;
+; NOOPT-NEXT: st.param.b64 [func_retval0], %rd7;
; NOOPT-NEXT: ret;
%val0 = sext i7 %a to i64
%val1 = sext i7 %b to i64
diff --git a/llvm/test/CodeGen/NVPTX/tanhf.ll b/llvm/test/CodeGen/NVPTX/tanhf.ll
index d7f6e02b349e4..b15819193aa8e 100644
--- a/llvm/test/CodeGen/NVPTX/tanhf.ll
+++ b/llvm/test/CodeGen/NVPTX/tanhf.ll
@@ -52,20 +52,21 @@ define bfloat @tanh_bf16(bfloat %in) local_unnamed_addr {
; CHECK-SM75-LABEL: tanh_bf16(
; CHECK-SM75: {
; CHECK-SM75-NEXT: .reg .pred %p<2>;
-; CHECK-SM75-NEXT: .reg .b32 %r<10>;
+; CHECK-SM75-NEXT: .reg .b32 %r<11>;
; CHECK-SM75-EMPTY:
; CHECK-SM75-NEXT: // %bb.0:
; CHECK-SM75-NEXT: ld.param.b16 %r1, [tanh_bf16_param_0];
; CHECK-SM75-NEXT: shl.b32 %r2, %r1, 16;
; CHECK-SM75-NEXT: tanh.approx.f32 %r3, %r2;
-; CHECK-SM75-NEXT: bfe.u32 %r4, %r3, 16, 1;
-; CHECK-SM75-NEXT: add.s32 %r5, %r4, %r3;
-; CHECK-SM75-NEXT: add.s32 %r6, %r5, 32767;
+; CHECK-SM75-NEXT: shr.u32 %r4, %r3, 16;
+; CHECK-SM75-NEXT: and.b32 %r5, %r4, 1;
+; CHECK-SM75-NEXT: add.s32 %r6, %r5, %r3;
+; CHECK-SM75-NEXT: add.s32 %r7, %r6, 32767;
; CHECK-SM75-NEXT: setp.nan.f32 %p1, %r3, %r3;
-; CHECK-SM75-NEXT: or.b32 %r7, %r3, 4194304;
-; CHECK-SM75-NEXT: selp.b32 %r8, %r7, %r6, %p1;
-; CHECK-SM75-NEXT: shr.u32 %r9, %r8, 16;
-; CHECK-SM75-NEXT: st.param.b16 [func_retval0], %r9;
+; CHECK-SM75-NEXT: or.b32 %r8, %r3, 4194304;
+; CHECK-SM75-NEXT: selp.b32 %r9, %r8, %r7, %p1;
+; CHECK-SM75-NEXT: shr.u32 %r10, %r9, 16;
+; CHECK-SM75-NEXT: st.param.b16 [func_retval0], %r10;
; CHECK-SM75-NEXT: ret;
;
; CHECK-SM90-LABEL: tanh_bf16(
@@ -86,30 +87,32 @@ define <2 x bfloat> @tanh_bf16x2(<2 x bfloat> %in) local_unnamed_addr {
; CHECK-SM75: {
; CHECK-SM75-NEXT: .reg .pred %p<3>;
; CHECK-SM75-NEXT: .reg .b16 %rs<3>;
-; CHECK-SM75-NEXT: .reg .b32 %r<18>;
+; CHECK-SM75-NEXT: .reg .b32 %r<20>;
; CHECK-SM75-EMPTY:
; CHECK-SM75-NEXT: // %bb.0:
; CHECK-SM75-NEXT: ld.param.v2.b16 {%rs1, %rs2}, [tanh_bf16x2_param_0];
; CHECK-SM75-NEXT: cvt.u32.u16 %r1, %rs2;
; CHECK-SM75-NEXT: shl.b32 %r2, %r1, 16;
; CHECK-SM75-NEXT: tanh.approx.f32 %r3, %r2;
-; CHECK-SM75-NEXT: bfe.u32 %r4, %r3, 16, 1;
-; CHECK-SM75-NEXT: add.s32 %r5, %r4, %r3;
-; CHECK-SM75-NEXT: add.s32 %r6, %r5, 32767;
+; CHECK-SM75-NEXT: shr.u32 %r4, %r3, 16;
+; CHECK-SM75-NEXT: and.b32 %r5, %r4, 1;
+; CHECK-SM75-NEXT: add.s32 %r6, %r5, %r3;
+; CHECK-SM75-NEXT: add.s32 %r7, %r6, 32767;
; CHECK-SM75-NEXT: setp.nan.f32 %p1, %r3, %r3;
-; CHECK-SM75-NEXT: or.b32 %r7, %r3, 4194304;
-; CHECK-SM75-NEXT: selp.b32 %r8, %r7, %r6, %p1;
-; CHECK-SM75-NEXT: cvt.u32.u16 %r9, %rs1;
-; CHECK-SM75-NEXT: shl.b32 %r10, %r9, 16;
-; CHECK-SM75-NEXT: tanh.approx.f32 %r11, %r10;
-; CHECK-SM75-NEXT: bfe.u32 %r12, %r11, 16, 1;
-; CHECK-SM75-NEXT: add.s32 %r13, %r12, %r11;
-; CHECK-SM75-NEXT: add.s32 %r14, %r13, 32767;
-; CHECK-SM75-NEXT: setp.nan.f32 %p2, %r11, %r11;
-; CHECK-SM75-NEXT: or.b32 %r15, %r11, 4194304;
-; CHECK-SM75-NEXT: selp.b32 %r16, %r15, %r14, %p2;
-; CHECK-SM75-NEXT: prmt.b32 %r17, %r16, %r8, 0x7632U;
-; CHECK-SM75-NEXT: st.param.b32 [func_retval0], %r17;
+; CHECK-SM75-NEXT: or.b32 %r8, %r3, 4194304;
+; CHECK-SM75-NEXT: selp.b32 %r9, %r8, %r7, %p1;
+; CHECK-SM75-NEXT: cvt.u32.u16 %r10, %rs1;
+; CHECK-SM75-NEXT: shl.b32 %r11, %r10, 16;
+; CHECK-SM75-NEXT: tanh.approx.f32 %r12, %r11;
+; CHECK-SM75-NEXT: shr.u32 %r13, %r12, 16;
+; CHECK-SM75-NEXT: and.b32 %r14, %r13, 1;
+; CHECK-SM75-NEXT: add.s32 %r15, %r14, %r12;
+; CHECK-SM75-NEXT: add.s32 %r16, %r15, 32767;
+; CHECK-SM75-NEXT: setp.nan.f32 %p2, %r12, %r12;
+; CHECK-SM75-NEXT: or.b32 %r17, %r12, 4194304;
+; CHECK-SM75-NEXT: selp.b32 %r18, %r17, %r16, %p2;
+; CHECK-SM75-NEXT: prmt.b32 %r19, %r18, %r9, 0x7632U;
+; CHECK-SM75-NEXT: st.param.b32 [func_retval0], %r19;
; CHECK-SM75-NEXT: ret;
;
; CHECK-SM90-LABEL: tanh_bf16x2(
More information about the llvm-commits
mailing list