[llvm] fix `llvm.fma.f16` double rounding issue when there is no native support (PR #171904)
Folkert de Vries via llvm-commits
llvm-commits at lists.llvm.org
Sun Dec 14 15:45:08 PST 2025
https://github.com/folkertdev updated https://github.com/llvm/llvm-project/pull/171904
>From 88b91597075324da2065edfcc80f747581b6849e Mon Sep 17 00:00:00 2001
From: Folkert de Vries <folkert at folkertdev.nl>
Date: Thu, 11 Dec 2025 20:45:57 +0100
Subject: [PATCH 1/5] promote f16 fma to f64 if there is no instruction support
---
.../SelectionDAG/LegalizeFloatTypes.cpp | 26 +++++-
llvm/test/CodeGen/ARM/fp16-promote.ll | 80 ++++++++++++-------
2 files changed, 73 insertions(+), 33 deletions(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp
index bf1abfe50327e..51bc335550a16 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp
@@ -3495,10 +3495,30 @@ SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FMAD(SDNode *N) {
Op1 = DAG.getNode(PromotionOpcode, dl, NVT, Op1);
Op2 = DAG.getNode(PromotionOpcode, dl, NVT, Op2);
- SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, Op0, Op1, Op2);
+ SDValue Res;
+ if (NVT == MVT::f32) {
+ // An f16 fma must go via f64 to prevent double rounding issues.
+ SDValue A64 = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Op0);
+ SDValue B64 = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Op1);
+ SDValue C64 = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Op2);
+
+ // Prefer a wide FMA node if available; otherwise expand to mul+add.
+ SDValue WideRes;
+ if (TLI.isOperationLegalOrCustom(ISD::FMA, MVT::f64)) {
+ WideRes =
+ DAG.getNode(ISD::FMA, dl, MVT::f64, A64, B64, C64, N->getFlags());
+ } else {
+ SDValue Mul =
+ DAG.getNode(ISD::FMUL, dl, MVT::f64, A64, B64, N->getFlags());
+ WideRes = DAG.getNode(ISD::FADD, dl, MVT::f64, Mul, C64, N->getFlags());
+ }
- // Convert back to FP16 as an integer.
- return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res);
+ return DAG.getNode(GetPromotionOpcode(MVT::f64, OVT), dl, MVT::i16,
+ WideRes);
+ } else {
+ Res = DAG.getNode(N->getOpcode(), dl, NVT, Op0, Op1, Op2, N->getFlags());
+ return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res);
+ }
}
SDValue DAGTypeLegalizer::SoftPromoteHalfRes_ExpOp(SDNode *N) {
diff --git a/llvm/test/CodeGen/ARM/fp16-promote.ll b/llvm/test/CodeGen/ARM/fp16-promote.ll
index 8230e47259dd8..27a0bf2eb9037 100644
--- a/llvm/test/CodeGen/ARM/fp16-promote.ll
+++ b/llvm/test/CodeGen/ARM/fp16-promote.ll
@@ -1508,61 +1508,81 @@ define void @test_fma(ptr %p, ptr %q, ptr %r) #0 {
; CHECK-FP16-NEXT: push {r4, lr}
; CHECK-FP16-NEXT: mov r4, r0
; CHECK-FP16-NEXT: ldrh r0, [r1]
-; CHECK-FP16-NEXT: ldrh r1, [r4]
-; CHECK-FP16-NEXT: ldrh r2, [r2]
-; CHECK-FP16-NEXT: vmov s2, r0
+; CHECK-FP16-NEXT: ldrh r1, [r2]
+; CHECK-FP16-NEXT: vmov s0, r0
+; CHECK-FP16-NEXT: ldrh r0, [r4]
+; CHECK-FP16-NEXT: vcvtb.f32.f16 s0, s0
+; CHECK-FP16-NEXT: vcvt.f64.f32 d16, s0
+; CHECK-FP16-NEXT: vmov s0, r0
+; CHECK-FP16-NEXT: vcvtb.f32.f16 s0, s0
+; CHECK-FP16-NEXT: vcvt.f64.f32 d17, s0
; CHECK-FP16-NEXT: vmov s0, r1
-; CHECK-FP16-NEXT: vcvtb.f32.f16 s1, s2
-; CHECK-FP16-NEXT: vmov s2, r2
; CHECK-FP16-NEXT: vcvtb.f32.f16 s0, s0
-; CHECK-FP16-NEXT: vcvtb.f32.f16 s2, s2
-; CHECK-FP16-NEXT: bl fmaf
-; CHECK-FP16-NEXT: vcvtb.f16.f32 s0, s0
-; CHECK-FP16-NEXT: vmov r0, s0
+; CHECK-FP16-NEXT: vcvt.f64.f32 d18, s0
+; CHECK-FP16-NEXT: vmla.f64 d18, d17, d16
+; CHECK-FP16-NEXT: vmov r0, r1, d18
+; CHECK-FP16-NEXT: bl __aeabi_d2h
; CHECK-FP16-NEXT: strh r0, [r4]
; CHECK-FP16-NEXT: pop {r4, pc}
;
; CHECK-LIBCALL-VFP-LABEL: test_fma:
; CHECK-LIBCALL-VFP: .save {r4, r5, r6, lr}
; CHECK-LIBCALL-VFP-NEXT: push {r4, r5, r6, lr}
+; CHECK-LIBCALL-VFP-NEXT: .vsave {d8, d9}
+; CHECK-LIBCALL-VFP-NEXT: vpush {d8, d9}
; CHECK-LIBCALL-VFP-NEXT: mov r4, r0
-; CHECK-LIBCALL-VFP-NEXT: ldrh r0, [r2]
-; CHECK-LIBCALL-VFP-NEXT: mov r5, r1
+; CHECK-LIBCALL-VFP-NEXT: ldrh r0, [r0]
+; CHECK-LIBCALL-VFP-NEXT: mov r5, r2
+; CHECK-LIBCALL-VFP-NEXT: mov r6, r1
; CHECK-LIBCALL-VFP-NEXT: bl __aeabi_h2f
-; CHECK-LIBCALL-VFP-NEXT: mov r6, r0
-; CHECK-LIBCALL-VFP-NEXT: ldrh r0, [r5]
+; CHECK-LIBCALL-VFP-NEXT: ldrh r1, [r6]
+; CHECK-LIBCALL-VFP-NEXT: vmov s16, r0
+; CHECK-LIBCALL-VFP-NEXT: ldrh r5, [r5]
+; CHECK-LIBCALL-VFP-NEXT: mov r0, r1
; CHECK-LIBCALL-VFP-NEXT: bl __aeabi_h2f
-; CHECK-LIBCALL-VFP-NEXT: mov r5, r0
-; CHECK-LIBCALL-VFP-NEXT: ldrh r0, [r4]
+; CHECK-LIBCALL-VFP-NEXT: vmov s18, r0
+; CHECK-LIBCALL-VFP-NEXT: mov r0, r5
; CHECK-LIBCALL-VFP-NEXT: bl __aeabi_h2f
; CHECK-LIBCALL-VFP-NEXT: vmov s0, r0
-; CHECK-LIBCALL-VFP-NEXT: vmov s1, r5
-; CHECK-LIBCALL-VFP-NEXT: vmov s2, r6
-; CHECK-LIBCALL-VFP-NEXT: bl fmaf
-; CHECK-LIBCALL-VFP-NEXT: vmov r0, s0
-; CHECK-LIBCALL-VFP-NEXT: bl __aeabi_f2h
+; CHECK-LIBCALL-VFP-NEXT: vcvt.f64.f32 d16, s18
+; CHECK-LIBCALL-VFP-NEXT: vcvt.f64.f32 d17, s16
+; CHECK-LIBCALL-VFP-NEXT: vcvt.f64.f32 d18, s0
+; CHECK-LIBCALL-VFP-NEXT: vmla.f64 d18, d17, d16
+; CHECK-LIBCALL-VFP-NEXT: vmov r0, r1, d18
+; CHECK-LIBCALL-VFP-NEXT: bl __aeabi_d2h
; CHECK-LIBCALL-VFP-NEXT: strh r0, [r4]
+; CHECK-LIBCALL-VFP-NEXT: vpop {d8, d9}
; CHECK-LIBCALL-VFP-NEXT: pop {r4, r5, r6, pc}
;
; CHECK-NOVFP-LABEL: test_fma:
-; CHECK-NOVFP: .save {r4, r5, r6, lr}
-; CHECK-NOVFP-NEXT: push {r4, r5, r6, lr}
+; CHECK-NOVFP: .save {r4, r5, r6, r7, r11, lr}
+; CHECK-NOVFP-NEXT: push {r4, r5, r6, r7, r11, lr}
; CHECK-NOVFP-NEXT: mov r4, r0
; CHECK-NOVFP-NEXT: ldrh r0, [r1]
; CHECK-NOVFP-NEXT: mov r5, r2
; CHECK-NOVFP-NEXT: bl __aeabi_h2f
+; CHECK-NOVFP-NEXT: bl __aeabi_f2d
; CHECK-NOVFP-NEXT: mov r6, r0
-; CHECK-NOVFP-NEXT: ldrh r0, [r5]
-; CHECK-NOVFP-NEXT: bl __aeabi_h2f
-; CHECK-NOVFP-NEXT: mov r5, r0
; CHECK-NOVFP-NEXT: ldrh r0, [r4]
+; CHECK-NOVFP-NEXT: mov r7, r1
; CHECK-NOVFP-NEXT: bl __aeabi_h2f
-; CHECK-NOVFP-NEXT: mov r1, r6
-; CHECK-NOVFP-NEXT: mov r2, r5
-; CHECK-NOVFP-NEXT: bl fmaf
-; CHECK-NOVFP-NEXT: bl __aeabi_f2h
+; CHECK-NOVFP-NEXT: bl __aeabi_f2d
+; CHECK-NOVFP-NEXT: mov r2, r6
+; CHECK-NOVFP-NEXT: mov r3, r7
+; CHECK-NOVFP-NEXT: bl __aeabi_dmul
+; CHECK-NOVFP-NEXT: mov r6, r0
+; CHECK-NOVFP-NEXT: ldrh r0, [r5]
+; CHECK-NOVFP-NEXT: mov r7, r1
+; CHECK-NOVFP-NEXT: bl __aeabi_h2f
+; CHECK-NOVFP-NEXT: bl __aeabi_f2d
+; CHECK-NOVFP-NEXT: mov r2, r0
+; CHECK-NOVFP-NEXT: mov r3, r1
+; CHECK-NOVFP-NEXT: mov r0, r6
+; CHECK-NOVFP-NEXT: mov r1, r7
+; CHECK-NOVFP-NEXT: bl __aeabi_dadd
+; CHECK-NOVFP-NEXT: bl __aeabi_d2h
; CHECK-NOVFP-NEXT: strh r0, [r4]
-; CHECK-NOVFP-NEXT: pop {r4, r5, r6, pc}
+; CHECK-NOVFP-NEXT: pop {r4, r5, r6, r7, r11, pc}
%a = load half, ptr %p, align 2
%b = load half, ptr %q, align 2
%c = load half, ptr %r, align 2
>From ac049af9f877dbaa5275f6e3c7847e0bd47b6714 Mon Sep 17 00:00:00 2001
From: Folkert de Vries <folkert at folkertdev.nl>
Date: Fri, 12 Dec 2025 14:15:00 +0100
Subject: [PATCH 2/5] respect flags in the fp_extend
---
.../SelectionDAG/LegalizeFloatTypes.cpp | 23 +++++++++----------
1 file changed, 11 insertions(+), 12 deletions(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp
index 51bc335550a16..3d9222ee88104 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp
@@ -3487,6 +3487,7 @@ SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FMAD(SDNode *N) {
SDValue Op0 = GetSoftPromotedHalf(N->getOperand(0));
SDValue Op1 = GetSoftPromotedHalf(N->getOperand(1));
SDValue Op2 = GetSoftPromotedHalf(N->getOperand(2));
+ SDNodeFlags Flags = N->getFlags();
SDLoc dl(N);
// Promote to the larger FP type.
@@ -3498,27 +3499,25 @@ SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FMAD(SDNode *N) {
SDValue Res;
if (NVT == MVT::f32) {
// An f16 fma must go via f64 to prevent double rounding issues.
- SDValue A64 = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Op0);
- SDValue B64 = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Op1);
- SDValue C64 = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Op2);
+ SDValue A64 = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Op0, Flags);
+ SDValue B64 = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Op1, Flags);
+ SDValue C64 = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Op2, Flags);
// Prefer a wide FMA node if available; otherwise expand to mul+add.
SDValue WideRes;
- if (TLI.isOperationLegalOrCustom(ISD::FMA, MVT::f64)) {
- WideRes =
- DAG.getNode(ISD::FMA, dl, MVT::f64, A64, B64, C64, N->getFlags());
+ if (TLI.isFMAFasterThanFMulAndFAdd(DAG.getMachineFunction(), MVT::f64)) {
+ WideRes = DAG.getNode(ISD::FMA, dl, MVT::f64, A64, B64, C64, Flags);
} else {
- SDValue Mul =
- DAG.getNode(ISD::FMUL, dl, MVT::f64, A64, B64, N->getFlags());
- WideRes = DAG.getNode(ISD::FADD, dl, MVT::f64, Mul, C64, N->getFlags());
+ SDValue Mul = DAG.getNode(ISD::FMUL, dl, MVT::f64, A64, B64, Flags);
+ WideRes = DAG.getNode(ISD::FADD, dl, MVT::f64, Mul, C64, Flags);
}
return DAG.getNode(GetPromotionOpcode(MVT::f64, OVT), dl, MVT::i16,
WideRes);
- } else {
- Res = DAG.getNode(N->getOpcode(), dl, NVT, Op0, Op1, Op2, N->getFlags());
- return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res);
}
+
+ Res = DAG.getNode(N->getOpcode(), dl, NVT, Op0, Op1, Op2, Flags);
+ return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res);
}
SDValue DAGTypeLegalizer::SoftPromoteHalfRes_ExpOp(SDNode *N) {
>From 72c5b15e06b7e5e0d571425382fbb4369ac23105 Mon Sep 17 00:00:00 2001
From: Folkert de Vries <folkert at folkertdev.nl>
Date: Fri, 12 Dec 2025 15:27:22 +0100
Subject: [PATCH 3/5] update riscv tests
---
llvm/test/CodeGen/RISCV/half-arith.ll | 366 ++++++++++++++-------
llvm/test/CodeGen/RISCV/half-intrinsics.ll | 44 ++-
2 files changed, 282 insertions(+), 128 deletions(-)
diff --git a/llvm/test/CodeGen/RISCV/half-arith.ll b/llvm/test/CodeGen/RISCV/half-arith.ll
index d089e3678756c..a0a51f4f994dc 100644
--- a/llvm/test/CodeGen/RISCV/half-arith.ll
+++ b/llvm/test/CodeGen/RISCV/half-arith.ll
@@ -1105,28 +1105,41 @@ define half @fmadd_h(half %a, half %b, half %c) nounwind {
; RV32I-NEXT: sw s1, 20(sp) # 4-byte Folded Spill
; RV32I-NEXT: sw s2, 16(sp) # 4-byte Folded Spill
; RV32I-NEXT: sw s3, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s4, 8(sp) # 4-byte Folded Spill
; RV32I-NEXT: mv s0, a2
; RV32I-NEXT: mv s1, a1
; RV32I-NEXT: lui a1, 16
-; RV32I-NEXT: addi s3, a1, -1
-; RV32I-NEXT: and a0, a0, s3
+; RV32I-NEXT: addi s4, a1, -1
+; RV32I-NEXT: and a0, a0, s4
; RV32I-NEXT: call __extendhfsf2
+; RV32I-NEXT: call __extendsfdf2
; RV32I-NEXT: mv s2, a0
-; RV32I-NEXT: and a0, s1, s3
+; RV32I-NEXT: mv s3, a1
+; RV32I-NEXT: and a0, s1, s4
; RV32I-NEXT: call __extendhfsf2
+; RV32I-NEXT: call __extendsfdf2
+; RV32I-NEXT: mv a2, a0
+; RV32I-NEXT: mv a3, a1
+; RV32I-NEXT: mv a0, s2
+; RV32I-NEXT: mv a1, s3
+; RV32I-NEXT: call __muldf3
; RV32I-NEXT: mv s1, a0
-; RV32I-NEXT: and a0, s0, s3
+; RV32I-NEXT: mv s2, a1
+; RV32I-NEXT: and a0, s0, s4
; RV32I-NEXT: call __extendhfsf2
+; RV32I-NEXT: call __extendsfdf2
; RV32I-NEXT: mv a2, a0
-; RV32I-NEXT: mv a0, s2
-; RV32I-NEXT: mv a1, s1
-; RV32I-NEXT: call fmaf
-; RV32I-NEXT: call __truncsfhf2
+; RV32I-NEXT: mv a3, a1
+; RV32I-NEXT: mv a0, s1
+; RV32I-NEXT: mv a1, s2
+; RV32I-NEXT: call __adddf3
+; RV32I-NEXT: call __truncdfhf2
; RV32I-NEXT: lw ra, 28(sp) # 4-byte Folded Reload
; RV32I-NEXT: lw s0, 24(sp) # 4-byte Folded Reload
; RV32I-NEXT: lw s1, 20(sp) # 4-byte Folded Reload
; RV32I-NEXT: lw s2, 16(sp) # 4-byte Folded Reload
; RV32I-NEXT: lw s3, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s4, 8(sp) # 4-byte Folded Reload
; RV32I-NEXT: addi sp, sp, 32
; RV32I-NEXT: ret
;
@@ -1144,17 +1157,22 @@ define half @fmadd_h(half %a, half %b, half %c) nounwind {
; RV64I-NEXT: addi s3, a1, -1
; RV64I-NEXT: and a0, a0, s3
; RV64I-NEXT: call __extendhfsf2
+; RV64I-NEXT: call __extendsfdf2
; RV64I-NEXT: mv s2, a0
; RV64I-NEXT: and a0, s1, s3
; RV64I-NEXT: call __extendhfsf2
+; RV64I-NEXT: call __extendsfdf2
+; RV64I-NEXT: mv a1, a0
+; RV64I-NEXT: mv a0, s2
+; RV64I-NEXT: call __muldf3
; RV64I-NEXT: mv s1, a0
; RV64I-NEXT: and a0, s0, s3
; RV64I-NEXT: call __extendhfsf2
-; RV64I-NEXT: mv a2, a0
-; RV64I-NEXT: mv a0, s2
-; RV64I-NEXT: mv a1, s1
-; RV64I-NEXT: call fmaf
-; RV64I-NEXT: call __truncsfhf2
+; RV64I-NEXT: call __extendsfdf2
+; RV64I-NEXT: mv a1, a0
+; RV64I-NEXT: mv a0, s1
+; RV64I-NEXT: call __adddf3
+; RV64I-NEXT: call __truncdfhf2
; RV64I-NEXT: ld ra, 40(sp) # 8-byte Folded Reload
; RV64I-NEXT: ld s0, 32(sp) # 8-byte Folded Reload
; RV64I-NEXT: ld s1, 24(sp) # 8-byte Folded Reload
@@ -1206,35 +1224,48 @@ define half @fmsub_h(half %a, half %b, half %c) nounwind {
; RV32I-NEXT: sw s1, 20(sp) # 4-byte Folded Spill
; RV32I-NEXT: sw s2, 16(sp) # 4-byte Folded Spill
; RV32I-NEXT: sw s3, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s4, 8(sp) # 4-byte Folded Spill
; RV32I-NEXT: mv s0, a1
; RV32I-NEXT: mv s1, a0
; RV32I-NEXT: lui a0, 16
-; RV32I-NEXT: addi s2, a0, -1
-; RV32I-NEXT: and a0, a2, s2
+; RV32I-NEXT: addi s3, a0, -1
+; RV32I-NEXT: and a0, a2, s3
; RV32I-NEXT: call __extendhfsf2
; RV32I-NEXT: li a1, 0
; RV32I-NEXT: call __addsf3
; RV32I-NEXT: call __truncsfhf2
; RV32I-NEXT: lui a1, 8
-; RV32I-NEXT: xor s3, a0, a1
-; RV32I-NEXT: and a0, s1, s2
+; RV32I-NEXT: xor s4, a0, a1
+; RV32I-NEXT: and a0, s1, s3
; RV32I-NEXT: call __extendhfsf2
+; RV32I-NEXT: call __extendsfdf2
; RV32I-NEXT: mv s1, a0
-; RV32I-NEXT: and a0, s0, s2
+; RV32I-NEXT: mv s2, a1
+; RV32I-NEXT: and a0, s0, s3
; RV32I-NEXT: call __extendhfsf2
+; RV32I-NEXT: call __extendsfdf2
+; RV32I-NEXT: mv a2, a0
+; RV32I-NEXT: mv a3, a1
+; RV32I-NEXT: mv a0, s1
+; RV32I-NEXT: mv a1, s2
+; RV32I-NEXT: call __muldf3
; RV32I-NEXT: mv s0, a0
-; RV32I-NEXT: and a0, s3, s2
+; RV32I-NEXT: mv s1, a1
+; RV32I-NEXT: and a0, s4, s3
; RV32I-NEXT: call __extendhfsf2
+; RV32I-NEXT: call __extendsfdf2
; RV32I-NEXT: mv a2, a0
-; RV32I-NEXT: mv a0, s1
-; RV32I-NEXT: mv a1, s0
-; RV32I-NEXT: call fmaf
-; RV32I-NEXT: call __truncsfhf2
+; RV32I-NEXT: mv a3, a1
+; RV32I-NEXT: mv a0, s0
+; RV32I-NEXT: mv a1, s1
+; RV32I-NEXT: call __adddf3
+; RV32I-NEXT: call __truncdfhf2
; RV32I-NEXT: lw ra, 28(sp) # 4-byte Folded Reload
; RV32I-NEXT: lw s0, 24(sp) # 4-byte Folded Reload
; RV32I-NEXT: lw s1, 20(sp) # 4-byte Folded Reload
; RV32I-NEXT: lw s2, 16(sp) # 4-byte Folded Reload
; RV32I-NEXT: lw s3, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s4, 8(sp) # 4-byte Folded Reload
; RV32I-NEXT: addi sp, sp, 32
; RV32I-NEXT: ret
;
@@ -1259,17 +1290,22 @@ define half @fmsub_h(half %a, half %b, half %c) nounwind {
; RV64I-NEXT: xor s3, a0, a1
; RV64I-NEXT: and a0, s1, s2
; RV64I-NEXT: call __extendhfsf2
+; RV64I-NEXT: call __extendsfdf2
; RV64I-NEXT: mv s1, a0
; RV64I-NEXT: and a0, s0, s2
; RV64I-NEXT: call __extendhfsf2
+; RV64I-NEXT: call __extendsfdf2
+; RV64I-NEXT: mv a1, a0
+; RV64I-NEXT: mv a0, s1
+; RV64I-NEXT: call __muldf3
; RV64I-NEXT: mv s0, a0
; RV64I-NEXT: and a0, s3, s2
; RV64I-NEXT: call __extendhfsf2
-; RV64I-NEXT: mv a2, a0
-; RV64I-NEXT: mv a0, s1
-; RV64I-NEXT: mv a1, s0
-; RV64I-NEXT: call fmaf
-; RV64I-NEXT: call __truncsfhf2
+; RV64I-NEXT: call __extendsfdf2
+; RV64I-NEXT: mv a1, a0
+; RV64I-NEXT: mv a0, s0
+; RV64I-NEXT: call __adddf3
+; RV64I-NEXT: call __truncdfhf2
; RV64I-NEXT: ld ra, 40(sp) # 8-byte Folded Reload
; RV64I-NEXT: ld s0, 32(sp) # 8-byte Folded Reload
; RV64I-NEXT: ld s1, 24(sp) # 8-byte Folded Reload
@@ -1341,8 +1377,8 @@ define half @fnmadd_h(half %a, half %b, half %c) nounwind {
; RV32I-NEXT: sw s4, 8(sp) # 4-byte Folded Spill
; RV32I-NEXT: mv s0, a2
; RV32I-NEXT: mv s1, a1
-; RV32I-NEXT: lui a1, 16
-; RV32I-NEXT: addi s3, a1, -1
+; RV32I-NEXT: lui s3, 16
+; RV32I-NEXT: addi s3, s3, -1
; RV32I-NEXT: and a0, a0, s3
; RV32I-NEXT: call __extendhfsf2
; RV32I-NEXT: li a1, 0
@@ -1359,17 +1395,26 @@ define half @fnmadd_h(half %a, half %b, half %c) nounwind {
; RV32I-NEXT: xor s4, a0, a1
; RV32I-NEXT: and a0, s1, s3
; RV32I-NEXT: call __extendhfsf2
+; RV32I-NEXT: call __extendsfdf2
; RV32I-NEXT: mv s0, a0
+; RV32I-NEXT: mv s1, a1
; RV32I-NEXT: and a0, s2, s3
; RV32I-NEXT: call __extendhfsf2
-; RV32I-NEXT: mv s1, a0
+; RV32I-NEXT: call __extendsfdf2
+; RV32I-NEXT: mv a2, s0
+; RV32I-NEXT: mv a3, s1
+; RV32I-NEXT: call __muldf3
+; RV32I-NEXT: mv s0, a0
+; RV32I-NEXT: mv s1, a1
; RV32I-NEXT: and a0, s4, s3
; RV32I-NEXT: call __extendhfsf2
+; RV32I-NEXT: call __extendsfdf2
; RV32I-NEXT: mv a2, a0
-; RV32I-NEXT: mv a0, s1
-; RV32I-NEXT: mv a1, s0
-; RV32I-NEXT: call fmaf
-; RV32I-NEXT: call __truncsfhf2
+; RV32I-NEXT: mv a3, a1
+; RV32I-NEXT: mv a0, s0
+; RV32I-NEXT: mv a1, s1
+; RV32I-NEXT: call __adddf3
+; RV32I-NEXT: call __truncdfhf2
; RV32I-NEXT: lw ra, 28(sp) # 4-byte Folded Reload
; RV32I-NEXT: lw s0, 24(sp) # 4-byte Folded Reload
; RV32I-NEXT: lw s1, 20(sp) # 4-byte Folded Reload
@@ -1390,8 +1435,8 @@ define half @fnmadd_h(half %a, half %b, half %c) nounwind {
; RV64I-NEXT: sd s4, 0(sp) # 8-byte Folded Spill
; RV64I-NEXT: mv s0, a2
; RV64I-NEXT: mv s1, a1
-; RV64I-NEXT: lui a1, 16
-; RV64I-NEXT: addi s3, a1, -1
+; RV64I-NEXT: lui s3, 16
+; RV64I-NEXT: addi s3, s3, -1
; RV64I-NEXT: and a0, a0, s3
; RV64I-NEXT: call __extendhfsf2
; RV64I-NEXT: li a1, 0
@@ -1408,17 +1453,21 @@ define half @fnmadd_h(half %a, half %b, half %c) nounwind {
; RV64I-NEXT: xor s4, a0, a1
; RV64I-NEXT: and a0, s1, s3
; RV64I-NEXT: call __extendhfsf2
+; RV64I-NEXT: call __extendsfdf2
; RV64I-NEXT: mv s0, a0
; RV64I-NEXT: and a0, s2, s3
; RV64I-NEXT: call __extendhfsf2
-; RV64I-NEXT: mv s1, a0
+; RV64I-NEXT: call __extendsfdf2
+; RV64I-NEXT: mv a1, s0
+; RV64I-NEXT: call __muldf3
+; RV64I-NEXT: mv s0, a0
; RV64I-NEXT: and a0, s4, s3
; RV64I-NEXT: call __extendhfsf2
-; RV64I-NEXT: mv a2, a0
-; RV64I-NEXT: mv a0, s1
-; RV64I-NEXT: mv a1, s0
-; RV64I-NEXT: call fmaf
-; RV64I-NEXT: call __truncsfhf2
+; RV64I-NEXT: call __extendsfdf2
+; RV64I-NEXT: mv a1, a0
+; RV64I-NEXT: mv a0, s0
+; RV64I-NEXT: call __adddf3
+; RV64I-NEXT: call __truncdfhf2
; RV64I-NEXT: ld ra, 40(sp) # 8-byte Folded Reload
; RV64I-NEXT: ld s0, 32(sp) # 8-byte Folded Reload
; RV64I-NEXT: ld s1, 24(sp) # 8-byte Folded Reload
@@ -1503,8 +1552,8 @@ define half @fnmadd_h_2(half %a, half %b, half %c) nounwind {
; RV32I-NEXT: sw s4, 8(sp) # 4-byte Folded Spill
; RV32I-NEXT: mv s0, a2
; RV32I-NEXT: mv s1, a0
-; RV32I-NEXT: lui a0, 16
-; RV32I-NEXT: addi s3, a0, -1
+; RV32I-NEXT: lui s3, 16
+; RV32I-NEXT: addi s3, s3, -1
; RV32I-NEXT: and a0, a1, s3
; RV32I-NEXT: call __extendhfsf2
; RV32I-NEXT: li a1, 0
@@ -1521,17 +1570,28 @@ define half @fnmadd_h_2(half %a, half %b, half %c) nounwind {
; RV32I-NEXT: xor s4, a0, a1
; RV32I-NEXT: and a0, s1, s3
; RV32I-NEXT: call __extendhfsf2
+; RV32I-NEXT: call __extendsfdf2
; RV32I-NEXT: mv s0, a0
+; RV32I-NEXT: mv s1, a1
; RV32I-NEXT: and a0, s2, s3
; RV32I-NEXT: call __extendhfsf2
-; RV32I-NEXT: mv s1, a0
+; RV32I-NEXT: call __extendsfdf2
+; RV32I-NEXT: mv a2, a0
+; RV32I-NEXT: mv a3, a1
+; RV32I-NEXT: mv a0, s0
+; RV32I-NEXT: mv a1, s1
+; RV32I-NEXT: call __muldf3
+; RV32I-NEXT: mv s0, a0
+; RV32I-NEXT: mv s1, a1
; RV32I-NEXT: and a0, s4, s3
; RV32I-NEXT: call __extendhfsf2
+; RV32I-NEXT: call __extendsfdf2
; RV32I-NEXT: mv a2, a0
+; RV32I-NEXT: mv a3, a1
; RV32I-NEXT: mv a0, s0
; RV32I-NEXT: mv a1, s1
-; RV32I-NEXT: call fmaf
-; RV32I-NEXT: call __truncsfhf2
+; RV32I-NEXT: call __adddf3
+; RV32I-NEXT: call __truncdfhf2
; RV32I-NEXT: lw ra, 28(sp) # 4-byte Folded Reload
; RV32I-NEXT: lw s0, 24(sp) # 4-byte Folded Reload
; RV32I-NEXT: lw s1, 20(sp) # 4-byte Folded Reload
@@ -1552,8 +1612,8 @@ define half @fnmadd_h_2(half %a, half %b, half %c) nounwind {
; RV64I-NEXT: sd s4, 0(sp) # 8-byte Folded Spill
; RV64I-NEXT: mv s0, a2
; RV64I-NEXT: mv s1, a0
-; RV64I-NEXT: lui a0, 16
-; RV64I-NEXT: addi s3, a0, -1
+; RV64I-NEXT: lui s3, 16
+; RV64I-NEXT: addi s3, s3, -1
; RV64I-NEXT: and a0, a1, s3
; RV64I-NEXT: call __extendhfsf2
; RV64I-NEXT: li a1, 0
@@ -1570,17 +1630,22 @@ define half @fnmadd_h_2(half %a, half %b, half %c) nounwind {
; RV64I-NEXT: xor s4, a0, a1
; RV64I-NEXT: and a0, s1, s3
; RV64I-NEXT: call __extendhfsf2
+; RV64I-NEXT: call __extendsfdf2
; RV64I-NEXT: mv s0, a0
; RV64I-NEXT: and a0, s2, s3
; RV64I-NEXT: call __extendhfsf2
-; RV64I-NEXT: mv s1, a0
+; RV64I-NEXT: call __extendsfdf2
+; RV64I-NEXT: mv a1, a0
+; RV64I-NEXT: mv a0, s0
+; RV64I-NEXT: call __muldf3
+; RV64I-NEXT: mv s0, a0
; RV64I-NEXT: and a0, s4, s3
; RV64I-NEXT: call __extendhfsf2
-; RV64I-NEXT: mv a2, a0
+; RV64I-NEXT: call __extendsfdf2
+; RV64I-NEXT: mv a1, a0
; RV64I-NEXT: mv a0, s0
-; RV64I-NEXT: mv a1, s1
-; RV64I-NEXT: call fmaf
-; RV64I-NEXT: call __truncsfhf2
+; RV64I-NEXT: call __adddf3
+; RV64I-NEXT: call __truncdfhf2
; RV64I-NEXT: ld ra, 40(sp) # 8-byte Folded Reload
; RV64I-NEXT: ld s0, 32(sp) # 8-byte Folded Reload
; RV64I-NEXT: ld s1, 24(sp) # 8-byte Folded Reload
@@ -1671,23 +1736,35 @@ define half @fnmadd_h_3(half %a, half %b, half %c) nounwind {
; RV32I-NEXT: sw s1, 20(sp) # 4-byte Folded Spill
; RV32I-NEXT: sw s2, 16(sp) # 4-byte Folded Spill
; RV32I-NEXT: sw s3, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s4, 8(sp) # 4-byte Folded Spill
; RV32I-NEXT: mv s0, a2
; RV32I-NEXT: mv s1, a1
; RV32I-NEXT: lui a1, 16
-; RV32I-NEXT: addi s3, a1, -1
-; RV32I-NEXT: and a0, a0, s3
+; RV32I-NEXT: addi s4, a1, -1
+; RV32I-NEXT: and a0, a0, s4
; RV32I-NEXT: call __extendhfsf2
+; RV32I-NEXT: call __extendsfdf2
; RV32I-NEXT: mv s2, a0
-; RV32I-NEXT: and a0, s1, s3
+; RV32I-NEXT: mv s3, a1
+; RV32I-NEXT: and a0, s1, s4
; RV32I-NEXT: call __extendhfsf2
+; RV32I-NEXT: call __extendsfdf2
+; RV32I-NEXT: mv a2, a0
+; RV32I-NEXT: mv a3, a1
+; RV32I-NEXT: mv a0, s2
+; RV32I-NEXT: mv a1, s3
+; RV32I-NEXT: call __muldf3
; RV32I-NEXT: mv s1, a0
-; RV32I-NEXT: and a0, s0, s3
+; RV32I-NEXT: mv s2, a1
+; RV32I-NEXT: and a0, s0, s4
; RV32I-NEXT: call __extendhfsf2
+; RV32I-NEXT: call __extendsfdf2
; RV32I-NEXT: mv a2, a0
-; RV32I-NEXT: mv a0, s2
-; RV32I-NEXT: mv a1, s1
-; RV32I-NEXT: call fmaf
-; RV32I-NEXT: call __truncsfhf2
+; RV32I-NEXT: mv a3, a1
+; RV32I-NEXT: mv a0, s1
+; RV32I-NEXT: mv a1, s2
+; RV32I-NEXT: call __adddf3
+; RV32I-NEXT: call __truncdfhf2
; RV32I-NEXT: lui a1, 1048568
; RV32I-NEXT: xor a0, a0, a1
; RV32I-NEXT: lw ra, 28(sp) # 4-byte Folded Reload
@@ -1695,6 +1772,7 @@ define half @fnmadd_h_3(half %a, half %b, half %c) nounwind {
; RV32I-NEXT: lw s1, 20(sp) # 4-byte Folded Reload
; RV32I-NEXT: lw s2, 16(sp) # 4-byte Folded Reload
; RV32I-NEXT: lw s3, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s4, 8(sp) # 4-byte Folded Reload
; RV32I-NEXT: addi sp, sp, 32
; RV32I-NEXT: ret
;
@@ -1712,17 +1790,22 @@ define half @fnmadd_h_3(half %a, half %b, half %c) nounwind {
; RV64I-NEXT: addi s3, a1, -1
; RV64I-NEXT: and a0, a0, s3
; RV64I-NEXT: call __extendhfsf2
+; RV64I-NEXT: call __extendsfdf2
; RV64I-NEXT: mv s2, a0
; RV64I-NEXT: and a0, s1, s3
; RV64I-NEXT: call __extendhfsf2
+; RV64I-NEXT: call __extendsfdf2
+; RV64I-NEXT: mv a1, a0
+; RV64I-NEXT: mv a0, s2
+; RV64I-NEXT: call __muldf3
; RV64I-NEXT: mv s1, a0
; RV64I-NEXT: and a0, s0, s3
; RV64I-NEXT: call __extendhfsf2
-; RV64I-NEXT: mv a2, a0
-; RV64I-NEXT: mv a0, s2
-; RV64I-NEXT: mv a1, s1
-; RV64I-NEXT: call fmaf
-; RV64I-NEXT: call __truncsfhf2
+; RV64I-NEXT: call __extendsfdf2
+; RV64I-NEXT: mv a1, a0
+; RV64I-NEXT: mv a0, s1
+; RV64I-NEXT: call __adddf3
+; RV64I-NEXT: call __truncdfhf2
; RV64I-NEXT: lui a1, 1048568
; RV64I-NEXT: xor a0, a0, a1
; RV64I-NEXT: ld ra, 40(sp) # 8-byte Folded Reload
@@ -1792,23 +1875,35 @@ define half @fnmadd_nsz(half %a, half %b, half %c) nounwind {
; RV32I-NEXT: sw s1, 20(sp) # 4-byte Folded Spill
; RV32I-NEXT: sw s2, 16(sp) # 4-byte Folded Spill
; RV32I-NEXT: sw s3, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s4, 8(sp) # 4-byte Folded Spill
; RV32I-NEXT: mv s0, a2
; RV32I-NEXT: mv s1, a1
; RV32I-NEXT: lui a1, 16
-; RV32I-NEXT: addi s3, a1, -1
-; RV32I-NEXT: and a0, a0, s3
+; RV32I-NEXT: addi s4, a1, -1
+; RV32I-NEXT: and a0, a0, s4
; RV32I-NEXT: call __extendhfsf2
+; RV32I-NEXT: call __extendsfdf2
; RV32I-NEXT: mv s2, a0
-; RV32I-NEXT: and a0, s1, s3
+; RV32I-NEXT: mv s3, a1
+; RV32I-NEXT: and a0, s1, s4
; RV32I-NEXT: call __extendhfsf2
+; RV32I-NEXT: call __extendsfdf2
+; RV32I-NEXT: mv a2, a0
+; RV32I-NEXT: mv a3, a1
+; RV32I-NEXT: mv a0, s2
+; RV32I-NEXT: mv a1, s3
+; RV32I-NEXT: call __muldf3
; RV32I-NEXT: mv s1, a0
-; RV32I-NEXT: and a0, s0, s3
+; RV32I-NEXT: mv s2, a1
+; RV32I-NEXT: and a0, s0, s4
; RV32I-NEXT: call __extendhfsf2
+; RV32I-NEXT: call __extendsfdf2
; RV32I-NEXT: mv a2, a0
-; RV32I-NEXT: mv a0, s2
-; RV32I-NEXT: mv a1, s1
-; RV32I-NEXT: call fmaf
-; RV32I-NEXT: call __truncsfhf2
+; RV32I-NEXT: mv a3, a1
+; RV32I-NEXT: mv a0, s1
+; RV32I-NEXT: mv a1, s2
+; RV32I-NEXT: call __adddf3
+; RV32I-NEXT: call __truncdfhf2
; RV32I-NEXT: lui a1, 1048568
; RV32I-NEXT: xor a0, a0, a1
; RV32I-NEXT: lw ra, 28(sp) # 4-byte Folded Reload
@@ -1816,6 +1911,7 @@ define half @fnmadd_nsz(half %a, half %b, half %c) nounwind {
; RV32I-NEXT: lw s1, 20(sp) # 4-byte Folded Reload
; RV32I-NEXT: lw s2, 16(sp) # 4-byte Folded Reload
; RV32I-NEXT: lw s3, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s4, 8(sp) # 4-byte Folded Reload
; RV32I-NEXT: addi sp, sp, 32
; RV32I-NEXT: ret
;
@@ -1833,17 +1929,22 @@ define half @fnmadd_nsz(half %a, half %b, half %c) nounwind {
; RV64I-NEXT: addi s3, a1, -1
; RV64I-NEXT: and a0, a0, s3
; RV64I-NEXT: call __extendhfsf2
+; RV64I-NEXT: call __extendsfdf2
; RV64I-NEXT: mv s2, a0
; RV64I-NEXT: and a0, s1, s3
; RV64I-NEXT: call __extendhfsf2
+; RV64I-NEXT: call __extendsfdf2
+; RV64I-NEXT: mv a1, a0
+; RV64I-NEXT: mv a0, s2
+; RV64I-NEXT: call __muldf3
; RV64I-NEXT: mv s1, a0
; RV64I-NEXT: and a0, s0, s3
; RV64I-NEXT: call __extendhfsf2
-; RV64I-NEXT: mv a2, a0
-; RV64I-NEXT: mv a0, s2
-; RV64I-NEXT: mv a1, s1
-; RV64I-NEXT: call fmaf
-; RV64I-NEXT: call __truncsfhf2
+; RV64I-NEXT: call __extendsfdf2
+; RV64I-NEXT: mv a1, a0
+; RV64I-NEXT: mv a0, s1
+; RV64I-NEXT: call __adddf3
+; RV64I-NEXT: call __truncdfhf2
; RV64I-NEXT: lui a1, 1048568
; RV64I-NEXT: xor a0, a0, a1
; RV64I-NEXT: ld ra, 40(sp) # 8-byte Folded Reload
@@ -1905,34 +2006,46 @@ define half @fnmsub_h(half %a, half %b, half %c) nounwind {
; RV32I-NEXT: sw s1, 20(sp) # 4-byte Folded Spill
; RV32I-NEXT: sw s2, 16(sp) # 4-byte Folded Spill
; RV32I-NEXT: sw s3, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s4, 8(sp) # 4-byte Folded Spill
; RV32I-NEXT: mv s0, a2
; RV32I-NEXT: mv s1, a1
; RV32I-NEXT: lui a1, 16
-; RV32I-NEXT: addi s2, a1, -1
-; RV32I-NEXT: and a0, a0, s2
+; RV32I-NEXT: addi s3, a1, -1
+; RV32I-NEXT: and a0, a0, s3
; RV32I-NEXT: call __extendhfsf2
; RV32I-NEXT: li a1, 0
; RV32I-NEXT: call __addsf3
; RV32I-NEXT: call __truncsfhf2
; RV32I-NEXT: lui a1, 8
-; RV32I-NEXT: xor s3, a0, a1
-; RV32I-NEXT: and a0, s1, s2
+; RV32I-NEXT: xor s4, a0, a1
+; RV32I-NEXT: and a0, s1, s3
; RV32I-NEXT: call __extendhfsf2
+; RV32I-NEXT: call __extendsfdf2
; RV32I-NEXT: mv s1, a0
-; RV32I-NEXT: and a0, s0, s2
+; RV32I-NEXT: mv s2, a1
+; RV32I-NEXT: and a0, s4, s3
; RV32I-NEXT: call __extendhfsf2
-; RV32I-NEXT: mv s0, a0
-; RV32I-NEXT: and a0, s3, s2
+; RV32I-NEXT: call __extendsfdf2
+; RV32I-NEXT: mv a2, s1
+; RV32I-NEXT: mv a3, s2
+; RV32I-NEXT: call __muldf3
+; RV32I-NEXT: mv s1, a0
+; RV32I-NEXT: mv s2, a1
+; RV32I-NEXT: and a0, s0, s3
; RV32I-NEXT: call __extendhfsf2
-; RV32I-NEXT: mv a1, s1
-; RV32I-NEXT: mv a2, s0
-; RV32I-NEXT: call fmaf
-; RV32I-NEXT: call __truncsfhf2
+; RV32I-NEXT: call __extendsfdf2
+; RV32I-NEXT: mv a2, a0
+; RV32I-NEXT: mv a3, a1
+; RV32I-NEXT: mv a0, s1
+; RV32I-NEXT: mv a1, s2
+; RV32I-NEXT: call __adddf3
+; RV32I-NEXT: call __truncdfhf2
; RV32I-NEXT: lw ra, 28(sp) # 4-byte Folded Reload
; RV32I-NEXT: lw s0, 24(sp) # 4-byte Folded Reload
; RV32I-NEXT: lw s1, 20(sp) # 4-byte Folded Reload
; RV32I-NEXT: lw s2, 16(sp) # 4-byte Folded Reload
; RV32I-NEXT: lw s3, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s4, 8(sp) # 4-byte Folded Reload
; RV32I-NEXT: addi sp, sp, 32
; RV32I-NEXT: ret
;
@@ -1957,16 +2070,21 @@ define half @fnmsub_h(half %a, half %b, half %c) nounwind {
; RV64I-NEXT: xor s3, a0, a1
; RV64I-NEXT: and a0, s1, s2
; RV64I-NEXT: call __extendhfsf2
+; RV64I-NEXT: call __extendsfdf2
; RV64I-NEXT: mv s1, a0
-; RV64I-NEXT: and a0, s0, s2
-; RV64I-NEXT: call __extendhfsf2
-; RV64I-NEXT: mv s0, a0
; RV64I-NEXT: and a0, s3, s2
; RV64I-NEXT: call __extendhfsf2
+; RV64I-NEXT: call __extendsfdf2
; RV64I-NEXT: mv a1, s1
-; RV64I-NEXT: mv a2, s0
-; RV64I-NEXT: call fmaf
-; RV64I-NEXT: call __truncsfhf2
+; RV64I-NEXT: call __muldf3
+; RV64I-NEXT: mv s1, a0
+; RV64I-NEXT: and a0, s0, s2
+; RV64I-NEXT: call __extendhfsf2
+; RV64I-NEXT: call __extendsfdf2
+; RV64I-NEXT: mv a1, a0
+; RV64I-NEXT: mv a0, s1
+; RV64I-NEXT: call __adddf3
+; RV64I-NEXT: call __truncdfhf2
; RV64I-NEXT: ld ra, 40(sp) # 8-byte Folded Reload
; RV64I-NEXT: ld s0, 32(sp) # 8-byte Folded Reload
; RV64I-NEXT: ld s1, 24(sp) # 8-byte Folded Reload
@@ -2033,35 +2151,48 @@ define half @fnmsub_h_2(half %a, half %b, half %c) nounwind {
; RV32I-NEXT: sw s1, 20(sp) # 4-byte Folded Spill
; RV32I-NEXT: sw s2, 16(sp) # 4-byte Folded Spill
; RV32I-NEXT: sw s3, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s4, 8(sp) # 4-byte Folded Spill
; RV32I-NEXT: mv s0, a2
; RV32I-NEXT: mv s1, a0
; RV32I-NEXT: lui a0, 16
-; RV32I-NEXT: addi s2, a0, -1
-; RV32I-NEXT: and a0, a1, s2
+; RV32I-NEXT: addi s3, a0, -1
+; RV32I-NEXT: and a0, a1, s3
; RV32I-NEXT: call __extendhfsf2
; RV32I-NEXT: li a1, 0
; RV32I-NEXT: call __addsf3
; RV32I-NEXT: call __truncsfhf2
; RV32I-NEXT: lui a1, 8
-; RV32I-NEXT: xor s3, a0, a1
-; RV32I-NEXT: and a0, s1, s2
+; RV32I-NEXT: xor s4, a0, a1
+; RV32I-NEXT: and a0, s1, s3
; RV32I-NEXT: call __extendhfsf2
+; RV32I-NEXT: call __extendsfdf2
; RV32I-NEXT: mv s1, a0
-; RV32I-NEXT: and a0, s0, s2
+; RV32I-NEXT: mv s2, a1
+; RV32I-NEXT: and a0, s4, s3
; RV32I-NEXT: call __extendhfsf2
-; RV32I-NEXT: mv s0, a0
-; RV32I-NEXT: and a0, s3, s2
+; RV32I-NEXT: call __extendsfdf2
+; RV32I-NEXT: mv a2, a0
+; RV32I-NEXT: mv a3, a1
+; RV32I-NEXT: mv a0, s1
+; RV32I-NEXT: mv a1, s2
+; RV32I-NEXT: call __muldf3
+; RV32I-NEXT: mv s1, a0
+; RV32I-NEXT: mv s2, a1
+; RV32I-NEXT: and a0, s0, s3
; RV32I-NEXT: call __extendhfsf2
-; RV32I-NEXT: mv a1, a0
+; RV32I-NEXT: call __extendsfdf2
+; RV32I-NEXT: mv a2, a0
+; RV32I-NEXT: mv a3, a1
; RV32I-NEXT: mv a0, s1
-; RV32I-NEXT: mv a2, s0
-; RV32I-NEXT: call fmaf
-; RV32I-NEXT: call __truncsfhf2
+; RV32I-NEXT: mv a1, s2
+; RV32I-NEXT: call __adddf3
+; RV32I-NEXT: call __truncdfhf2
; RV32I-NEXT: lw ra, 28(sp) # 4-byte Folded Reload
; RV32I-NEXT: lw s0, 24(sp) # 4-byte Folded Reload
; RV32I-NEXT: lw s1, 20(sp) # 4-byte Folded Reload
; RV32I-NEXT: lw s2, 16(sp) # 4-byte Folded Reload
; RV32I-NEXT: lw s3, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s4, 8(sp) # 4-byte Folded Reload
; RV32I-NEXT: addi sp, sp, 32
; RV32I-NEXT: ret
;
@@ -2086,17 +2217,22 @@ define half @fnmsub_h_2(half %a, half %b, half %c) nounwind {
; RV64I-NEXT: xor s3, a0, a1
; RV64I-NEXT: and a0, s1, s2
; RV64I-NEXT: call __extendhfsf2
+; RV64I-NEXT: call __extendsfdf2
; RV64I-NEXT: mv s1, a0
-; RV64I-NEXT: and a0, s0, s2
-; RV64I-NEXT: call __extendhfsf2
-; RV64I-NEXT: mv s0, a0
; RV64I-NEXT: and a0, s3, s2
; RV64I-NEXT: call __extendhfsf2
+; RV64I-NEXT: call __extendsfdf2
; RV64I-NEXT: mv a1, a0
; RV64I-NEXT: mv a0, s1
-; RV64I-NEXT: mv a2, s0
-; RV64I-NEXT: call fmaf
-; RV64I-NEXT: call __truncsfhf2
+; RV64I-NEXT: call __muldf3
+; RV64I-NEXT: mv s1, a0
+; RV64I-NEXT: and a0, s0, s2
+; RV64I-NEXT: call __extendhfsf2
+; RV64I-NEXT: call __extendsfdf2
+; RV64I-NEXT: mv a1, a0
+; RV64I-NEXT: mv a0, s1
+; RV64I-NEXT: call __adddf3
+; RV64I-NEXT: call __truncdfhf2
; RV64I-NEXT: ld ra, 40(sp) # 8-byte Folded Reload
; RV64I-NEXT: ld s0, 32(sp) # 8-byte Folded Reload
; RV64I-NEXT: ld s1, 24(sp) # 8-byte Folded Reload
diff --git a/llvm/test/CodeGen/RISCV/half-intrinsics.ll b/llvm/test/CodeGen/RISCV/half-intrinsics.ll
index 847054d96968a..10bcc444b5d77 100644
--- a/llvm/test/CodeGen/RISCV/half-intrinsics.ll
+++ b/llvm/test/CodeGen/RISCV/half-intrinsics.ll
@@ -1712,28 +1712,41 @@ define half @fma_f16(half %a, half %b, half %c) nounwind {
; RV32I-NEXT: sw s1, 20(sp) # 4-byte Folded Spill
; RV32I-NEXT: sw s2, 16(sp) # 4-byte Folded Spill
; RV32I-NEXT: sw s3, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT: sw s4, 8(sp) # 4-byte Folded Spill
; RV32I-NEXT: mv s0, a2
; RV32I-NEXT: mv s1, a1
; RV32I-NEXT: lui a1, 16
-; RV32I-NEXT: addi s3, a1, -1
-; RV32I-NEXT: and a0, a0, s3
+; RV32I-NEXT: addi s4, a1, -1
+; RV32I-NEXT: and a0, a0, s4
; RV32I-NEXT: call __extendhfsf2
+; RV32I-NEXT: call __extendsfdf2
; RV32I-NEXT: mv s2, a0
-; RV32I-NEXT: and a0, s1, s3
+; RV32I-NEXT: mv s3, a1
+; RV32I-NEXT: and a0, s1, s4
; RV32I-NEXT: call __extendhfsf2
+; RV32I-NEXT: call __extendsfdf2
+; RV32I-NEXT: mv a2, a0
+; RV32I-NEXT: mv a3, a1
+; RV32I-NEXT: mv a0, s2
+; RV32I-NEXT: mv a1, s3
+; RV32I-NEXT: call __muldf3
; RV32I-NEXT: mv s1, a0
-; RV32I-NEXT: and a0, s0, s3
+; RV32I-NEXT: mv s2, a1
+; RV32I-NEXT: and a0, s0, s4
; RV32I-NEXT: call __extendhfsf2
+; RV32I-NEXT: call __extendsfdf2
; RV32I-NEXT: mv a2, a0
-; RV32I-NEXT: mv a0, s2
-; RV32I-NEXT: mv a1, s1
-; RV32I-NEXT: call fmaf
-; RV32I-NEXT: call __truncsfhf2
+; RV32I-NEXT: mv a3, a1
+; RV32I-NEXT: mv a0, s1
+; RV32I-NEXT: mv a1, s2
+; RV32I-NEXT: call __adddf3
+; RV32I-NEXT: call __truncdfhf2
; RV32I-NEXT: lw ra, 28(sp) # 4-byte Folded Reload
; RV32I-NEXT: lw s0, 24(sp) # 4-byte Folded Reload
; RV32I-NEXT: lw s1, 20(sp) # 4-byte Folded Reload
; RV32I-NEXT: lw s2, 16(sp) # 4-byte Folded Reload
; RV32I-NEXT: lw s3, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT: lw s4, 8(sp) # 4-byte Folded Reload
; RV32I-NEXT: addi sp, sp, 32
; RV32I-NEXT: ret
;
@@ -1751,17 +1764,22 @@ define half @fma_f16(half %a, half %b, half %c) nounwind {
; RV64I-NEXT: addi s3, a1, -1
; RV64I-NEXT: and a0, a0, s3
; RV64I-NEXT: call __extendhfsf2
+; RV64I-NEXT: call __extendsfdf2
; RV64I-NEXT: mv s2, a0
; RV64I-NEXT: and a0, s1, s3
; RV64I-NEXT: call __extendhfsf2
+; RV64I-NEXT: call __extendsfdf2
+; RV64I-NEXT: mv a1, a0
+; RV64I-NEXT: mv a0, s2
+; RV64I-NEXT: call __muldf3
; RV64I-NEXT: mv s1, a0
; RV64I-NEXT: and a0, s0, s3
; RV64I-NEXT: call __extendhfsf2
-; RV64I-NEXT: mv a2, a0
-; RV64I-NEXT: mv a0, s2
-; RV64I-NEXT: mv a1, s1
-; RV64I-NEXT: call fmaf
-; RV64I-NEXT: call __truncsfhf2
+; RV64I-NEXT: call __extendsfdf2
+; RV64I-NEXT: mv a1, a0
+; RV64I-NEXT: mv a0, s1
+; RV64I-NEXT: call __adddf3
+; RV64I-NEXT: call __truncdfhf2
; RV64I-NEXT: ld ra, 40(sp) # 8-byte Folded Reload
; RV64I-NEXT: ld s0, 32(sp) # 8-byte Folded Reload
; RV64I-NEXT: ld s1, 24(sp) # 8-byte Folded Reload
>From 61c3b85ab9e2029aec5b06df52ba66a8fdfb070d Mon Sep 17 00:00:00 2001
From: Folkert de Vries <folkert at folkertdev.nl>
Date: Fri, 12 Dec 2025 15:28:59 +0100
Subject: [PATCH 4/5] stop considering bf16 and update tests
---
.../SelectionDAG/LegalizeFloatTypes.cpp | 2 +-
llvm/test/CodeGen/AMDGPU/bf16.ll | 393 ++++++++++--------
llvm/test/CodeGen/Generic/half-op.ll | 32 +-
3 files changed, 232 insertions(+), 195 deletions(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp
index 3d9222ee88104..ec844adb4e46e 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp
@@ -3497,7 +3497,7 @@ SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FMAD(SDNode *N) {
Op2 = DAG.getNode(PromotionOpcode, dl, NVT, Op2);
SDValue Res;
- if (NVT == MVT::f32) {
+ if (OVT == MVT::f16) {
// An f16 fma must go via f64 to prevent double rounding issues.
SDValue A64 = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Op0, Flags);
SDValue B64 = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Op1, Flags);
diff --git a/llvm/test/CodeGen/AMDGPU/bf16.ll b/llvm/test/CodeGen/AMDGPU/bf16.ll
index 711d57baac15f..6b0424e50983e 100644
--- a/llvm/test/CodeGen/AMDGPU/bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/bf16.ll
@@ -968,14 +968,14 @@ define <64 x bfloat> @v_load_global_v64bf16(ptr addrspace(1) %ptr) {
; GFX8-NEXT: v_addc_u32_e32 v25, vcc, 0, v29, vcc
; GFX8-NEXT: s_movk_i32 s4, 0x70
; GFX8-NEXT: flat_load_dwordx4 v[0:3], v[28:29]
-; GFX8-NEXT: flat_load_dwordx4 v[12:15], v[12:13]
; GFX8-NEXT: v_add_u32_e32 v28, vcc, s4, v28
; GFX8-NEXT: v_addc_u32_e32 v29, vcc, 0, v29, vcc
; GFX8-NEXT: flat_load_dwordx4 v[4:7], v[4:5]
; GFX8-NEXT: flat_load_dwordx4 v[8:11], v[8:9]
+; GFX8-NEXT: flat_load_dwordx4 v[12:15], v[12:13]
+; GFX8-NEXT: flat_load_dwordx4 v[24:27], v[24:25]
; GFX8-NEXT: flat_load_dwordx4 v[16:19], v[16:17]
; GFX8-NEXT: flat_load_dwordx4 v[20:23], v[20:21]
-; GFX8-NEXT: flat_load_dwordx4 v[24:27], v[24:25]
; GFX8-NEXT: flat_load_dwordx4 v[28:31], v[28:29]
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: s_setpc_b64 s[30:31]
@@ -2520,6 +2520,7 @@ define void @test_load_store_f32_to_bf16(ptr addrspace(1) %in, ptr addrspace(1)
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: global_load_b32 v0, v[0:1], off
; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
; GFX1250-NEXT: global_store_b16 v[2:3], v0, off
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
@@ -2783,6 +2784,7 @@ define void @test_load_store_bf16_to_f32(ptr addrspace(1) %in, ptr addrspace(1)
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: global_load_u16 v0, v[0:1], off
; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX1250-NEXT: global_store_b32 v[2:3], v0, off
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
@@ -2872,6 +2874,7 @@ define void @test_load_store_bf16_to_f64(ptr addrspace(1) %in, ptr addrspace(1)
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: global_load_u16 v0, v[0:1], off
; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
@@ -4527,7 +4530,7 @@ define void @test_call(bfloat %in, ptr addrspace(5) %out) {
; GFX10-NEXT: s_mov_b32 s33, s32
; GFX10-NEXT: s_xor_saveexec_b32 s16, -1
; GFX10-NEXT: buffer_store_dword v2, off, s[0:3], s33 ; 4-byte Folded Spill
-; GFX10-NEXT: s_waitcnt_depctr 0xffe3
+; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
; GFX10-NEXT: s_mov_b32 exec_lo, s16
; GFX10-NEXT: s_addk_i32 s32, 0x200
; GFX10-NEXT: s_getpc_b64 s[16:17]
@@ -4545,7 +4548,7 @@ define void @test_call(bfloat %in, ptr addrspace(5) %out) {
; GFX10-NEXT: s_mov_b32 s32, s33
; GFX10-NEXT: s_xor_saveexec_b32 s4, -1
; GFX10-NEXT: buffer_load_dword v2, off, s[0:3], s33 ; 4-byte Folded Reload
-; GFX10-NEXT: s_waitcnt_depctr 0xffe3
+; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
; GFX10-NEXT: s_mov_b32 exec_lo, s4
; GFX10-NEXT: s_mov_b32 s33, s18
; GFX10-NEXT: s_waitcnt vmcnt(0)
@@ -4785,7 +4788,7 @@ define void @test_call_v2bf16(<2 x bfloat> %in, ptr addrspace(5) %out) {
; GFX10-NEXT: s_mov_b32 s33, s32
; GFX10-NEXT: s_xor_saveexec_b32 s16, -1
; GFX10-NEXT: buffer_store_dword v2, off, s[0:3], s33 ; 4-byte Folded Spill
-; GFX10-NEXT: s_waitcnt_depctr 0xffe3
+; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
; GFX10-NEXT: s_mov_b32 exec_lo, s16
; GFX10-NEXT: s_addk_i32 s32, 0x200
; GFX10-NEXT: s_getpc_b64 s[16:17]
@@ -4803,7 +4806,7 @@ define void @test_call_v2bf16(<2 x bfloat> %in, ptr addrspace(5) %out) {
; GFX10-NEXT: s_mov_b32 s32, s33
; GFX10-NEXT: s_xor_saveexec_b32 s4, -1
; GFX10-NEXT: buffer_load_dword v2, off, s[0:3], s33 ; 4-byte Folded Reload
-; GFX10-NEXT: s_waitcnt_depctr 0xffe3
+; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
; GFX10-NEXT: s_mov_b32 exec_lo, s4
; GFX10-NEXT: s_mov_b32 s33, s18
; GFX10-NEXT: s_waitcnt vmcnt(0)
@@ -5055,7 +5058,7 @@ define void @test_call_v3bf16(<3 x bfloat> %in, ptr addrspace(5) %out) {
; GFX10-NEXT: s_mov_b32 s33, s32
; GFX10-NEXT: s_xor_saveexec_b32 s16, -1
; GFX10-NEXT: buffer_store_dword v3, off, s[0:3], s33 ; 4-byte Folded Spill
-; GFX10-NEXT: s_waitcnt_depctr 0xffe3
+; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
; GFX10-NEXT: s_mov_b32 exec_lo, s16
; GFX10-NEXT: s_addk_i32 s32, 0x200
; GFX10-NEXT: s_getpc_b64 s[16:17]
@@ -5075,7 +5078,7 @@ define void @test_call_v3bf16(<3 x bfloat> %in, ptr addrspace(5) %out) {
; GFX10-NEXT: s_mov_b32 s32, s33
; GFX10-NEXT: s_xor_saveexec_b32 s4, -1
; GFX10-NEXT: buffer_load_dword v3, off, s[0:3], s33 ; 4-byte Folded Reload
-; GFX10-NEXT: s_waitcnt_depctr 0xffe3
+; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
; GFX10-NEXT: s_mov_b32 exec_lo, s4
; GFX10-NEXT: s_mov_b32 s33, s18
; GFX10-NEXT: s_waitcnt vmcnt(0)
@@ -5133,6 +5136,7 @@ define void @test_call_v3bf16(<3 x bfloat> %in, ptr addrspace(5) %out) {
; GFX1250-NEXT: s_swap_pc_i64 s[30:31], s[0:1]
; GFX1250-NEXT: scratch_store_b16 v4, v1, off offset:4 scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: scratch_store_b32 v4, v0, off scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: v_readlane_b32 s31, v5, 1
@@ -5346,7 +5350,7 @@ define void @test_call_v4bf16(<4 x bfloat> %in, ptr addrspace(5) %out) {
; GFX10-NEXT: s_mov_b32 s33, s32
; GFX10-NEXT: s_xor_saveexec_b32 s16, -1
; GFX10-NEXT: buffer_store_dword v3, off, s[0:3], s33 ; 4-byte Folded Spill
-; GFX10-NEXT: s_waitcnt_depctr 0xffe3
+; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
; GFX10-NEXT: s_mov_b32 exec_lo, s16
; GFX10-NEXT: s_addk_i32 s32, 0x200
; GFX10-NEXT: s_getpc_b64 s[16:17]
@@ -5366,7 +5370,7 @@ define void @test_call_v4bf16(<4 x bfloat> %in, ptr addrspace(5) %out) {
; GFX10-NEXT: s_mov_b32 s32, s33
; GFX10-NEXT: s_xor_saveexec_b32 s4, -1
; GFX10-NEXT: buffer_load_dword v3, off, s[0:3], s33 ; 4-byte Folded Reload
-; GFX10-NEXT: s_waitcnt_depctr 0xffe3
+; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
; GFX10-NEXT: s_mov_b32 exec_lo, s4
; GFX10-NEXT: s_mov_b32 s33, s18
; GFX10-NEXT: s_waitcnt vmcnt(0)
@@ -5682,7 +5686,7 @@ define void @test_call_v8bf16(<8 x bfloat> %in, ptr addrspace(5) %out) {
; GFX10-NEXT: s_mov_b32 s33, s32
; GFX10-NEXT: s_xor_saveexec_b32 s16, -1
; GFX10-NEXT: buffer_store_dword v5, off, s[0:3], s33 ; 4-byte Folded Spill
-; GFX10-NEXT: s_waitcnt_depctr 0xffe3
+; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
; GFX10-NEXT: s_mov_b32 exec_lo, s16
; GFX10-NEXT: s_addk_i32 s32, 0x200
; GFX10-NEXT: s_getpc_b64 s[16:17]
@@ -5706,7 +5710,7 @@ define void @test_call_v8bf16(<8 x bfloat> %in, ptr addrspace(5) %out) {
; GFX10-NEXT: s_mov_b32 s32, s33
; GFX10-NEXT: s_xor_saveexec_b32 s4, -1
; GFX10-NEXT: buffer_load_dword v5, off, s[0:3], s33 ; 4-byte Folded Reload
-; GFX10-NEXT: s_waitcnt_depctr 0xffe3
+; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
; GFX10-NEXT: s_mov_b32 exec_lo, s4
; GFX10-NEXT: s_mov_b32 s33, s18
; GFX10-NEXT: s_waitcnt vmcnt(0)
@@ -6123,7 +6127,7 @@ define void @test_call_v16bf16(<16 x bfloat> %in, ptr addrspace(5) %out) {
; GFX10-NEXT: s_mov_b32 s33, s32
; GFX10-NEXT: s_xor_saveexec_b32 s16, -1
; GFX10-NEXT: buffer_store_dword v9, off, s[0:3], s33 ; 4-byte Folded Spill
-; GFX10-NEXT: s_waitcnt_depctr 0xffe3
+; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
; GFX10-NEXT: s_mov_b32 exec_lo, s16
; GFX10-NEXT: s_addk_i32 s32, 0x200
; GFX10-NEXT: s_getpc_b64 s[16:17]
@@ -6155,7 +6159,7 @@ define void @test_call_v16bf16(<16 x bfloat> %in, ptr addrspace(5) %out) {
; GFX10-NEXT: s_mov_b32 s32, s33
; GFX10-NEXT: s_xor_saveexec_b32 s4, -1
; GFX10-NEXT: buffer_load_dword v9, off, s[0:3], s33 ; 4-byte Folded Reload
-; GFX10-NEXT: s_waitcnt_depctr 0xffe3
+; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
; GFX10-NEXT: s_mov_b32 exec_lo, s4
; GFX10-NEXT: s_mov_b32 s33, s18
; GFX10-NEXT: s_waitcnt vmcnt(0)
@@ -6212,6 +6216,7 @@ define void @test_call_v16bf16(<16 x bfloat> %in, ptr addrspace(5) %out) {
; GFX1250-NEXT: s_swap_pc_i64 s[30:31], s[0:1]
; GFX1250-NEXT: scratch_store_b128 v8, v[4:7], off offset:16 scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: scratch_store_b128 v8, v[0:3], off scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: v_readlane_b32 s31, v9, 1
@@ -6850,6 +6855,7 @@ define <2 x float> @global_extload_v2bf16_to_v2f32(ptr addrspace(1) %ptr) {
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: global_load_b32 v1, v[0:1], off
; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 16, v1
; GFX1250-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
@@ -6943,6 +6949,7 @@ define <3 x float> @global_extload_v3bf16_to_v3f32(ptr addrspace(1) %ptr) {
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: global_load_b64 v[2:3], v[0:1], off
; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 16, v2
; GFX1250-NEXT: v_and_b32_e32 v1, 0xffff0000, v2
; GFX1250-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -7033,6 +7040,7 @@ define <4 x float> @global_extload_v4bf16_to_v4f32(ptr addrspace(1) %ptr) {
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: global_load_b64 v[2:3], v[0:1], off
; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 16, v2
; GFX1250-NEXT: v_and_b32_e32 v1, 0xffff0000, v2
; GFX1250-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -7134,6 +7142,7 @@ define <5 x float> @global_extload_v5bf16_to_v5f32(ptr addrspace(1) %ptr) {
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: global_load_b128 v[2:5], v[0:1], off
; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 16, v2
; GFX1250-NEXT: v_and_b32_e32 v1, 0xffff0000, v2
; GFX1250-NEXT: v_lshlrev_b32_e32 v2, 16, v3
@@ -7251,6 +7260,7 @@ define <6 x float> @global_extload_v6bf16_to_v6f32(ptr addrspace(1) %ptr) {
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: global_load_b96 v[4:6], v[0:1], off
; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: v_dual_lshlrev_b32 v0, 16, v4 :: v_dual_lshlrev_b32 v2, 16, v5
; GFX1250-NEXT: v_and_b32_e32 v1, 0xffff0000, v4
; GFX1250-NEXT: v_and_b32_e32 v3, 0xffff0000, v5
@@ -7367,6 +7377,7 @@ define <8 x float> @global_extload_v8bf16_to_v8f32(ptr addrspace(1) %ptr) {
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: global_load_b128 v[4:7], v[0:1], off
; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: v_dual_lshlrev_b32 v0, 16, v4 :: v_dual_lshlrev_b32 v2, 16, v5
; GFX1250-NEXT: v_and_b32_e32 v1, 0xffff0000, v4
; GFX1250-NEXT: v_and_b32_e32 v3, 0xffff0000, v5
@@ -8001,6 +8012,7 @@ define <2 x double> @global_extload_v2bf16_to_v2f64(ptr addrspace(1) %ptr) {
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: global_load_b32 v0, v[0:1], off
; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: v_lshlrev_b32_e32 v1, 16, v0
; GFX1250-NEXT: v_and_b32_e32 v2, 0xffff0000, v0
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
@@ -8241,6 +8253,7 @@ define <4 x double> @global_extload_v4bf16_to_v4f64(ptr addrspace(1) %ptr) {
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: global_load_b64 v[2:3], v[0:1], off
; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: v_dual_lshlrev_b32 v0, 16, v2 :: v_dual_lshlrev_b32 v4, 16, v3
; GFX1250-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX1250-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
@@ -8377,6 +8390,7 @@ define <5 x double> @global_extload_v5bf16_to_v5f64(ptr addrspace(1) %ptr) {
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: global_load_b128 v[2:5], v[0:1], off
; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: v_dual_lshlrev_b32 v0, 16, v2 :: v_dual_lshlrev_b32 v5, 16, v3
; GFX1250-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX1250-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
@@ -8522,6 +8536,7 @@ define <6 x double> @global_extload_v6bf16_to_v6f64(ptr addrspace(1) %ptr) {
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: global_load_b96 v[4:6], v[0:1], off
; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX1250-NEXT: v_and_b32_e32 v2, 0xffff0000, v4
; GFX1250-NEXT: v_lshlrev_b32_e32 v4, 16, v5
@@ -8693,6 +8708,7 @@ define <8 x double> @global_extload_v8bf16_to_v8f64(ptr addrspace(1) %ptr) {
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: global_load_b128 v[8:11], v[0:1], off
; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: v_dual_lshlrev_b32 v0, 16, v8 :: v_dual_lshlrev_b32 v4, 16, v9
; GFX1250-NEXT: v_and_b32_e32 v2, 0xffff0000, v8
; GFX1250-NEXT: v_and_b32_e32 v6, 0xffff0000, v9
@@ -9552,6 +9568,7 @@ define <32 x double> @global_extload_v32bf16_to_v32f64(ptr addrspace(1) %ptr) {
; GFX8-NEXT: v_addc_u32_e32 v34, vcc, 0, v2, vcc
; GFX8-NEXT: v_add_u32_e32 v35, vcc, 36, v1
; GFX8-NEXT: v_addc_u32_e32 v36, vcc, 0, v2, vcc
+; GFX8-NEXT: v_add_u32_e32 v37, vcc, 38, v1
; GFX8-NEXT: buffer_store_dword v40, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
; GFX8-NEXT: buffer_store_dword v41, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
; GFX8-NEXT: buffer_store_dword v42, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
@@ -9563,7 +9580,6 @@ define <32 x double> @global_extload_v32bf16_to_v32f64(ptr addrspace(1) %ptr) {
; GFX8-NEXT: buffer_store_dword v56, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
; GFX8-NEXT: buffer_store_dword v57, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
; GFX8-NEXT: buffer_store_dword v58, off, s[0:3], s32 ; 4-byte Folded Spill
-; GFX8-NEXT: v_add_u32_e32 v37, vcc, 38, v1
; GFX8-NEXT: flat_load_ushort v44, v[1:2]
; GFX8-NEXT: v_addc_u32_e32 v38, vcc, 0, v2, vcc
; GFX8-NEXT: v_add_u32_e32 v48, vcc, 40, v1
@@ -21085,7 +21101,7 @@ define bfloat @v_fdiv_bf16(bfloat %a, bfloat %b) {
; GFX11TRUE16-NEXT: v_div_scale_f32 v5, vcc_lo, v2, v0, v2
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
; GFX11TRUE16-NEXT: v_rcp_f32_e32 v3, v1
-; GFX11TRUE16-NEXT: s_waitcnt_depctr 0xfff
+; GFX11TRUE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
; GFX11TRUE16-NEXT: v_fma_f32 v4, -v1, v3, 1.0
; GFX11TRUE16-NEXT: v_fmac_f32_e32 v3, v4, v3
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -21115,7 +21131,7 @@ define bfloat @v_fdiv_bf16(bfloat %a, bfloat %b) {
; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11FAKE16-NEXT: v_div_scale_f32 v2, null, v1, v1, v0
; GFX11FAKE16-NEXT: v_rcp_f32_e32 v3, v2
-; GFX11FAKE16-NEXT: s_waitcnt_depctr 0xfff
+; GFX11FAKE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
; GFX11FAKE16-NEXT: v_fma_f32 v4, -v2, v3, 1.0
; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11FAKE16-NEXT: v_fmac_f32_e32 v3, v4, v3
@@ -21303,6 +21319,7 @@ define amdgpu_ps i32 @s_fabs_bf16(bfloat inreg %a) {
;
; GFX1250-LABEL: s_fabs_bf16:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
; GFX1250-NEXT: s_and_b32 s0, s0, 0x7fff
; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1250-NEXT: s_and_b32 s0, 0xffff, s0
@@ -21419,6 +21436,7 @@ define amdgpu_ps i32 @s_fneg_bf16(bfloat inreg %a) {
;
; GFX1250-LABEL: s_fneg_bf16:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
; GFX1250-NEXT: s_xor_b32 s0, s0, 0x8000
; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1250-NEXT: s_and_b32 s0, 0xffff, s0
@@ -21540,6 +21558,7 @@ define amdgpu_ps i32 @s_fneg_fabs_bf16(bfloat inreg %a) {
;
; GFX1250-LABEL: s_fneg_fabs_bf16:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
; GFX1250-NEXT: s_bitset1_b32 s0, 15
; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1250-NEXT: s_and_b32 s0, 0xffff, s0
@@ -30889,7 +30908,7 @@ define bfloat @v_sqrt_bf16(bfloat %a) {
; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc_lo
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
; GFX11TRUE16-NEXT: v_sqrt_f32_e32 v1, v0
-; GFX11TRUE16-NEXT: s_waitcnt_depctr 0xfff
+; GFX11TRUE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
; GFX11TRUE16-NEXT: v_add_nc_u32_e32 v2, -1, v1
; GFX11TRUE16-NEXT: v_add_nc_u32_e32 v3, 1, v1
; GFX11TRUE16-NEXT: v_fma_f32 v4, -v2, v1, v0
@@ -30926,7 +30945,7 @@ define bfloat @v_sqrt_bf16(bfloat %a) {
; GFX11FAKE16-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc_lo
; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
; GFX11FAKE16-NEXT: v_sqrt_f32_e32 v1, v0
-; GFX11FAKE16-NEXT: s_waitcnt_depctr 0xfff
+; GFX11FAKE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
; GFX11FAKE16-NEXT: v_add_nc_u32_e32 v2, -1, v1
; GFX11FAKE16-NEXT: v_add_nc_u32_e32 v3, 1, v1
; GFX11FAKE16-NEXT: v_fma_f32 v4, -v2, v1, v0
@@ -31421,7 +31440,7 @@ define bfloat @v_log_bf16(bfloat %a) {
; GFX11TRUE16-NEXT: v_ldexp_f32 v0, v1, v0
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
; GFX11TRUE16-NEXT: v_log_f32_e32 v0, v0
-; GFX11TRUE16-NEXT: s_waitcnt_depctr 0xfff
+; GFX11TRUE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
; GFX11TRUE16-NEXT: v_mul_f32_e32 v1, 0x3f317217, v0
; GFX11TRUE16-NEXT: v_fma_f32 v2, 0x3f317217, v0, -v1
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -31452,7 +31471,7 @@ define bfloat @v_log_bf16(bfloat %a) {
; GFX11FAKE16-NEXT: v_ldexp_f32 v0, v0, v1
; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
; GFX11FAKE16-NEXT: v_log_f32_e32 v0, v0
-; GFX11FAKE16-NEXT: s_waitcnt_depctr 0xfff
+; GFX11FAKE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
; GFX11FAKE16-NEXT: v_mul_f32_e32 v1, 0x3f317217, v0
; GFX11FAKE16-NEXT: v_fma_f32 v2, 0x3f317217, v0, -v1
; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -31650,7 +31669,7 @@ define bfloat @v_log2_bf16(bfloat %a) {
; GFX11TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, 0x42000000, vcc_lo
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
; GFX11TRUE16-NEXT: v_log_f32_e32 v0, v0
-; GFX11TRUE16-NEXT: s_waitcnt_depctr 0xfff
+; GFX11TRUE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
; GFX11TRUE16-NEXT: v_sub_f32_e32 v0, v0, v1
; GFX11TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX11TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
@@ -31673,7 +31692,7 @@ define bfloat @v_log2_bf16(bfloat %a) {
; GFX11FAKE16-NEXT: v_ldexp_f32 v0, v0, v2
; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
; GFX11FAKE16-NEXT: v_log_f32_e32 v0, v0
-; GFX11FAKE16-NEXT: s_waitcnt_depctr 0xfff
+; GFX11FAKE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
; GFX11FAKE16-NEXT: v_sub_f32_e32 v0, v0, v1
; GFX11FAKE16-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX11FAKE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
@@ -31880,7 +31899,7 @@ define bfloat @v_log10_bf16(bfloat %a) {
; GFX11TRUE16-NEXT: v_ldexp_f32 v0, v1, v0
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
; GFX11TRUE16-NEXT: v_log_f32_e32 v0, v0
-; GFX11TRUE16-NEXT: s_waitcnt_depctr 0xfff
+; GFX11TRUE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
; GFX11TRUE16-NEXT: v_mul_f32_e32 v1, 0x3e9a209a, v0
; GFX11TRUE16-NEXT: v_fma_f32 v2, 0x3e9a209a, v0, -v1
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -31911,7 +31930,7 @@ define bfloat @v_log10_bf16(bfloat %a) {
; GFX11FAKE16-NEXT: v_ldexp_f32 v0, v0, v1
; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
; GFX11FAKE16-NEXT: v_log_f32_e32 v0, v0
-; GFX11FAKE16-NEXT: s_waitcnt_depctr 0xfff
+; GFX11FAKE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
; GFX11FAKE16-NEXT: v_mul_f32_e32 v1, 0x3e9a209a, v0
; GFX11FAKE16-NEXT: v_fma_f32 v2, 0x3e9a209a, v0, -v1
; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -32178,7 +32197,7 @@ define bfloat @v_exp_bf16(bfloat %a) {
; GFX11TRUE16-NEXT: v_add_f32_e32 v0, v0, v2
; GFX11TRUE16-NEXT: v_cvt_i32_f32_e32 v2, v3
; GFX11TRUE16-NEXT: v_exp_f32_e32 v0, v0
-; GFX11TRUE16-NEXT: s_waitcnt_depctr 0xfff
+; GFX11TRUE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
; GFX11TRUE16-NEXT: v_ldexp_f32 v0, v0, v2
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc_lo
@@ -32210,7 +32229,7 @@ define bfloat @v_exp_bf16(bfloat %a) {
; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11FAKE16-NEXT: v_add_f32_e32 v1, v1, v3
; GFX11FAKE16-NEXT: v_exp_f32_e32 v1, v1
-; GFX11FAKE16-NEXT: s_waitcnt_depctr 0xfff
+; GFX11FAKE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
; GFX11FAKE16-NEXT: v_ldexp_f32 v1, v1, v2
; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11FAKE16-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc_lo
@@ -32414,7 +32433,7 @@ define bfloat @v_exp2_bf16(bfloat %a) {
; GFX11TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, 0xffffffc0, vcc_lo
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
; GFX11TRUE16-NEXT: v_exp_f32_e32 v0, v0
-; GFX11TRUE16-NEXT: s_waitcnt_depctr 0xfff
+; GFX11TRUE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
; GFX11TRUE16-NEXT: v_ldexp_f32 v0, v0, v1
; GFX11TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX11TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
@@ -32437,7 +32456,7 @@ define bfloat @v_exp2_bf16(bfloat %a) {
; GFX11FAKE16-NEXT: v_add_f32_e32 v0, v0, v2
; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
; GFX11FAKE16-NEXT: v_exp_f32_e32 v0, v0
-; GFX11FAKE16-NEXT: s_waitcnt_depctr 0xfff
+; GFX11FAKE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
; GFX11FAKE16-NEXT: v_ldexp_f32 v0, v0, v1
; GFX11FAKE16-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX11FAKE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
@@ -32652,7 +32671,7 @@ define bfloat @v_exp10_bf16(bfloat %a) {
; GFX11TRUE16-NEXT: v_add_f32_e32 v0, v0, v2
; GFX11TRUE16-NEXT: v_cvt_i32_f32_e32 v2, v3
; GFX11TRUE16-NEXT: v_exp_f32_e32 v0, v0
-; GFX11TRUE16-NEXT: s_waitcnt_depctr 0xfff
+; GFX11TRUE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
; GFX11TRUE16-NEXT: v_ldexp_f32 v0, v0, v2
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc_lo
@@ -32684,7 +32703,7 @@ define bfloat @v_exp10_bf16(bfloat %a) {
; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11FAKE16-NEXT: v_add_f32_e32 v1, v1, v3
; GFX11FAKE16-NEXT: v_exp_f32_e32 v1, v1
-; GFX11FAKE16-NEXT: s_waitcnt_depctr 0xfff
+; GFX11FAKE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
; GFX11FAKE16-NEXT: v_ldexp_f32 v1, v1, v2
; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11FAKE16-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc_lo
@@ -39131,21 +39150,21 @@ define bfloat @v_sitofp_i64_to_bf16(i64 %x) {
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_xor_b32_e32 v2, v0, v1
; GFX1250-NEXT: v_cls_i32_e32 v3, v1
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_dual_add_nc_u32 v3, -1, v3 :: v_dual_bitop2_b32 v2, v0, v1 bitop3:0x14
; GFX1250-NEXT: v_ashrrev_i32_e32 v2, 31, v2
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-NEXT: v_add_nc_u32_e32 v2, 32, v2
+; GFX1250-NEXT: v_min_u32_e32 v2, v3, v2
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_add_min_u32 v2, v3, -1, v2
; GFX1250-NEXT: v_lshlrev_b64_e32 v[0:1], v2, v[0:1]
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-NEXT: v_min_u32_e32 v0, 1, v0
-; GFX1250-NEXT: v_dual_sub_nc_u32 v1, 32, v2 :: v_dual_bitop2_b32 v0, v1, v0 bitop3:0x54
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_dual_sub_nc_u32 v1, 32, v2 :: v_dual_bitop2_b32 v0, v1, v0 bitop3:0x54
; GFX1250-NEXT: v_cvt_f32_i32_e32 v0, v0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-NEXT: v_ldexp_f32 v0, v0, v1
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%op = sitofp i64 %x to bfloat
@@ -39483,29 +39502,30 @@ define <2 x bfloat> @v_sitofp_v2i64_to_v2bf16(<2 x i64> %x) {
; GFX1250-NEXT: v_xor_b32_e32 v4, v2, v3
; GFX1250-NEXT: v_cls_i32_e32 v6, v3
; GFX1250-NEXT: v_cls_i32_e32 v7, v1
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1250-NEXT: v_dual_ashrrev_i32 v5, 31, v5 :: v_dual_ashrrev_i32 v4, 31, v4
+; GFX1250-NEXT: v_dual_add_nc_u32 v6, -1, v6 :: v_dual_add_nc_u32 v7, -1, v7
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-NEXT: v_dual_add_nc_u32 v5, 32, v5 :: v_dual_add_nc_u32 v4, 32, v4
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1250-NEXT: v_add_min_u32 v5, v7, -1, v5
-; GFX1250-NEXT: v_add_min_u32 v4, v6, -1, v4
+; GFX1250-NEXT: v_min_u32_e32 v5, v7, v5
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-NEXT: v_min_u32_e32 v4, v6, v4
; GFX1250-NEXT: v_lshlrev_b64_e32 v[0:1], v5, v[0:1]
-; GFX1250-NEXT: v_lshlrev_b64_e32 v[2:3], v4, v[2:3]
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-NEXT: v_lshlrev_b64_e32 v[2:3], v4, v[2:3]
; GFX1250-NEXT: v_min_u32_e32 v0, 1, v0
-; GFX1250-NEXT: v_min_u32_e32 v2, 1, v2
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-NEXT: v_min_u32_e32 v2, 1, v2
; GFX1250-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX1250-NEXT: v_dual_sub_nc_u32 v1, 32, v4 :: v_dual_bitop2_b32 v2, v3, v2 bitop3:0x54
; GFX1250-NEXT: v_sub_nc_u32_e32 v3, 32, v5
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX1250-NEXT: v_cvt_f32_i32_e32 v0, v0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1250-NEXT: v_cvt_f32_i32_e32 v2, v2
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1250-NEXT: v_ldexp_f32 v0, v0, v3
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-NEXT: v_ldexp_f32 v1, v2, v1
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-NEXT: v_cvt_pk_bf16_f32 v0, v0, v1
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%op = sitofp <2 x i64> %x to <2 x bfloat>
@@ -39968,41 +39988,42 @@ define <3 x bfloat> @v_sitofp_v3i64_to_v3bf16(<3 x i64> %x) {
; GFX1250TRUE16: ; %bb.0:
; GFX1250TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250TRUE16-NEXT: v_xor_b32_e32 v7, v2, v3
-; GFX1250TRUE16-NEXT: v_xor_b32_e32 v6, v4, v5
+; GFX1250TRUE16-NEXT: v_cls_i32_e32 v6, v5
+; GFX1250TRUE16-NEXT: v_xor_b32_e32 v7, v4, v5
; GFX1250TRUE16-NEXT: v_cls_i32_e32 v10, v3
-; GFX1250TRUE16-NEXT: v_cls_i32_e32 v9, v5
; GFX1250TRUE16-NEXT: v_cls_i32_e32 v11, v1
-; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1250TRUE16-NEXT: v_dual_ashrrev_i32 v7, 31, v7 :: v_dual_ashrrev_i32 v6, 31, v6
-; GFX1250TRUE16-NEXT: v_xor_b32_e32 v8, v0, v1
-; GFX1250TRUE16-NEXT: v_dual_add_nc_u32 v7, 32, v7 :: v_dual_add_nc_u32 v6, 32, v6
-; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1250TRUE16-NEXT: v_ashrrev_i32_e32 v8, 31, v8
-; GFX1250TRUE16-NEXT: v_add_min_u32 v7, v10, -1, v7
-; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1250TRUE16-NEXT: v_add_min_u32 v6, v9, -1, v6
-; GFX1250TRUE16-NEXT: v_lshlrev_b64_e32 v[2:3], v7, v[2:3]
-; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1250TRUE16-NEXT: v_lshlrev_b64_e32 v[4:5], v6, v[4:5]
-; GFX1250TRUE16-NEXT: v_min_u32_e32 v2, 1, v2
-; GFX1250TRUE16-NEXT: v_add_nc_u32_e32 v8, 32, v8
+; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX1250TRUE16-NEXT: v_dual_add_nc_u32 v6, -1, v6 :: v_dual_bitop2_b32 v9, v0, v1 bitop3:0x14
+; GFX1250TRUE16-NEXT: v_ashrrev_i32_e32 v7, 31, v7
+; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250TRUE16-NEXT: v_dual_ashrrev_i32 v9, 31, v9 :: v_dual_bitop2_b32 v8, v2, v3 bitop3:0x14
+; GFX1250TRUE16-NEXT: v_dual_add_nc_u32 v7, 32, v7 :: v_dual_ashrrev_i32 v8, 31, v8
+; GFX1250TRUE16-NEXT: v_dual_add_nc_u32 v10, -1, v10 :: v_dual_add_nc_u32 v11, -1, v11
; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1250TRUE16-NEXT: v_add_nc_u32_e32 v9, 32, v9
+; GFX1250TRUE16-NEXT: v_min_u32_e32 v6, v6, v7
+; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250TRUE16-NEXT: v_add_nc_u32_e32 v8, 32, v8
+; GFX1250TRUE16-NEXT: v_lshlrev_b64_e32 v[4:5], v6, v[4:5]
+; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1250TRUE16-NEXT: v_min_u32_e32 v7, v10, v8
+; GFX1250TRUE16-NEXT: v_min_u32_e32 v8, v11, v9
+; GFX1250TRUE16-NEXT: v_lshlrev_b64_e32 v[2:3], v7, v[2:3]
+; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX1250TRUE16-NEXT: v_lshlrev_b64_e32 v[0:1], v8, v[0:1]
; GFX1250TRUE16-NEXT: v_min_u32_e32 v4, 1, v4
-; GFX1250TRUE16-NEXT: v_or_b32_e32 v2, v3, v2
-; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1250TRUE16-NEXT: v_add_min_u32 v8, v11, -1, v8
-; GFX1250TRUE16-NEXT: v_dual_sub_nc_u32 v3, 32, v6 :: v_dual_bitop2_b32 v4, v5, v4 bitop3:0x54
+; GFX1250TRUE16-NEXT: v_min_u32_e32 v2, 1, v2
; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1250TRUE16-NEXT: v_cvt_f32_i32_e32 v2, v2
-; GFX1250TRUE16-NEXT: v_lshlrev_b64_e32 v[0:1], v8, v[0:1]
-; GFX1250TRUE16-NEXT: v_sub_nc_u32_e32 v5, 32, v8
-; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250TRUE16-NEXT: v_min_u32_e32 v0, 1, v0
-; GFX1250TRUE16-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX1250TRUE16-NEXT: v_or_b32_e32 v4, v5, v4
+; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1250TRUE16-NEXT: v_dual_sub_nc_u32 v5, 32, v8 :: v_dual_bitop2_b32 v2, v3, v2 bitop3:0x54
+; GFX1250TRUE16-NEXT: v_dual_sub_nc_u32 v3, 32, v6 :: v_dual_bitop2_b32 v0, v1, v0 bitop3:0x54
+; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
; GFX1250TRUE16-NEXT: v_cvt_f32_i32_e32 v1, v4
; GFX1250TRUE16-NEXT: v_sub_nc_u32_e32 v4, 32, v7
-; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1250TRUE16-NEXT: v_cvt_f32_i32_e32 v2, v2
+; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX1250TRUE16-NEXT: v_cvt_f32_i32_e32 v0, v0
; GFX1250TRUE16-NEXT: v_ldexp_f32 v1, v1, v3
; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
@@ -40017,44 +40038,47 @@ define <3 x bfloat> @v_sitofp_v3i64_to_v3bf16(<3 x i64> %x) {
; GFX1250FAKE16: ; %bb.0:
; GFX1250FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250FAKE16-NEXT: v_xor_b32_e32 v8, v4, v5
-; GFX1250FAKE16-NEXT: v_xor_b32_e32 v6, v2, v3
+; GFX1250FAKE16-NEXT: v_cls_i32_e32 v6, v5
+; GFX1250FAKE16-NEXT: v_xor_b32_e32 v7, v2, v3
; GFX1250FAKE16-NEXT: v_cls_i32_e32 v10, v3
-; GFX1250FAKE16-NEXT: v_cls_i32_e32 v9, v5
; GFX1250FAKE16-NEXT: v_cls_i32_e32 v11, v1
-; GFX1250FAKE16-NEXT: v_dual_ashrrev_i32 v8, 31, v8 :: v_dual_bitop2_b32 v7, v0, v1 bitop3:0x14
-; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250FAKE16-NEXT: v_dual_ashrrev_i32 v6, 31, v6 :: v_dual_ashrrev_i32 v7, 31, v7
-; GFX1250FAKE16-NEXT: v_dual_add_nc_u32 v6, 32, v6 :: v_dual_add_nc_u32 v7, 32, v7
-; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1250FAKE16-NEXT: v_add_min_u32 v6, v10, -1, v6
-; GFX1250FAKE16-NEXT: v_add_min_u32 v7, v11, -1, v7
+; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX1250FAKE16-NEXT: v_dual_add_nc_u32 v6, -1, v6 :: v_dual_bitop2_b32 v8, v4, v5 bitop3:0x14
+; GFX1250FAKE16-NEXT: v_dual_ashrrev_i32 v7, 31, v7 :: v_dual_bitop2_b32 v9, v0, v1 bitop3:0x14
; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1250FAKE16-NEXT: v_lshlrev_b64_e32 v[2:3], v6, v[2:3]
-; GFX1250FAKE16-NEXT: v_lshlrev_b64_e32 v[0:1], v7, v[0:1]
-; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX1250FAKE16-NEXT: v_min_u32_e32 v2, 1, v2
-; GFX1250FAKE16-NEXT: v_add_nc_u32_e32 v8, 32, v8
-; GFX1250FAKE16-NEXT: v_min_u32_e32 v0, 1, v0
+; GFX1250FAKE16-NEXT: v_dual_add_nc_u32 v10, -1, v10 :: v_dual_ashrrev_i32 v8, 31, v8
+; GFX1250FAKE16-NEXT: v_dual_add_nc_u32 v11, -1, v11 :: v_dual_ashrrev_i32 v9, 31, v9
+; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250FAKE16-NEXT: v_dual_add_nc_u32 v7, 32, v7 :: v_dual_add_nc_u32 v8, 32, v8
+; GFX1250FAKE16-NEXT: v_add_nc_u32_e32 v9, 32, v9
+; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1250FAKE16-NEXT: v_min_u32_e32 v7, v10, v7
+; GFX1250FAKE16-NEXT: v_min_u32_e32 v6, v6, v8
; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1250FAKE16-NEXT: v_or_b32_e32 v2, v3, v2
-; GFX1250FAKE16-NEXT: v_add_min_u32 v8, v9, -1, v8
+; GFX1250FAKE16-NEXT: v_min_u32_e32 v9, v11, v9
+; GFX1250FAKE16-NEXT: v_lshlrev_b64_e32 v[2:3], v7, v[2:3]
; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1250FAKE16-NEXT: v_dual_sub_nc_u32 v3, 32, v6 :: v_dual_bitop2_b32 v0, v1, v0 bitop3:0x54
-; GFX1250FAKE16-NEXT: v_cvt_f32_i32_e32 v2, v2
-; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX1250FAKE16-NEXT: v_lshlrev_b64_e32 v[4:5], v8, v[4:5]
-; GFX1250FAKE16-NEXT: v_sub_nc_u32_e32 v8, 32, v8
-; GFX1250FAKE16-NEXT: v_ldexp_f32 v2, v2, v3
-; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250FAKE16-NEXT: v_lshlrev_b64_e32 v[4:5], v6, v[4:5]
+; GFX1250FAKE16-NEXT: v_lshlrev_b64_e32 v[0:1], v9, v[0:1]
+; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1250FAKE16-NEXT: v_min_u32_e32 v2, 1, v2
; GFX1250FAKE16-NEXT: v_min_u32_e32 v4, 1, v4
-; GFX1250FAKE16-NEXT: v_dual_sub_nc_u32 v4, 32, v7 :: v_dual_bitop2_b32 v1, v5, v4 bitop3:0x54
+; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX1250FAKE16-NEXT: v_min_u32_e32 v0, 1, v0
+; GFX1250FAKE16-NEXT: v_sub_nc_u32_e32 v6, 32, v6
+; GFX1250FAKE16-NEXT: v_dual_sub_nc_u32 v3, 32, v7 :: v_dual_bitop2_b32 v2, v3, v2 bitop3:0x54
+; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX1250FAKE16-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX1250FAKE16-NEXT: v_dual_sub_nc_u32 v4, 32, v9 :: v_dual_bitop2_b32 v1, v5, v4 bitop3:0x54
+; GFX1250FAKE16-NEXT: v_cvt_f32_i32_e32 v2, v2
+; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX1250FAKE16-NEXT: v_cvt_f32_i32_e32 v0, v0
-; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1250FAKE16-NEXT: v_cvt_f32_i32_e32 v1, v1
+; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1250FAKE16-NEXT: v_ldexp_f32 v2, v2, v3
; GFX1250FAKE16-NEXT: v_ldexp_f32 v0, v0, v4
-; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1250FAKE16-NEXT: v_ldexp_f32 v1, v1, v8
+; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250FAKE16-NEXT: v_ldexp_f32 v1, v1, v6
; GFX1250FAKE16-NEXT: v_cvt_pk_bf16_f32 v0, v0, v2
; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1250FAKE16-NEXT: v_cvt_pk_bf16_f32 v1, v1, s0
@@ -40644,51 +40668,54 @@ define <4 x bfloat> @v_sitofp_v4i64_to_v4bf16(<4 x i64> %x) {
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_xor_b32_e32 v9, v4, v5
+; GFX1250-NEXT: v_cls_i32_e32 v9, v7
; GFX1250-NEXT: v_xor_b32_e32 v8, v6, v7
-; GFX1250-NEXT: v_cls_i32_e32 v12, v7
-; GFX1250-NEXT: v_cls_i32_e32 v13, v5
-; GFX1250-NEXT: v_cls_i32_e32 v14, v3
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX1250-NEXT: v_dual_ashrrev_i32 v9, 31, v9 :: v_dual_ashrrev_i32 v8, 31, v8
-; GFX1250-NEXT: v_xor_b32_e32 v10, v2, v3
-; GFX1250-NEXT: v_cls_i32_e32 v15, v1
-; GFX1250-NEXT: v_dual_add_nc_u32 v9, 32, v9 :: v_dual_add_nc_u32 v8, 32, v8
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1250-NEXT: v_dual_ashrrev_i32 v10, 31, v10 :: v_dual_bitop2_b32 v11, v0, v1 bitop3:0x14
-; GFX1250-NEXT: v_add_min_u32 v9, v13, -1, v9
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1250-NEXT: v_add_min_u32 v8, v12, -1, v8
-; GFX1250-NEXT: v_dual_ashrrev_i32 v11, 31, v11 :: v_dual_add_nc_u32 v10, 32, v10
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1250-NEXT: v_lshlrev_b64_e32 v[4:5], v9, v[4:5]
-; GFX1250-NEXT: v_lshlrev_b64_e32 v[6:7], v8, v[6:7]
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX1250-NEXT: v_add_nc_u32_e32 v11, 32, v11
-; GFX1250-NEXT: v_add_min_u32 v10, v14, -1, v10
+; GFX1250-NEXT: v_cls_i32_e32 v10, v5
+; GFX1250-NEXT: v_xor_b32_e32 v14, v0, v1
+; GFX1250-NEXT: v_cls_i32_e32 v12, v3
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX1250-NEXT: v_dual_add_nc_u32 v9, -1, v9 :: v_dual_ashrrev_i32 v8, 31, v8
+; GFX1250-NEXT: v_dual_add_nc_u32 v10, -1, v10 :: v_dual_bitop2_b32 v11, v4, v5 bitop3:0x14
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1250-NEXT: v_add_min_u32 v11, v15, -1, v11
-; GFX1250-NEXT: v_lshlrev_b64_e32 v[2:3], v10, v[2:3]
-; GFX1250-NEXT: v_min_u32_e32 v6, 1, v6
-; GFX1250-NEXT: v_min_u32_e32 v4, 1, v4
+; GFX1250-NEXT: v_dual_add_nc_u32 v8, 32, v8 :: v_dual_bitop2_b32 v13, v2, v3 bitop3:0x14
+; GFX1250-NEXT: v_ashrrev_i32_e32 v11, 31, v11
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1250-NEXT: v_min_u32_e32 v8, v9, v8
+; GFX1250-NEXT: v_ashrrev_i32_e32 v9, 31, v13
+; GFX1250-NEXT: v_cls_i32_e32 v13, v1
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1250-NEXT: v_dual_ashrrev_i32 v14, 31, v14 :: v_dual_add_nc_u32 v11, 32, v11
+; GFX1250-NEXT: v_dual_add_nc_u32 v12, -1, v12 :: v_dual_add_nc_u32 v9, 32, v9
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1250-NEXT: v_dual_add_nc_u32 v13, -1, v13 :: v_dual_add_nc_u32 v14, 32, v14
+; GFX1250-NEXT: v_min_u32_e32 v10, v10, v11
+; GFX1250-NEXT: v_lshlrev_b64_e32 v[6:7], v8, v[6:7]
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX1250-NEXT: v_min_u32_e32 v9, v12, v9
+; GFX1250-NEXT: v_min_u32_e32 v11, v13, v14
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1250-NEXT: v_lshlrev_b64_e32 v[4:5], v10, v[4:5]
+; GFX1250-NEXT: v_lshlrev_b64_e32 v[2:3], v9, v[2:3]
+; GFX1250-NEXT: v_min_u32_e32 v6, 1, v6
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX1250-NEXT: v_lshlrev_b64_e32 v[0:1], v11, v[0:1]
+; GFX1250-NEXT: v_min_u32_e32 v4, 1, v4
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_4)
; GFX1250-NEXT: v_or_b32_e32 v6, v7, v6
; GFX1250-NEXT: v_min_u32_e32 v2, 1, v2
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX1250-NEXT: v_dual_sub_nc_u32 v5, 32, v10 :: v_dual_bitop2_b32 v4, v5, v4 bitop3:0x54
-; GFX1250-NEXT: v_sub_nc_u32_e32 v7, 32, v9
+; GFX1250-NEXT: v_sub_nc_u32_e32 v7, 32, v10
; GFX1250-NEXT: v_min_u32_e32 v0, 1, v0
-; GFX1250-NEXT: v_dual_sub_nc_u32 v3, 32, v8 :: v_dual_bitop2_b32 v2, v3, v2 bitop3:0x54
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1250-NEXT: v_cvt_f32_i32_e32 v4, v4
+; GFX1250-NEXT: v_dual_sub_nc_u32 v5, 32, v9 :: v_dual_bitop2_b32 v4, v5, v4 bitop3:0x54
+; GFX1250-NEXT: v_or_b32_e32 v2, v3, v2
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX1250-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX1250-NEXT: v_cvt_f32_i32_e32 v1, v6
+; GFX1250-NEXT: v_sub_nc_u32_e32 v1, 32, v8
+; GFX1250-NEXT: v_cvt_f32_i32_e32 v3, v6
+; GFX1250-NEXT: v_cvt_f32_i32_e32 v4, v4
; GFX1250-NEXT: v_sub_nc_u32_e32 v6, 32, v11
; GFX1250-NEXT: v_cvt_f32_i32_e32 v2, v2
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX1250-NEXT: v_cvt_f32_i32_e32 v0, v0
-; GFX1250-NEXT: v_ldexp_f32 v1, v1, v3
+; GFX1250-NEXT: v_ldexp_f32 v1, v3, v1
; GFX1250-NEXT: v_ldexp_f32 v3, v4, v7
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX1250-NEXT: v_ldexp_f32 v2, v2, v5
@@ -44214,6 +44241,7 @@ define amdgpu_ps i32 @s_select_bf16(bfloat inreg %a, bfloat inreg %b, i32 %c) {
;
; GFX1250TRUE16-LABEL: s_select_bf16:
; GFX1250TRUE16: ; %bb.0:
+; GFX1250TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
; GFX1250TRUE16-NEXT: v_mov_b16_e32 v1.l, s0
; GFX1250TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1250TRUE16-NEXT: v_mov_b16_e32 v0.h, 0
@@ -44224,6 +44252,7 @@ define amdgpu_ps i32 @s_select_bf16(bfloat inreg %a, bfloat inreg %b, i32 %c) {
;
; GFX1250FAKE16-LABEL: s_select_bf16:
; GFX1250FAKE16: ; %bb.0:
+; GFX1250FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
; GFX1250FAKE16-NEXT: v_mov_b32_e32 v1, s0
; GFX1250FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -44370,6 +44399,7 @@ define amdgpu_ps i32 @s_select_v2bf16(<2 x bfloat> inreg %a, <2 x bfloat> inreg
;
; GFX1250TRUE16-LABEL: s_select_v2bf16:
; GFX1250TRUE16: ; %bb.0:
+; GFX1250TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
; GFX1250TRUE16-NEXT: s_lshr_b32 s2, s0, 16
; GFX1250TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1250TRUE16-NEXT: v_mov_b16_e32 v1.l, s2
@@ -44384,6 +44414,7 @@ define amdgpu_ps i32 @s_select_v2bf16(<2 x bfloat> inreg %a, <2 x bfloat> inreg
;
; GFX1250FAKE16-LABEL: s_select_v2bf16:
; GFX1250FAKE16: ; %bb.0:
+; GFX1250FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
; GFX1250FAKE16-NEXT: s_lshr_b32 s2, s0, 16
; GFX1250FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1250FAKE16-NEXT: v_dual_mov_b32 v1, s2 :: v_dual_mov_b32 v2, s0
@@ -44536,6 +44567,7 @@ define amdgpu_ps i32 @s_vselect_v2bf16(<2 x bfloat> inreg %a, <2 x bfloat> inreg
;
; GFX1250TRUE16-LABEL: s_vselect_v2bf16:
; GFX1250TRUE16: ; %bb.0:
+; GFX1250TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
; GFX1250TRUE16-NEXT: s_lshr_b32 s3, s0, 16
; GFX1250TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1250TRUE16-NEXT: v_cmp_eq_u32_e64 s2, 0, v1
@@ -44551,6 +44583,7 @@ define amdgpu_ps i32 @s_vselect_v2bf16(<2 x bfloat> inreg %a, <2 x bfloat> inreg
;
; GFX1250FAKE16-LABEL: s_vselect_v2bf16:
; GFX1250FAKE16: ; %bb.0:
+; GFX1250FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
; GFX1250FAKE16-NEXT: s_lshr_b32 s2, s0, 16
; GFX1250FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
; GFX1250FAKE16-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s0
@@ -46057,18 +46090,18 @@ define <32 x bfloat> @v_select_v32bf16(i1 %cond, <32 x bfloat> %a, <32 x bfloat>
define amdgpu_ps <2 x i32> @s_select_v3bf16(<3 x bfloat> inreg %a, <3 x bfloat> inreg %b, i32 %c) {
; GCN-LABEL: s_select_v3bf16:
; GCN: ; %bb.0:
-; GCN-NEXT: v_mul_f32_e64 v1, 1.0, s1
-; GCN-NEXT: v_mul_f32_e64 v2, 1.0, s0
-; GCN-NEXT: v_mul_f32_e64 v3, 1.0, s4
-; GCN-NEXT: v_mul_f32_e64 v4, 1.0, s3
+; GCN-NEXT: v_mul_f32_e64 v2, 1.0, s1
+; GCN-NEXT: v_mul_f32_e64 v1, 1.0, s0
+; GCN-NEXT: v_mul_f32_e64 v4, 1.0, s4
+; GCN-NEXT: v_mul_f32_e64 v3, 1.0, s3
; GCN-NEXT: v_mul_f32_e64 v5, 1.0, s2
; GCN-NEXT: v_mul_f32_e64 v6, 1.0, s5
-; GCN-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GCN-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GCN-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GCN-NEXT: v_lshrrev_b32_e32 v4, 16, v4
; GCN-NEXT: v_lshrrev_b32_e32 v5, 16, v5
; GCN-NEXT: v_lshrrev_b32_e32 v6, 16, v6
-; GCN-NEXT: v_alignbit_b32 v1, v1, v2, 16
-; GCN-NEXT: v_alignbit_b32 v2, v3, v4, 16
+; GCN-NEXT: v_lshr_b64 v[1:2], v[1:2], 16
+; GCN-NEXT: v_lshr_b64 v[2:3], v[3:4], 16
; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GCN-NEXT: v_cndmask_b32_e32 v0, v6, v5, vcc
; GCN-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc
@@ -46079,13 +46112,13 @@ define amdgpu_ps <2 x i32> @s_select_v3bf16(<3 x bfloat> inreg %a, <3 x bfloat>
; GFX7-LABEL: s_select_v3bf16:
; GFX7: ; %bb.0:
; GFX7-NEXT: v_mul_f32_e64 v1, 1.0, s1
-; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX7-NEXT: v_mul_f32_e64 v2, 1.0, s0
-; GFX7-NEXT: v_alignbit_b32 v1, v1, v2, 16
+; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v1
+; GFX7-NEXT: v_mul_f32_e64 v1, 1.0, s0
+; GFX7-NEXT: v_lshr_b64 v[1:2], v[1:2], 16
; GFX7-NEXT: v_mul_f32_e64 v2, 1.0, s4
-; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GFX7-NEXT: v_mul_f32_e64 v3, 1.0, s3
-; GFX7-NEXT: v_alignbit_b32 v2, v2, v3, 16
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v2
+; GFX7-NEXT: v_mul_f32_e64 v2, 1.0, s3
+; GFX7-NEXT: v_lshr_b64 v[2:3], v[2:3], 16
; GFX7-NEXT: v_mul_f32_e64 v3, 1.0, s2
; GFX7-NEXT: v_mul_f32_e64 v4, 1.0, s5
; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v3
@@ -46168,6 +46201,7 @@ define amdgpu_ps <2 x i32> @s_select_v3bf16(<3 x bfloat> inreg %a, <3 x bfloat>
;
; GFX1250-LABEL: s_select_v3bf16:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
; GFX1250-NEXT: v_dual_mov_b32 v1, s0 :: v_dual_mov_b32 v2, s1
; GFX1250-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
@@ -46195,22 +46229,22 @@ define amdgpu_ps <2 x i32> @s_select_v3bf16(<3 x bfloat> inreg %a, <3 x bfloat>
define amdgpu_ps <2 x i32> @s_select_v4bf16(<4 x bfloat> inreg %a, <4 x bfloat> inreg %b, i32 %c) {
; GCN-LABEL: s_select_v4bf16:
; GCN: ; %bb.0:
-; GCN-NEXT: v_mul_f32_e64 v1, 1.0, s1
-; GCN-NEXT: v_mul_f32_e64 v2, 1.0, s0
-; GCN-NEXT: v_mul_f32_e64 v3, 1.0, s5
-; GCN-NEXT: v_mul_f32_e64 v4, 1.0, s4
-; GCN-NEXT: v_mul_f32_e64 v5, 1.0, s3
-; GCN-NEXT: v_mul_f32_e64 v6, 1.0, s2
-; GCN-NEXT: v_mul_f32_e64 v7, 1.0, s7
-; GCN-NEXT: v_mul_f32_e64 v8, 1.0, s6
-; GCN-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GCN-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GCN-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GCN-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; GCN-NEXT: v_alignbit_b32 v1, v1, v2, 16
-; GCN-NEXT: v_alignbit_b32 v2, v3, v4, 16
-; GCN-NEXT: v_alignbit_b32 v3, v5, v6, 16
-; GCN-NEXT: v_alignbit_b32 v4, v7, v8, 16
+; GCN-NEXT: v_mul_f32_e64 v2, 1.0, s1
+; GCN-NEXT: v_mul_f32_e64 v1, 1.0, s0
+; GCN-NEXT: v_mul_f32_e64 v4, 1.0, s5
+; GCN-NEXT: v_mul_f32_e64 v3, 1.0, s4
+; GCN-NEXT: v_mul_f32_e64 v6, 1.0, s3
+; GCN-NEXT: v_mul_f32_e64 v5, 1.0, s2
+; GCN-NEXT: v_mul_f32_e64 v8, 1.0, s7
+; GCN-NEXT: v_mul_f32_e64 v7, 1.0, s6
+; GCN-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GCN-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; GCN-NEXT: v_lshrrev_b32_e32 v6, 16, v6
+; GCN-NEXT: v_lshrrev_b32_e32 v8, 16, v8
+; GCN-NEXT: v_lshr_b64 v[1:2], v[1:2], 16
+; GCN-NEXT: v_lshr_b64 v[2:3], v[3:4], 16
+; GCN-NEXT: v_lshr_b64 v[3:4], v[5:6], 16
+; GCN-NEXT: v_lshr_b64 v[4:5], v[7:8], 16
; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GCN-NEXT: v_cndmask_b32_e32 v0, v4, v3, vcc
; GCN-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc
@@ -46221,21 +46255,21 @@ define amdgpu_ps <2 x i32> @s_select_v4bf16(<4 x bfloat> inreg %a, <4 x bfloat>
; GFX7-LABEL: s_select_v4bf16:
; GFX7: ; %bb.0:
; GFX7-NEXT: v_mul_f32_e64 v1, 1.0, s1
-; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX7-NEXT: v_mul_f32_e64 v2, 1.0, s0
-; GFX7-NEXT: v_alignbit_b32 v1, v1, v2, 16
+; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v1
+; GFX7-NEXT: v_mul_f32_e64 v1, 1.0, s0
+; GFX7-NEXT: v_lshr_b64 v[1:2], v[1:2], 16
; GFX7-NEXT: v_mul_f32_e64 v2, 1.0, s5
-; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GFX7-NEXT: v_mul_f32_e64 v3, 1.0, s4
-; GFX7-NEXT: v_alignbit_b32 v2, v2, v3, 16
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v2
+; GFX7-NEXT: v_mul_f32_e64 v2, 1.0, s4
+; GFX7-NEXT: v_lshr_b64 v[2:3], v[2:3], 16
; GFX7-NEXT: v_mul_f32_e64 v3, 1.0, s3
-; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX7-NEXT: v_mul_f32_e64 v4, 1.0, s2
-; GFX7-NEXT: v_alignbit_b32 v3, v3, v4, 16
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v3
+; GFX7-NEXT: v_mul_f32_e64 v3, 1.0, s2
+; GFX7-NEXT: v_lshr_b64 v[3:4], v[3:4], 16
; GFX7-NEXT: v_mul_f32_e64 v4, 1.0, s7
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX7-NEXT: v_mul_f32_e64 v5, 1.0, s6
-; GFX7-NEXT: v_alignbit_b32 v4, v4, v5, 16
+; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v4
+; GFX7-NEXT: v_mul_f32_e64 v4, 1.0, s6
+; GFX7-NEXT: v_lshr_b64 v[4:5], v[4:5], 16
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX7-NEXT: v_cndmask_b32_e32 v0, v4, v3, vcc
; GFX7-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc
@@ -46308,6 +46342,7 @@ define amdgpu_ps <2 x i32> @s_select_v4bf16(<4 x bfloat> inreg %a, <4 x bfloat>
;
; GFX1250-LABEL: s_select_v4bf16:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
; GFX1250-NEXT: v_dual_mov_b32 v1, s1 :: v_dual_mov_b32 v2, s0
; GFX1250-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
@@ -46554,6 +46589,7 @@ define amdgpu_ps <2 x i32> @s_vselect_v4bf16(<4 x bfloat> inreg %a, <4 x bfloat>
;
; GFX1250TRUE16-LABEL: s_vselect_v4bf16:
; GFX1250TRUE16: ; %bb.0:
+; GFX1250TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
; GFX1250TRUE16-NEXT: s_lshr_b32 s7, s1, 16
; GFX1250TRUE16-NEXT: s_lshr_b32 s9, s0, 16
; GFX1250TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
@@ -46577,6 +46613,7 @@ define amdgpu_ps <2 x i32> @s_vselect_v4bf16(<4 x bfloat> inreg %a, <4 x bfloat>
;
; GFX1250FAKE16-LABEL: s_vselect_v4bf16:
; GFX1250FAKE16: ; %bb.0:
+; GFX1250FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
; GFX1250FAKE16-NEXT: s_lshr_b32 s4, s1, 16
; GFX1250FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v3
; GFX1250FAKE16-NEXT: v_dual_mov_b32 v4, s4 :: v_dual_mov_b32 v5, s1
diff --git a/llvm/test/CodeGen/Generic/half-op.ll b/llvm/test/CodeGen/Generic/half-op.ll
index 1037d8e20cc10..30509efcba8bb 100644
--- a/llvm/test/CodeGen/Generic/half-op.ll
+++ b/llvm/test/CodeGen/Generic/half-op.ll
@@ -8,37 +8,37 @@
; RUN: %if aarch64-registered-target %{ llc %s -o - -mtriple=arm64ec-pc-windows-msvc | FileCheck %s --check-prefixes=ALL,CHECK-NEG-ABS,CHECK-COPYSIGN,CHECK-FMA %}
; RUN: %if amdgpu-registered-target %{ llc %s -o - -mtriple=amdgcn-amd-amdhsa | FileCheck %s --check-prefixes=ALL,CHECK-NEG-ABS,CHECK-COPYSIGN,CHECK-FMA %}
; RUN: %if arc-registered-target %{ llc %s -o - -mtriple=arc-elf | FileCheck %s --check-prefixes=ALL,BAD-NEG-ABS,BAD-COPYSIGN,BAD-FMA %}
-; RUN: %if arm-registered-target %{ llc %s -o - -mtriple=arm-unknown-linux-gnueabi | FileCheck %s --check-prefixes=ALL,CHECK-NEG-ABS,CHECK-COPYSIGN,BAD-FMA %}
-; RUN: %if arm-registered-target %{ llc %s -o - -mtriple=thumbv7em-none-eabi | FileCheck %s --check-prefixes=ALL,CHECK-NEG-ABS,CHECK-COPYSIGN,BAD-FMA %}
-; RUN: %if avr-registered-target %{ llc %s -o - -mtriple=avr-none | FileCheck %s --check-prefixes=ALL,CHECK-NEG-ABS,CHECK-COPYSIGN,BAD-FMA %}
+; RUN: %if arm-registered-target %{ llc %s -o - -mtriple=arm-unknown-linux-gnueabi | FileCheck %s --check-prefixes=ALL,CHECK-NEG-ABS,CHECK-COPYSIGN %}
+; RUN: %if arm-registered-target %{ llc %s -o - -mtriple=thumbv7em-none-eabi | FileCheck %s --check-prefixes=ALL,CHECK-NEG-ABS,CHECK-COPYSIGN,CHECK-FMA %}
+; RUN: %if avr-registered-target %{ llc %s -o - -mtriple=avr-none | FileCheck %s --check-prefixes=ALL,CHECK-NEG-ABS,CHECK-COPYSIGN,CHECK-FMA %}
; FIXME: BPF has a compiler error
; RUN: %if csky-registered-target %{ llc %s -o - -mtriple=csky-unknown-linux-gnuabiv2 | FileCheck %s --check-prefixes=ALL,BAD-NEG-ABS,BAD-COPYSIGN,BAD-FMA %}
; RUN: %if csky-registered-target %{ llc %s -o - -mtriple=csky-unknown-linux-gnuabiv2 -mcpu=ck860fv -mattr=+hard-float | FileCheck %s --check-prefixes=ALL,BAD-NEG-ABS,BAD-COPYSIGN,BAD-FMA %}
; FIXME: directx has a compiler error
-; RUN: %if hexagon-registered-target %{ llc %s -o - -mtriple=hexagon-unknown-linux-musl | FileCheck %s --check-prefixes=ALL,CHECK-NEG-ABS,CHECK-COPYSIGN,BAD-FMA %}
+; RUN: %if hexagon-registered-target %{ llc %s -o - -mtriple=hexagon-unknown-linux-musl | FileCheck %s --check-prefixes=ALL,CHECK-NEG-ABS,CHECK-COPYSIGN,CHECK-FMA %}
; RUN: %if lanai-registered-target %{ llc %s -o - -mtriple=lanai-unknown-unknown | FileCheck %s --check-prefixes=ALL,BAD-NEG-ABS,BAD-COPYSIGN,BAD-FMA %}
-; RUN: %if loongarch-registered-target %{ llc %s -o - -mtriple=loongarch32-unknown-linux-gnu | FileCheck %s --check-prefixes=ALL,CHECK-NEG-ABS,CHECK-COPYSIGN,BAD-FMA %}
-; RUN: %if loongarch-registered-target %{ llc %s -o - -mtriple=loongarch64-unknown-linux-gnu | FileCheck %s --check-prefixes=ALL,CHECK-NEG-ABS,CHECK-COPYSIGN,BAD-FMA %}
-; RUN: %if loongarch-registered-target %{ llc %s -o - -mtriple=loongarch64-unknown-linux-gnu -mattr=+f | FileCheck %s --check-prefixes=ALL,CHECK-NEG-ABS,CHECK-COPYSIGN,BAD-FMA %}
+; RUN: %if loongarch-registered-target %{ llc %s -o - -mtriple=loongarch32-unknown-linux-gnu | FileCheck %s --check-prefixes=ALL,CHECK-NEG-ABS,CHECK-COPYSIGN,CHECK-FMA %}
+; RUN: %if loongarch-registered-target %{ llc %s -o - -mtriple=loongarch64-unknown-linux-gnu | FileCheck %s --check-prefixes=ALL,CHECK-NEG-ABS,CHECK-COPYSIGN,CHECK-FMA %}
+; RUN: %if loongarch-registered-target %{ llc %s -o - -mtriple=loongarch64-unknown-linux-gnu -mattr=+f | FileCheck %s --check-prefixes=ALL,CHECK-NEG-ABS,CHECK-COPYSIGN,CHECK-FMA %}
; RUN: %if m68k-registered-target %{ llc %s -o - -mtriple=m68k-unknown-linux-gnu | FileCheck %s --check-prefixes=ALL,BAD-NEG-ABS,BAD-COPYSIGN,BAD-FMA %}
-; RUN: %if mips-registered-target %{ llc %s -o - -mtriple=mips-unknown-linux-gnu | FileCheck %s --check-prefixes=ALL,CHECK-NEG-ABS,CHECK-COPYSIGN,BAD-FMA %}
-; RUN: %if mips-registered-target %{ llc %s -o - -mtriple=mips64-unknown-linux-gnuabi64 | FileCheck %s --check-prefixes=ALL,CHECK-NEG-ABS,CHECK-COPYSIGN,BAD-FMA %}
-; RUN: %if mips-registered-target %{ llc %s -o - -mtriple=mips64el-unknown-linux-gnuabi64 | FileCheck %s --check-prefixes=ALL,CHECK-NEG-ABS,CHECK-COPYSIGN,BAD-FMA %}
-; RUN: %if mips-registered-target %{ llc %s -o - -mtriple=mipsel-unknown-linux-gnu | FileCheck %s --check-prefixes=ALL,CHECK-NEG-ABS,CHECK-COPYSIGN,BAD-FMA %}
+; RUN: %if mips-registered-target %{ llc %s -o - -mtriple=mips-unknown-linux-gnu | FileCheck %s --check-prefixes=ALL,CHECK-NEG-ABS,CHECK-COPYSIGN,CHECK-FMA %}
+; RUN: %if mips-registered-target %{ llc %s -o - -mtriple=mips64-unknown-linux-gnuabi64 | FileCheck %s --check-prefixes=ALL,CHECK-NEG-ABS,CHECK-COPYSIGN,CHECK-FMA %}
+; RUN: %if mips-registered-target %{ llc %s -o - -mtriple=mips64el-unknown-linux-gnuabi64 | FileCheck %s --check-prefixes=ALL,CHECK-NEG-ABS,CHECK-COPYSIGN,CHECK-FMA %}
+; RUN: %if mips-registered-target %{ llc %s -o - -mtriple=mipsel-unknown-linux-gnu | FileCheck %s --check-prefixes=ALL,CHECK-NEG-ABS,CHECK-COPYSIGN,CHECK-FMA %}
; RUN: %if msp430-registered-target %{ llc %s -o - -mtriple=msp430-none-elf | FileCheck %s --check-prefixes=ALL,BAD-NEG-ABS,BAD-COPYSIGN,BAD-FMA %}
; RUN: %if nvptx-registered-target %{ llc %s -o - -mtriple=nvptx64-nvidia-cuda | FileCheck %s --check-prefixes=NOCRASH %}
; RUN: %if powerpc-registered-target %{ llc %s -o - -mtriple=powerpc-unknown-linux-gnu | FileCheck %s --check-prefixes=ALL,BAD-NEG-ABS,BAD-COPYSIGN,BAD-FMA %}
; RUN: %if powerpc-registered-target %{ llc %s -o - -mtriple=powerpc64-unknown-linux-gnu | FileCheck %s --check-prefixes=ALL,BAD-NEG-ABS,BAD-COPYSIGN,BAD-FMA %}
; RUN: %if powerpc-registered-target %{ llc %s -o - -mtriple=powerpc64le-unknown-linux-gnu | FileCheck %s --check-prefixes=ALL,BAD-NEG-ABS,BAD-COPYSIGN,BAD-FMA %}
-; RUN: %if riscv-registered-target %{ llc %s -o - -mtriple=riscv32-unknown-linux-gnu | FileCheck %s --check-prefixes=ALL,CHECK-NEG-ABS,CHECK-COPYSIGN,BAD-FMA %}
-; RUN: %if riscv-registered-target %{ llc %s -o - -mtriple=riscv64-unknown-linux-gnu | FileCheck %s --check-prefixes=ALL,CHECK-NEG-ABS,CHECK-COPYSIGN,BAD-FMA %}
-; RUN: %if sparc-registered-target %{ llc %s -o - -mtriple=sparc-unknown-linux-gnu | FileCheck %s --check-prefixes=ALL,CHECK-NEG-ABS,CHECK-COPYSIGN,BAD-FMA %}
-; RUN: %if sparc-registered-target %{ llc %s -o - -mtriple=sparc64-unknown-linux-gnu | FileCheck %s --check-prefixes=ALL,CHECK-NEG-ABS,CHECK-COPYSIGN,BAD-FMA %}
+; RUN: %if riscv-registered-target %{ llc %s -o - -mtriple=riscv32-unknown-linux-gnu | FileCheck %s --check-prefixes=ALL,CHECK-NEG-ABS,CHECK-COPYSIGN,CHECK-FMA %}
+; RUN: %if riscv-registered-target %{ llc %s -o - -mtriple=riscv64-unknown-linux-gnu | FileCheck %s --check-prefixes=ALL,CHECK-NEG-ABS,CHECK-COPYSIGN,CHECK-FMA %}
+; RUN: %if sparc-registered-target %{ llc %s -o - -mtriple=sparc-unknown-linux-gnu | FileCheck %s --check-prefixes=ALL,CHECK-NEG-ABS,CHECK-COPYSIGN,CHECK-FMA %}
+; RUN: %if sparc-registered-target %{ llc %s -o - -mtriple=sparc64-unknown-linux-gnu | FileCheck %s --check-prefixes=ALL,CHECK-NEG-ABS,CHECK-COPYSIGN,CHECK-FMA %}
; RUN: %if spirv-registered-target %{ llc %s -o - -mtriple=spirv-unknown-unknown | FileCheck %s --check-prefixes=NOCRASH %}
; RUN: %if systemz-registered-target %{ llc %s -o - -mtriple=s390x-unknown-linux-gnu | FileCheck %s --check-prefixes=ALL,CHECK-NEG-ABS,CHECK-COPYSIGN,BAD-FMA %}
; RUN: %if ve-registered-target %{ llc %s -o - -mtriple=ve-unknown-unknown | FileCheck %s --check-prefixes=ALL,BAD-NEG-ABS,BAD-COPYSIGN,BAD-FMA %}
; RUN: %if webassembly-registered-target %{ llc %s -o - -mtriple=wasm32-unknown-unknown | FileCheck %s --check-prefixes=ALL,BAD-NEG-ABS,BAD-COPYSIGN,BAD-FMA %}
-; RUN: %if x86-registered-target %{ llc %s -o - -mtriple=i686-unknown-linux-gnu | FileCheck %s --check-prefixes=ALL,CHECK-NEG-ABS,CHECK-COPYSIGN,BAD-FMA %}
+; RUN: %if x86-registered-target %{ llc %s -o - -mtriple=i686-unknown-linux-gnu | FileCheck %s --check-prefixes=ALL,CHECK-NEG-ABS,CHECK-COPYSIGN,CHECK-FMA %}
; RUN: %if x86-registered-target %{ llc %s -o - -mtriple=x86_64-pc-windows-msvc | FileCheck %s --check-prefixes=ALL,CHECK-NEG-ABS,CHECK-COPYSIGN,BAD-FMA %}
; RUN: %if x86-registered-target %{ llc %s -o - -mtriple=x86_64-unknown-linux-gnu | FileCheck %s --check-prefixes=ALL,CHECK-NEG-ABS,CHECK-COPYSIGN,BAD-FMA %}
; RUN: %if xcore-registered-target %{ llc %s -o - -mtriple=xcore-unknown-unknown | FileCheck %s --check-prefixes=ALL,BAD-NEG-ABS,BAD-COPYSIGN,BAD-FMA %}
>From 7106650dbec428fb4cd771e7dcd8ce273506bf1c Mon Sep 17 00:00:00 2001
From: Folkert de Vries <folkert at folkertdev.nl>
Date: Mon, 15 Dec 2025 00:44:51 +0100
Subject: [PATCH 5/5] fix the other targets
---
.../SelectionDAG/LegalizeFloatTypes.cpp | 27 ++++++++++++++++++-
llvm/lib/CodeGen/TargetLoweringBase.cpp | 5 ++++
llvm/test/CodeGen/Generic/half-op.ll | 6 ++---
3 files changed, 34 insertions(+), 4 deletions(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp
index ec844adb4e46e..debd659d9902e 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp
@@ -3074,10 +3074,34 @@ SDValue DAGTypeLegalizer::PromoteFloatRes_BinOp(SDNode *N) {
SDValue DAGTypeLegalizer::PromoteFloatRes_FMAD(SDNode *N) {
EVT VT = N->getValueType(0);
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
+
SDValue Op0 = GetPromotedFloat(N->getOperand(0));
SDValue Op1 = GetPromotedFloat(N->getOperand(1));
SDValue Op2 = GetPromotedFloat(N->getOperand(2));
+ // If f16 fma is not natively supported, the value must be promoted to an f64
+ // (and not to f32!) to prevent double rounding issues.
+ if (VT == MVT::f16 && NVT == MVT::f32) {
+ SDNodeFlags Flags = N->getFlags();
+ SDLoc dl(N);
+
+ SDValue A64 = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Op0, Flags);
+ SDValue B64 = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Op1, Flags);
+ SDValue C64 = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Op2, Flags);
+
+ // Prefer a wide FMA node if available; otherwise expand to mul+add.
+ SDValue WideRes;
+ if (TLI.isFMAFasterThanFMulAndFAdd(DAG.getMachineFunction(), MVT::f64)) {
+ WideRes = DAG.getNode(ISD::FMA, dl, MVT::f64, A64, B64, C64, Flags);
+ } else {
+ SDValue Mul = DAG.getNode(ISD::FMUL, dl, MVT::f64, A64, B64, Flags);
+ WideRes = DAG.getNode(ISD::FADD, dl, MVT::f64, Mul, C64, Flags);
+ }
+
+ return DAG.getNode(ISD::FP_ROUND, dl, NVT, WideRes,
+ DAG.getIntPtrConstant(0, dl, /*isTarget=*/true));
+ }
+
return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1, Op2);
}
@@ -3498,7 +3522,8 @@ SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FMAD(SDNode *N) {
SDValue Res;
if (OVT == MVT::f16) {
- // An f16 fma must go via f64 to prevent double rounding issues.
+ // If f16 fma is not natively supported, the value must be promoted to an
+ // f64 (and not to f32!) to prevent double rounding issues.
SDValue A64 = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Op0, Flags);
SDValue B64 = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Op1, Flags);
SDValue C64 = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Op2, Flags);
diff --git a/llvm/lib/CodeGen/TargetLoweringBase.cpp b/llvm/lib/CodeGen/TargetLoweringBase.cpp
index 060b1ddc2ef39..bb5ecc21aa1ab 100644
--- a/llvm/lib/CodeGen/TargetLoweringBase.cpp
+++ b/llvm/lib/CodeGen/TargetLoweringBase.cpp
@@ -773,6 +773,11 @@ void TargetLoweringBase::initActions() {
}
}
+ // If f16 fma is not natively supported, the value must be promoted to an f64
+ // (and not to f32!) to prevent double rounding issues.
+ AddPromotedToType(ISD::FMA, MVT::f16, MVT::f64);
+ AddPromotedToType(ISD::STRICT_FMA, MVT::f16, MVT::f64);
+
// Set default actions for various operations.
for (MVT VT : MVT::all_valuetypes()) {
// Default all indexed load / store to expand.
diff --git a/llvm/test/CodeGen/Generic/half-op.ll b/llvm/test/CodeGen/Generic/half-op.ll
index 30509efcba8bb..09365a60c962d 100644
--- a/llvm/test/CodeGen/Generic/half-op.ll
+++ b/llvm/test/CodeGen/Generic/half-op.ll
@@ -39,9 +39,9 @@
; RUN: %if ve-registered-target %{ llc %s -o - -mtriple=ve-unknown-unknown | FileCheck %s --check-prefixes=ALL,BAD-NEG-ABS,BAD-COPYSIGN,BAD-FMA %}
; RUN: %if webassembly-registered-target %{ llc %s -o - -mtriple=wasm32-unknown-unknown | FileCheck %s --check-prefixes=ALL,BAD-NEG-ABS,BAD-COPYSIGN,BAD-FMA %}
; RUN: %if x86-registered-target %{ llc %s -o - -mtriple=i686-unknown-linux-gnu | FileCheck %s --check-prefixes=ALL,CHECK-NEG-ABS,CHECK-COPYSIGN,CHECK-FMA %}
-; RUN: %if x86-registered-target %{ llc %s -o - -mtriple=x86_64-pc-windows-msvc | FileCheck %s --check-prefixes=ALL,CHECK-NEG-ABS,CHECK-COPYSIGN,BAD-FMA %}
-; RUN: %if x86-registered-target %{ llc %s -o - -mtriple=x86_64-unknown-linux-gnu | FileCheck %s --check-prefixes=ALL,CHECK-NEG-ABS,CHECK-COPYSIGN,BAD-FMA %}
-; RUN: %if xcore-registered-target %{ llc %s -o - -mtriple=xcore-unknown-unknown | FileCheck %s --check-prefixes=ALL,BAD-NEG-ABS,BAD-COPYSIGN,BAD-FMA %}
+; RUN: %if x86-registered-target %{ llc %s -o - -mtriple=x86_64-pc-windows-msvc | FileCheck %s --check-prefixes=ALL,CHECK-NEG-ABS,CHECK-COPYSIGN,CHECK-FMA %}
+; RUN: %if x86-registered-target %{ llc %s -o - -mtriple=x86_64-unknown-linux-gnu | FileCheck %s --check-prefixes=ALL,CHECK-NEG-ABS,CHECK-COPYSIGN,CHECK-FMA %}
+; RUN: %if xcore-registered-target %{ llc %s -o - -mtriple=xcore-unknown-unknown | FileCheck %s --check-prefixes=ALL,BAD-NEG-ABS,BAD-COPYSIGN,CHECK-FMA %}
; RUN: %if xtensa-registered-target %{ llc %s -o - -mtriple=xtensa-none-elf | FileCheck %s --check-prefixes=ALL,BAD-NEG-ABS,BAD-COPYSIGN,CHECK-FMA %}
; Note that arm64ec labels are quoted, hence the `{{"?}}:`.
More information about the llvm-commits
mailing list