[llvm] 6849789 - [NFC][AMDGPU] Add tests for an fmul split from its fadd/fsub user (#213303)
via llvm-commits
llvm-commits at lists.llvm.org
Sun Aug 2 13:00:42 PDT 2026
Author: Dmitry Sidorov
Date: 2026-08-02T22:00:37+02:00
New Revision: 6849789a8fa1fc4a09f4f81dc655235e648eb15a
URL: https://github.com/llvm/llvm-project/commit/6849789a8fa1fc4a09f4f81dc655235e648eb15a
DIFF: https://github.com/llvm/llvm-project/commit/6849789a8fa1fc4a09f4f81dc655235e648eb15a.diff
LOG: [NFC][AMDGPU] Add tests for an fmul split from its fadd/fsub user (#213303)
Record the current behaviour over the types and subtargets where the
fusion rules differ, f32, f16, bf16, f64, v2f32 and v2f16, with
denormals both enabled and flushed, with and without the contract flags,
and under -fp-contract=fast.
Contributes to https://github.com/llvm/llvm-project/issues/211092
Added:
llvm/test/CodeGen/AMDGPU/sink-fmul-fadd.ll
llvm/test/Transforms/CodeGenPrepare/AMDGPU/sink-fmul-fadd-contract-fast.ll
llvm/test/Transforms/CodeGenPrepare/AMDGPU/sink-fmul-fadd.ll
Modified:
Removed:
################################################################################
diff --git a/llvm/test/CodeGen/AMDGPU/sink-fmul-fadd.ll b/llvm/test/CodeGen/AMDGPU/sink-fmul-fadd.ll
new file mode 100644
index 0000000000000..c01e3b436a3f1
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/sink-fmul-fadd.ll
@@ -0,0 +1,1011 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=amdgpu8.03-amd-amdhsa < %s | FileCheck -check-prefix=GFX8 %s
+; RUN: llc -mtriple=amdgpu9.00-amd-amdhsa < %s | FileCheck -check-prefix=GFX9 %s
+; RUN: llc -mtriple=amdgpu10.30-amd-amdhsa < %s | FileCheck -check-prefix=GFX10 %s
+
+define float @fma_in_loop_f32(float %a, float %b, i32 %n) {
+; CHECK-LABEL: fma_in_loop_f32:
+; CHECK: ; %bb.0: ; %entry
+; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT: v_mov_b32_e32 v3, v0
+; CHECK-NEXT: v_mov_b32_e32 v0, 0
+; CHECK-NEXT: s_mov_b32 s6, 0
+; CHECK-NEXT: s_mov_b64 s[4:5], 0
+; CHECK-NEXT: .LBB0_1: ; %loop
+; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: s_add_i32 s6, s6, 1
+; CHECK-NEXT: v_cmp_ge_i32_e32 vcc, s6, v2
+; CHECK-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; CHECK-NEXT: v_fma_f32 v0, v3, v1, v0
+; CHECK-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; CHECK-NEXT: s_cbranch_execnz .LBB0_1
+; CHECK-NEXT: ; %bb.2: ; %exit
+; CHECK-NEXT: s_or_b64 exec, exec, s[4:5]
+; CHECK-NEXT: s_setpc_b64 s[30:31]
+; GFX8-LABEL: fma_in_loop_f32:
+; GFX8: ; %bb.0: ; %entry
+; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT: v_mul_f32_e32 v1, v0, v1
+; GFX8-NEXT: v_mov_b32_e32 v0, 0
+; GFX8-NEXT: s_mov_b32 s6, 0
+; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: .LBB0_1: ; %loop
+; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT: s_add_i32 s6, s6, 1
+; GFX8-NEXT: v_cmp_ge_i32_e32 vcc, s6, v2
+; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX8-NEXT: s_cbranch_execnz .LBB0_1
+; GFX8-NEXT: ; %bb.2: ; %exit
+; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX8-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: fma_in_loop_f32:
+; GFX9: ; %bb.0: ; %entry
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_mul_f32_e32 v1, v0, v1
+; GFX9-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NEXT: s_mov_b32 s6, 0
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: .LBB0_1: ; %loop
+; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT: s_add_i32 s6, s6, 1
+; GFX9-NEXT: v_cmp_ge_i32_e32 vcc, s6, v2
+; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX9-NEXT: s_cbranch_execnz .LBB0_1
+; GFX9-NEXT: ; %bb.2: ; %exit
+; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: fma_in_loop_f32:
+; GFX10: ; %bb.0: ; %entry
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_mul_f32_e32 v1, v0, v1
+; GFX10-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-NEXT: s_mov_b32 s4, 0
+; GFX10-NEXT: s_mov_b32 s5, 0
+; GFX10-NEXT: .LBB0_1: ; %loop
+; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX10-NEXT: s_add_i32 s5, s5, 1
+; GFX10-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX10-NEXT: v_cmp_ge_i32_e32 vcc_lo, s5, v2
+; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: s_cbranch_execnz .LBB0_1
+; GFX10-NEXT: ; %bb.2: ; %exit
+; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+entry:
+ %mul = fmul contract float %a, %b
+ br label %loop
+
+loop:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+ %acc = phi float [ 0.000000e+00, %entry ], [ %add, %loop ]
+ %add = fadd contract float %acc, %mul
+ %i.next = add i32 %i, 1
+ %cmp = icmp slt i32 %i.next, %n
+ br i1 %cmp, label %loop, label %exit
+
+exit:
+ ret float %add
+}
+
+define float @fsub_in_loop_f32(float %a, float %b, i32 %n) {
+; CHECK-LABEL: fsub_in_loop_f32:
+; CHECK: ; %bb.0: ; %entry
+; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT: v_mov_b32_e32 v3, v0
+; CHECK-NEXT: v_mov_b32_e32 v0, 0
+; CHECK-NEXT: s_mov_b32 s6, 0
+; CHECK-NEXT: s_mov_b64 s[4:5], 0
+; CHECK-NEXT: .LBB1_1: ; %loop
+; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: s_add_i32 s6, s6, 1
+; CHECK-NEXT: v_cmp_ge_i32_e32 vcc, s6, v2
+; CHECK-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; CHECK-NEXT: v_fma_f32 v0, -v3, v1, v0
+; CHECK-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; CHECK-NEXT: s_cbranch_execnz .LBB1_1
+; CHECK-NEXT: ; %bb.2: ; %exit
+; CHECK-NEXT: s_or_b64 exec, exec, s[4:5]
+; CHECK-NEXT: s_setpc_b64 s[30:31]
+; GFX8-LABEL: fsub_in_loop_f32:
+; GFX8: ; %bb.0: ; %entry
+; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT: v_mul_f32_e32 v1, v0, v1
+; GFX8-NEXT: v_mov_b32_e32 v0, 0
+; GFX8-NEXT: s_mov_b32 s6, 0
+; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: .LBB1_1: ; %loop
+; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT: s_add_i32 s6, s6, 1
+; GFX8-NEXT: v_cmp_ge_i32_e32 vcc, s6, v2
+; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT: v_sub_f32_e32 v0, v0, v1
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX8-NEXT: s_cbranch_execnz .LBB1_1
+; GFX8-NEXT: ; %bb.2: ; %exit
+; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX8-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: fsub_in_loop_f32:
+; GFX9: ; %bb.0: ; %entry
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_mul_f32_e32 v1, v0, v1
+; GFX9-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NEXT: s_mov_b32 s6, 0
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: .LBB1_1: ; %loop
+; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT: s_add_i32 s6, s6, 1
+; GFX9-NEXT: v_cmp_ge_i32_e32 vcc, s6, v2
+; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-NEXT: v_sub_f32_e32 v0, v0, v1
+; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX9-NEXT: s_cbranch_execnz .LBB1_1
+; GFX9-NEXT: ; %bb.2: ; %exit
+; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: fsub_in_loop_f32:
+; GFX10: ; %bb.0: ; %entry
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_mul_f32_e32 v1, v0, v1
+; GFX10-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-NEXT: s_mov_b32 s4, 0
+; GFX10-NEXT: s_mov_b32 s5, 0
+; GFX10-NEXT: .LBB1_1: ; %loop
+; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX10-NEXT: s_add_i32 s5, s5, 1
+; GFX10-NEXT: v_sub_f32_e32 v0, v0, v1
+; GFX10-NEXT: v_cmp_ge_i32_e32 vcc_lo, s5, v2
+; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: s_cbranch_execnz .LBB1_1
+; GFX10-NEXT: ; %bb.2: ; %exit
+; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+entry:
+ %mul = fmul contract float %a, %b
+ br label %loop
+
+loop:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+ %acc = phi float [ 0.000000e+00, %entry ], [ %sub, %loop ]
+ %sub = fsub contract float %acc, %mul
+ %i.next = add i32 %i, 1
+ %cmp = icmp slt i32 %i.next, %n
+ br i1 %cmp, label %loop, label %exit
+
+exit:
+ ret float %sub
+}
+
+define half @fma_in_loop_f16(half %a, half %b, i32 %n) {
+; CHECK-LABEL: fma_in_loop_f16:
+; CHECK: ; %bb.0: ; %entry
+; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT: v_mov_b32_e32 v3, v0
+; CHECK-NEXT: v_mov_b32_e32 v0, 0
+; CHECK-NEXT: s_mov_b32 s6, 0
+; CHECK-NEXT: s_mov_b64 s[4:5], 0
+; CHECK-NEXT: .LBB2_1: ; %loop
+; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: s_add_i32 s6, s6, 1
+; CHECK-NEXT: v_cmp_ge_i32_e32 vcc, s6, v2
+; CHECK-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; CHECK-NEXT: v_fma_f16 v0, v3, v1, v0
+; CHECK-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; CHECK-NEXT: s_cbranch_execnz .LBB2_1
+; CHECK-NEXT: ; %bb.2: ; %exit
+; CHECK-NEXT: s_or_b64 exec, exec, s[4:5]
+; CHECK-NEXT: s_setpc_b64 s[30:31]
+; GFX8-LABEL: fma_in_loop_f16:
+; GFX8: ; %bb.0: ; %entry
+; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT: v_mul_f16_e32 v1, v0, v1
+; GFX8-NEXT: v_mov_b32_e32 v0, 0
+; GFX8-NEXT: s_mov_b32 s6, 0
+; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: .LBB2_1: ; %loop
+; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT: s_add_i32 s6, s6, 1
+; GFX8-NEXT: v_cmp_ge_i32_e32 vcc, s6, v2
+; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT: v_add_f16_e32 v0, v0, v1
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX8-NEXT: s_cbranch_execnz .LBB2_1
+; GFX8-NEXT: ; %bb.2: ; %exit
+; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX8-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: fma_in_loop_f16:
+; GFX9: ; %bb.0: ; %entry
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_mul_f16_e32 v1, v0, v1
+; GFX9-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NEXT: s_mov_b32 s6, 0
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: .LBB2_1: ; %loop
+; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT: s_add_i32 s6, s6, 1
+; GFX9-NEXT: v_cmp_ge_i32_e32 vcc, s6, v2
+; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-NEXT: v_add_f16_e32 v0, v0, v1
+; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX9-NEXT: s_cbranch_execnz .LBB2_1
+; GFX9-NEXT: ; %bb.2: ; %exit
+; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: fma_in_loop_f16:
+; GFX10: ; %bb.0: ; %entry
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_mul_f16_e32 v1, v0, v1
+; GFX10-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-NEXT: s_mov_b32 s4, 0
+; GFX10-NEXT: s_mov_b32 s5, 0
+; GFX10-NEXT: .LBB2_1: ; %loop
+; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX10-NEXT: s_add_i32 s5, s5, 1
+; GFX10-NEXT: v_add_f16_e32 v0, v0, v1
+; GFX10-NEXT: v_cmp_ge_i32_e32 vcc_lo, s5, v2
+; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: s_cbranch_execnz .LBB2_1
+; GFX10-NEXT: ; %bb.2: ; %exit
+; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+entry:
+ %mul = fmul contract half %a, %b
+ br label %loop
+
+loop:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+ %acc = phi half [ 0.000000e+00, %entry ], [ %add, %loop ]
+ %add = fadd contract half %acc, %mul
+ %i.next = add i32 %i, 1
+ %cmp = icmp slt i32 %i.next, %n
+ br i1 %cmp, label %loop, label %exit
+
+exit:
+ ret half %add
+}
+
+define double @fma_in_loop_f64(double %a, double %b, i32 %n) {
+; CHECK-LABEL: fma_in_loop_f64:
+; CHECK: ; %bb.0: ; %entry
+; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT: v_mov_b32_e32 v5, 0
+; CHECK-NEXT: v_mov_b32_e32 v6, 0
+; CHECK-NEXT: s_mov_b32 s6, 0
+; CHECK-NEXT: s_mov_b64 s[4:5], 0
+; CHECK-NEXT: .LBB3_1: ; %loop
+; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: v_fma_f64 v[5:6], v[0:1], v[2:3], v[5:6]
+; CHECK-NEXT: s_add_i32 s6, s6, 1
+; CHECK-NEXT: v_cmp_ge_i32_e32 vcc, s6, v4
+; CHECK-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; CHECK-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; CHECK-NEXT: s_cbranch_execnz .LBB3_1
+; CHECK-NEXT: ; %bb.2: ; %exit
+; CHECK-NEXT: s_or_b64 exec, exec, s[4:5]
+; CHECK-NEXT: v_mov_b32_e32 v0, v5
+; CHECK-NEXT: v_mov_b32_e32 v1, v6
+; CHECK-NEXT: s_setpc_b64 s[30:31]
+; GFX8-LABEL: fma_in_loop_f64:
+; GFX8: ; %bb.0: ; %entry
+; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT: v_mul_f64 v[2:3], v[0:1], v[2:3]
+; GFX8-NEXT: v_mov_b32_e32 v0, 0
+; GFX8-NEXT: v_mov_b32_e32 v1, 0
+; GFX8-NEXT: s_mov_b32 s6, 0
+; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: .LBB3_1: ; %loop
+; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT: v_add_f64 v[0:1], v[0:1], v[2:3]
+; GFX8-NEXT: s_add_i32 s6, s6, 1
+; GFX8-NEXT: v_cmp_ge_i32_e32 vcc, s6, v4
+; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX8-NEXT: s_cbranch_execnz .LBB3_1
+; GFX8-NEXT: ; %bb.2: ; %exit
+; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX8-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: fma_in_loop_f64:
+; GFX9: ; %bb.0: ; %entry
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_mul_f64 v[2:3], v[0:1], v[2:3]
+; GFX9-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NEXT: v_mov_b32_e32 v1, 0
+; GFX9-NEXT: s_mov_b32 s6, 0
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: .LBB3_1: ; %loop
+; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT: v_add_f64 v[0:1], v[0:1], v[2:3]
+; GFX9-NEXT: s_add_i32 s6, s6, 1
+; GFX9-NEXT: v_cmp_ge_i32_e32 vcc, s6, v4
+; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX9-NEXT: s_cbranch_execnz .LBB3_1
+; GFX9-NEXT: ; %bb.2: ; %exit
+; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: fma_in_loop_f64:
+; GFX10: ; %bb.0: ; %entry
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_mul_f64 v[2:3], v[0:1], v[2:3]
+; GFX10-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-NEXT: s_mov_b32 s4, 0
+; GFX10-NEXT: s_mov_b32 s5, 0
+; GFX10-NEXT: .LBB3_1: ; %loop
+; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX10-NEXT: v_add_f64 v[0:1], v[0:1], v[2:3]
+; GFX10-NEXT: s_add_i32 s5, s5, 1
+; GFX10-NEXT: v_cmp_ge_i32_e32 vcc_lo, s5, v4
+; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: s_cbranch_execnz .LBB3_1
+; GFX10-NEXT: ; %bb.2: ; %exit
+; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+entry:
+ %mul = fmul contract double %a, %b
+ br label %loop
+
+loop:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+ %acc = phi double [ 0.000000e+00, %entry ], [ %add, %loop ]
+ %add = fadd contract double %acc, %mul
+ %i.next = add i32 %i, 1
+ %cmp = icmp slt i32 %i.next, %n
+ br i1 %cmp, label %loop, label %exit
+
+exit:
+ ret double %add
+}
+
+; No fast-math flags needed, v_mad_f32 is exact with denormals flushed.
+define float @mad_in_loop_f32(float %a, float %b, i32 %n) #0 {
+; CHECK-LABEL: mad_in_loop_f32:
+; CHECK: ; %bb.0: ; %entry
+; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT: v_mov_b32_e32 v3, v0
+; CHECK-NEXT: v_mov_b32_e32 v0, 0
+; CHECK-NEXT: s_mov_b32 s6, 0
+; CHECK-NEXT: s_mov_b64 s[4:5], 0
+; CHECK-NEXT: .LBB4_1: ; %loop
+; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: s_add_i32 s6, s6, 1
+; CHECK-NEXT: v_cmp_ge_i32_e32 vcc, s6, v2
+; CHECK-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; CHECK-NEXT: v_mac_f32_e32 v0, v3, v1
+; CHECK-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; CHECK-NEXT: s_cbranch_execnz .LBB4_1
+; CHECK-NEXT: ; %bb.2: ; %exit
+; CHECK-NEXT: s_or_b64 exec, exec, s[4:5]
+; CHECK-NEXT: s_setpc_b64 s[30:31]
+; GFX8-LABEL: mad_in_loop_f32:
+; GFX8: ; %bb.0: ; %entry
+; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT: v_mul_f32_e32 v1, v0, v1
+; GFX8-NEXT: v_mov_b32_e32 v0, 0
+; GFX8-NEXT: s_mov_b32 s6, 0
+; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: .LBB4_1: ; %loop
+; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT: s_add_i32 s6, s6, 1
+; GFX8-NEXT: v_cmp_ge_i32_e32 vcc, s6, v2
+; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX8-NEXT: s_cbranch_execnz .LBB4_1
+; GFX8-NEXT: ; %bb.2: ; %exit
+; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX8-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: mad_in_loop_f32:
+; GFX9: ; %bb.0: ; %entry
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_mul_f32_e32 v1, v0, v1
+; GFX9-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NEXT: s_mov_b32 s6, 0
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: .LBB4_1: ; %loop
+; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT: s_add_i32 s6, s6, 1
+; GFX9-NEXT: v_cmp_ge_i32_e32 vcc, s6, v2
+; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX9-NEXT: s_cbranch_execnz .LBB4_1
+; GFX9-NEXT: ; %bb.2: ; %exit
+; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: mad_in_loop_f32:
+; GFX10: ; %bb.0: ; %entry
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_mul_f32_e32 v1, v0, v1
+; GFX10-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-NEXT: s_mov_b32 s4, 0
+; GFX10-NEXT: s_mov_b32 s5, 0
+; GFX10-NEXT: .LBB4_1: ; %loop
+; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX10-NEXT: s_add_i32 s5, s5, 1
+; GFX10-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX10-NEXT: v_cmp_ge_i32_e32 vcc_lo, s5, v2
+; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: s_cbranch_execnz .LBB4_1
+; GFX10-NEXT: ; %bb.2: ; %exit
+; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+entry:
+ %mul = fmul float %a, %b
+ br label %loop
+
+loop:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+ %acc = phi float [ 0.000000e+00, %entry ], [ %add, %loop ]
+ %add = fadd float %acc, %mul
+ %i.next = add i32 %i, 1
+ %cmp = icmp slt i32 %i.next, %n
+ br i1 %cmp, label %loop, label %exit
+
+exit:
+ ret float %add
+}
+
+; Nothing to fuse without contract on both operations.
+define float @no_sink_in_loop_no_contract(float %a, float %b, i32 %n) {
+; CHECK-LABEL: no_sink_in_loop_no_contract:
+; CHECK: ; %bb.0: ; %entry
+; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT: v_mul_f32_e32 v1, v0, v1
+; CHECK-NEXT: v_mov_b32_e32 v0, 0
+; CHECK-NEXT: s_mov_b32 s6, 0
+; CHECK-NEXT: s_mov_b64 s[4:5], 0
+; CHECK-NEXT: .LBB5_1: ; %loop
+; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: s_add_i32 s6, s6, 1
+; CHECK-NEXT: v_cmp_ge_i32_e32 vcc, s6, v2
+; CHECK-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; CHECK-NEXT: v_add_f32_e32 v0, v0, v1
+; CHECK-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; CHECK-NEXT: s_cbranch_execnz .LBB5_1
+; CHECK-NEXT: ; %bb.2: ; %exit
+; CHECK-NEXT: s_or_b64 exec, exec, s[4:5]
+; CHECK-NEXT: s_setpc_b64 s[30:31]
+; GFX8-LABEL: no_sink_in_loop_no_contract:
+; GFX8: ; %bb.0: ; %entry
+; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT: v_mul_f32_e32 v1, v0, v1
+; GFX8-NEXT: v_mov_b32_e32 v0, 0
+; GFX8-NEXT: s_mov_b32 s6, 0
+; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: .LBB5_1: ; %loop
+; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT: s_add_i32 s6, s6, 1
+; GFX8-NEXT: v_cmp_ge_i32_e32 vcc, s6, v2
+; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX8-NEXT: s_cbranch_execnz .LBB5_1
+; GFX8-NEXT: ; %bb.2: ; %exit
+; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX8-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: no_sink_in_loop_no_contract:
+; GFX9: ; %bb.0: ; %entry
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_mul_f32_e32 v1, v0, v1
+; GFX9-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NEXT: s_mov_b32 s6, 0
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: .LBB5_1: ; %loop
+; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT: s_add_i32 s6, s6, 1
+; GFX9-NEXT: v_cmp_ge_i32_e32 vcc, s6, v2
+; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX9-NEXT: s_cbranch_execnz .LBB5_1
+; GFX9-NEXT: ; %bb.2: ; %exit
+; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: no_sink_in_loop_no_contract:
+; GFX10: ; %bb.0: ; %entry
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_mul_f32_e32 v1, v0, v1
+; GFX10-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-NEXT: s_mov_b32 s4, 0
+; GFX10-NEXT: s_mov_b32 s5, 0
+; GFX10-NEXT: .LBB5_1: ; %loop
+; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX10-NEXT: s_add_i32 s5, s5, 1
+; GFX10-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX10-NEXT: v_cmp_ge_i32_e32 vcc_lo, s5, v2
+; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: s_cbranch_execnz .LBB5_1
+; GFX10-NEXT: ; %bb.2: ; %exit
+; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+entry:
+ %mul = fmul float %a, %b
+ br label %loop
+
+loop:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+ %acc = phi float [ 0.000000e+00, %entry ], [ %add, %loop ]
+ %add = fadd float %acc, %mul
+ %i.next = add i32 %i, 1
+ %cmp = icmp slt i32 %i.next, %n
+ br i1 %cmp, label %loop, label %exit
+
+exit:
+ ret float %add
+}
+
+; A multi-use fmul would have to be duplicated rather than moved.
+define float @no_sink_in_loop_multi_use(float %a, float %b, i32 %n) {
+; CHECK-LABEL: no_sink_in_loop_multi_use:
+; CHECK: ; %bb.0: ; %entry
+; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT: v_mul_f32_e32 v0, v0, v1
+; CHECK-NEXT: v_mov_b32_e32 v1, 0
+; CHECK-NEXT: s_mov_b32 s6, 0
+; CHECK-NEXT: s_mov_b64 s[4:5], 0
+; CHECK-NEXT: .LBB6_1: ; %loop
+; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: s_add_i32 s6, s6, 1
+; CHECK-NEXT: v_cmp_ge_i32_e32 vcc, s6, v2
+; CHECK-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; CHECK-NEXT: v_add_f32_e32 v1, v1, v0
+; CHECK-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; CHECK-NEXT: s_cbranch_execnz .LBB6_1
+; CHECK-NEXT: ; %bb.2: ; %exit
+; CHECK-NEXT: s_or_b64 exec, exec, s[4:5]
+; CHECK-NEXT: v_add_f32_e32 v0, v1, v0
+; CHECK-NEXT: s_setpc_b64 s[30:31]
+; GFX8-LABEL: no_sink_in_loop_multi_use:
+; GFX8: ; %bb.0: ; %entry
+; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX8-NEXT: v_mov_b32_e32 v1, 0
+; GFX8-NEXT: s_mov_b32 s6, 0
+; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: .LBB6_1: ; %loop
+; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT: s_add_i32 s6, s6, 1
+; GFX8-NEXT: v_cmp_ge_i32_e32 vcc, s6, v2
+; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT: v_add_f32_e32 v1, v1, v0
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX8-NEXT: s_cbranch_execnz .LBB6_1
+; GFX8-NEXT: ; %bb.2: ; %exit
+; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX8-NEXT: v_add_f32_e32 v0, v1, v0
+; GFX8-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: no_sink_in_loop_multi_use:
+; GFX9: ; %bb.0: ; %entry
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX9-NEXT: v_mov_b32_e32 v1, 0
+; GFX9-NEXT: s_mov_b32 s6, 0
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: .LBB6_1: ; %loop
+; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT: s_add_i32 s6, s6, 1
+; GFX9-NEXT: v_cmp_ge_i32_e32 vcc, s6, v2
+; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-NEXT: v_add_f32_e32 v1, v1, v0
+; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX9-NEXT: s_cbranch_execnz .LBB6_1
+; GFX9-NEXT: ; %bb.2: ; %exit
+; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT: v_add_f32_e32 v0, v1, v0
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: no_sink_in_loop_multi_use:
+; GFX10: ; %bb.0: ; %entry
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX10-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-NEXT: s_mov_b32 s4, 0
+; GFX10-NEXT: s_mov_b32 s5, 0
+; GFX10-NEXT: .LBB6_1: ; %loop
+; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX10-NEXT: s_add_i32 s5, s5, 1
+; GFX10-NEXT: v_add_f32_e32 v1, v1, v0
+; GFX10-NEXT: v_cmp_ge_i32_e32 vcc_lo, s5, v2
+; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: s_cbranch_execnz .LBB6_1
+; GFX10-NEXT: ; %bb.2: ; %exit
+; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: v_add_f32_e32 v0, v1, v0
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+entry:
+ %mul = fmul contract float %a, %b
+ br label %loop
+
+loop:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+ %acc = phi float [ 0.000000e+00, %entry ], [ %add, %loop ]
+ %add = fadd contract float %acc, %mul
+ %i.next = add i32 %i, 1
+ %cmp = icmp slt i32 %i.next, %n
+ br i1 %cmp, label %loop, label %exit
+
+exit:
+ %r = fadd float %add, %mul
+ ret float %r
+}
+
+; gfx10 has no v_mad_f16, and v_fma_f16 is not formed with denormals flushed.
+define half @mad_in_loop_f16(half %a, half %b, i32 %n) #0 {
+; GFX8-LABEL: mad_in_loop_f16:
+; GFX8: ; %bb.0: ; %entry
+; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT: v_mul_f16_e32 v1, v0, v1
+; GFX8-NEXT: v_mov_b32_e32 v0, 0
+; GFX8-NEXT: s_mov_b32 s6, 0
+; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: .LBB7_1: ; %loop
+; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT: s_add_i32 s6, s6, 1
+; GFX8-NEXT: v_cmp_ge_i32_e32 vcc, s6, v2
+; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT: v_add_f16_e32 v0, v0, v1
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX8-NEXT: s_cbranch_execnz .LBB7_1
+; GFX8-NEXT: ; %bb.2: ; %exit
+; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX8-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: mad_in_loop_f16:
+; GFX9: ; %bb.0: ; %entry
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_mul_f16_e32 v1, v0, v1
+; GFX9-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NEXT: s_mov_b32 s6, 0
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: .LBB7_1: ; %loop
+; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT: s_add_i32 s6, s6, 1
+; GFX9-NEXT: v_cmp_ge_i32_e32 vcc, s6, v2
+; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-NEXT: v_add_f16_e32 v0, v0, v1
+; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX9-NEXT: s_cbranch_execnz .LBB7_1
+; GFX9-NEXT: ; %bb.2: ; %exit
+; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: mad_in_loop_f16:
+; GFX10: ; %bb.0: ; %entry
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_mul_f16_e32 v1, v0, v1
+; GFX10-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-NEXT: s_mov_b32 s4, 0
+; GFX10-NEXT: s_mov_b32 s5, 0
+; GFX10-NEXT: .LBB7_1: ; %loop
+; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX10-NEXT: s_add_i32 s5, s5, 1
+; GFX10-NEXT: v_add_f16_e32 v0, v0, v1
+; GFX10-NEXT: v_cmp_ge_i32_e32 vcc_lo, s5, v2
+; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: s_cbranch_execnz .LBB7_1
+; GFX10-NEXT: ; %bb.2: ; %exit
+; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+entry:
+ %mul = fmul half %a, %b
+ br label %loop
+
+loop:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+ %acc = phi half [ 0.000000e+00, %entry ], [ %add, %loop ]
+ %add = fadd half %acc, %mul
+ %i.next = add i32 %i, 1
+ %cmp = icmp slt i32 %i.next, %n
+ br i1 %cmp, label %loop, label %exit
+
+exit:
+ ret half %add
+}
+
+; A legal v2f16 has no packed mad, and v_pk_fma_f16 needs denormals enabled.
+define <2 x half> @no_fma_in_loop_v2f16(<2 x half> %a, <2 x half> %b, i32 %n) #0 {
+; GFX8-LABEL: no_fma_in_loop_v2f16:
+; GFX8: ; %bb.0: ; %entry
+; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT: v_mul_f16_sdwa v3, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_mul_f16_e32 v0, v0, v1
+; GFX8-NEXT: v_or_b32_e32 v1, v0, v3
+; GFX8-NEXT: v_mov_b32_e32 v0, 0
+; GFX8-NEXT: s_mov_b32 s6, 0
+; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: .LBB8_1: ; %loop
+; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT: s_add_i32 s6, s6, 1
+; GFX8-NEXT: v_add_f16_sdwa v3, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_add_f16_e32 v0, v0, v1
+; GFX8-NEXT: v_cmp_ge_i32_e32 vcc, s6, v2
+; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT: v_or_b32_e32 v0, v0, v3
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX8-NEXT: s_cbranch_execnz .LBB8_1
+; GFX8-NEXT: ; %bb.2: ; %exit
+; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX8-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: no_fma_in_loop_v2f16:
+; GFX9: ; %bb.0: ; %entry
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_pk_mul_f16 v1, v0, v1
+; GFX9-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NEXT: s_mov_b32 s6, 0
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: .LBB8_1: ; %loop
+; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT: s_add_i32 s6, s6, 1
+; GFX9-NEXT: v_cmp_ge_i32_e32 vcc, s6, v2
+; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-NEXT: v_pk_add_f16 v0, v0, v1
+; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX9-NEXT: s_cbranch_execnz .LBB8_1
+; GFX9-NEXT: ; %bb.2: ; %exit
+; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: no_fma_in_loop_v2f16:
+; GFX10: ; %bb.0: ; %entry
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_pk_mul_f16 v1, v0, v1
+; GFX10-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-NEXT: s_mov_b32 s4, 0
+; GFX10-NEXT: s_mov_b32 s5, 0
+; GFX10-NEXT: .LBB8_1: ; %loop
+; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX10-NEXT: s_add_i32 s5, s5, 1
+; GFX10-NEXT: v_pk_add_f16 v0, v0, v1
+; GFX10-NEXT: v_cmp_ge_i32_e32 vcc_lo, s5, v2
+; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: s_cbranch_execnz .LBB8_1
+; GFX10-NEXT: ; %bb.2: ; %exit
+; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+entry:
+ %mul = fmul contract <2 x half> %a, %b
+ br label %loop
+
+loop:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+ %acc = phi <2 x half> [ zeroinitializer, %entry ], [ %add, %loop ]
+ %add = fadd contract <2 x half> %acc, %mul
+ %i.next = add i32 %i, 1
+ %cmp = icmp slt i32 %i.next, %n
+ br i1 %cmp, label %loop, label %exit
+
+exit:
+ ret <2 x half> %add
+}
+
+define <2 x half> @fma_in_loop_v2f16_denormals(<2 x half> %a, <2 x half> %b, i32 %n) {
+; GFX8-LABEL: fma_in_loop_v2f16_denormals:
+; GFX8: ; %bb.0: ; %entry
+; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT: v_mul_f16_sdwa v3, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_mul_f16_e32 v0, v0, v1
+; GFX8-NEXT: v_or_b32_e32 v1, v0, v3
+; GFX8-NEXT: v_mov_b32_e32 v0, 0
+; GFX8-NEXT: s_mov_b32 s6, 0
+; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: .LBB9_1: ; %loop
+; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT: s_add_i32 s6, s6, 1
+; GFX8-NEXT: v_add_f16_sdwa v3, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_add_f16_e32 v0, v0, v1
+; GFX8-NEXT: v_cmp_ge_i32_e32 vcc, s6, v2
+; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT: v_or_b32_e32 v0, v0, v3
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX8-NEXT: s_cbranch_execnz .LBB9_1
+; GFX8-NEXT: ; %bb.2: ; %exit
+; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX8-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: fma_in_loop_v2f16_denormals:
+; GFX9: ; %bb.0: ; %entry
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_pk_mul_f16 v1, v0, v1
+; GFX9-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NEXT: s_mov_b32 s6, 0
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: .LBB9_1: ; %loop
+; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT: s_add_i32 s6, s6, 1
+; GFX9-NEXT: v_cmp_ge_i32_e32 vcc, s6, v2
+; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-NEXT: v_pk_add_f16 v0, v0, v1
+; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX9-NEXT: s_cbranch_execnz .LBB9_1
+; GFX9-NEXT: ; %bb.2: ; %exit
+; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: fma_in_loop_v2f16_denormals:
+; GFX10: ; %bb.0: ; %entry
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_pk_mul_f16 v1, v0, v1
+; GFX10-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-NEXT: s_mov_b32 s4, 0
+; GFX10-NEXT: s_mov_b32 s5, 0
+; GFX10-NEXT: .LBB9_1: ; %loop
+; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX10-NEXT: s_add_i32 s5, s5, 1
+; GFX10-NEXT: v_pk_add_f16 v0, v0, v1
+; GFX10-NEXT: v_cmp_ge_i32_e32 vcc_lo, s5, v2
+; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: s_cbranch_execnz .LBB9_1
+; GFX10-NEXT: ; %bb.2: ; %exit
+; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+entry:
+ %mul = fmul contract <2 x half> %a, %b
+ br label %loop
+
+loop:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+ %acc = phi <2 x half> [ zeroinitializer, %entry ], [ %add, %loop ]
+ %add = fadd contract <2 x half> %acc, %mul
+ %i.next = add i32 %i, 1
+ %cmp = icmp slt i32 %i.next, %n
+ br i1 %cmp, label %loop, label %exit
+
+exit:
+ ret <2 x half> %add
+}
+
+; The bf16 product is rounded back to bf16 only for the fadd to widen it again.
+define bfloat @fma_in_loop_bf16(bfloat %a, bfloat %b, i32 %n) {
+; GFX8-LABEL: fma_in_loop_bf16:
+; GFX8: ; %bb.0: ; %entry
+; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX8-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX8-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v1, vcc, v1, v0
+; GFX8-NEXT: v_add_u32_e32 v1, vcc, 0x7fff, v1
+; GFX8-NEXT: v_or_b32_e32 v3, 0x400000, v0
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v1, v3, vcc
+; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX8-NEXT: v_mov_b32_e32 v0, 0
+; GFX8-NEXT: s_mov_b32 s6, 0
+; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX8-NEXT: .LBB10_1: ; %loop
+; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX8-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX8-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v3, vcc, v3, v0
+; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fff, v3
+; GFX8-NEXT: v_or_b32_e32 v4, 0x400000, v0
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX8-NEXT: s_add_i32 s6, s6, 1
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v3, v4, vcc
+; GFX8-NEXT: v_cmp_ge_i32_e32 vcc, s6, v2
+; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX8-NEXT: s_cbranch_execnz .LBB10_1
+; GFX8-NEXT: ; %bb.2: ; %exit
+; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX8-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: fma_in_loop_bf16:
+; GFX9: ; %bb.0: ; %entry
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX9-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX9-NEXT: s_movk_i32 s6, 0x7fff
+; GFX9-NEXT: v_add3_u32 v1, v1, v0, s6
+; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v0
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v1, v3, vcc
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX9-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NEXT: s_mov_b32 s7, 0
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: .LBB10_1: ; %loop
+; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX9-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX9-NEXT: v_add3_u32 v3, v3, v0, s6
+; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v0
+; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX9-NEXT: s_add_i32 s7, s7, 1
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v3, v4, vcc
+; GFX9-NEXT: v_cmp_ge_i32_e32 vcc, s7, v2
+; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX9-NEXT: s_cbranch_execnz .LBB10_1
+; GFX9-NEXT: ; %bb.2: ; %exit
+; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: fma_in_loop_bf16:
+; GFX10: ; %bb.0: ; %entry
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX10-NEXT: s_mov_b32 s4, 0
+; GFX10-NEXT: s_mov_b32 s5, 0
+; GFX10-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX10-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX10-NEXT: v_or_b32_e32 v3, 0x400000, v0
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX10-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v1, v3, vcc_lo
+; GFX10-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX10-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX10-NEXT: .p2align 6
+; GFX10-NEXT: .LBB10_1: ; %loop
+; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX10-NEXT: s_add_i32 s5, s5, 1
+; GFX10-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX10-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX10-NEXT: v_or_b32_e32 v4, 0x400000, v0
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX10-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v3, v4, vcc_lo
+; GFX10-NEXT: v_cmp_ge_i32_e32 vcc_lo, s5, v2
+; GFX10-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: s_cbranch_execnz .LBB10_1
+; GFX10-NEXT: ; %bb.2: ; %exit
+; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+entry:
+ %mul = fmul contract bfloat %a, %b
+ br label %loop
+
+loop:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+ %acc = phi bfloat [ 0.000000e+00, %entry ], [ %add, %loop ]
+ %add = fadd contract bfloat %acc, %mul
+ %i.next = add i32 %i, 1
+ %cmp = icmp slt i32 %i.next, %n
+ br i1 %cmp, label %loop, label %exit
+
+exit:
+ ret bfloat %add
+}
+
+attributes #0 = { denormal_fpenv(preservesign) }
diff --git a/llvm/test/Transforms/CodeGenPrepare/AMDGPU/sink-fmul-fadd-contract-fast.ll b/llvm/test/Transforms/CodeGenPrepare/AMDGPU/sink-fmul-fadd-contract-fast.ll
new file mode 100644
index 0000000000000..a8a4d12a502e3
--- /dev/null
+++ b/llvm/test/Transforms/CodeGenPrepare/AMDGPU/sink-fmul-fadd-contract-fast.ll
@@ -0,0 +1,57 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -S -passes='require<profile-summary>,function(codegenprepare)' -mtriple=amdgpu9.00-amd-amdhsa -fp-contract=fast < %s | FileCheck %s
+
+; -fp-contract=fast fuses without the per-instruction contract flags.
+
+define float @sink_fmul_fadd_f32(i1 %cond, float %a, float %b, float %c) {
+; CHECK-LABEL: define float @sink_fmul_fadd_f32(
+; CHECK-SAME: i1 [[COND:%.*]], float [[A:%.*]], float [[B:%.*]], float [[C:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = fmul float [[A]], [[B]]
+; CHECK-NEXT: br i1 [[COND]], label %[[IF:.*]], label %[[EXIT:.*]]
+; CHECK: [[IF]]:
+; CHECK-NEXT: [[ADD:%.*]] = fadd float [[TMP0]], [[C]]
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[R:%.*]] = phi float [ [[ADD]], %[[IF]] ], [ 0.000000e+00, %[[ENTRY]] ]
+; CHECK-NEXT: ret float [[R]]
+;
+entry:
+ %mul = fmul float %a, %b
+ br i1 %cond, label %if, label %exit
+
+if:
+ %add = fadd float %mul, %c
+ br label %exit
+
+exit:
+ %r = phi float [ %add, %if ], [ 0.000000e+00, %entry ]
+ ret float %r
+}
+
+; A multi-use fmul is still left alone.
+define float @no_sink_fmul_multi_use(i1 %cond, float %a, float %b, float %c) {
+; CHECK-LABEL: define float @no_sink_fmul_multi_use(
+; CHECK-SAME: i1 [[COND:%.*]], float [[A:%.*]], float [[B:%.*]], float [[C:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[MUL:%.*]] = fmul float [[A]], [[B]]
+; CHECK-NEXT: br i1 [[COND]], label %[[IF:.*]], label %[[EXIT:.*]]
+; CHECK: [[IF]]:
+; CHECK-NEXT: [[ADD:%.*]] = fadd float [[MUL]], [[C]]
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[R:%.*]] = phi float [ [[ADD]], %[[IF]] ], [ [[MUL]], %[[ENTRY]] ]
+; CHECK-NEXT: ret float [[R]]
+;
+entry:
+ %mul = fmul float %a, %b
+ br i1 %cond, label %if, label %exit
+
+if:
+ %add = fadd float %mul, %c
+ br label %exit
+
+exit:
+ %r = phi float [ %add, %if ], [ %mul, %entry ]
+ ret float %r
+}
diff --git a/llvm/test/Transforms/CodeGenPrepare/AMDGPU/sink-fmul-fadd.ll b/llvm/test/Transforms/CodeGenPrepare/AMDGPU/sink-fmul-fadd.ll
new file mode 100644
index 0000000000000..b548b99663f9e
--- /dev/null
+++ b/llvm/test/Transforms/CodeGenPrepare/AMDGPU/sink-fmul-fadd.ll
@@ -0,0 +1,571 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -S -passes='require<profile-summary>,function(codegenprepare)' -mtriple=amdgpu8.03-amd-amdhsa < %s | FileCheck -check-prefixes=CHECK,GFX8 %s
+; RUN: opt -S -passes='require<profile-summary>,function(codegenprepare)' -mtriple=amdgpu9.00-amd-amdhsa < %s | FileCheck -check-prefixes=CHECK,GFX9 %s
+; RUN: opt -S -passes='require<profile-summary>,function(codegenprepare)' -mtriple=amdgpu10.30-amd-amdhsa < %s | FileCheck -check-prefixes=CHECK,GFX10 %s
+
+
+define float @sink_fmul_fadd_f32(i1 %cond, float %a, float %b, float %c) {
+; CHECK-LABEL: define float @sink_fmul_fadd_f32(
+; CHECK-SAME: i1 [[COND:%.*]], float [[A:%.*]], float [[B:%.*]], float [[C:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[MUL:%.*]] = fmul contract float [[A]], [[B]]
+; CHECK-NEXT: br i1 [[COND]], label %[[IF:.*]], label %[[EXIT:.*]]
+; CHECK: [[IF]]:
+; CHECK-NEXT: [[ADD:%.*]] = fadd contract float [[MUL]], [[C]]
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[R:%.*]] = phi float [ [[ADD]], %[[IF]] ], [ 0.000000e+00, %[[ENTRY]] ]
+; CHECK-NEXT: ret float [[R]]
+;
+entry:
+ %mul = fmul contract float %a, %b
+ br i1 %cond, label %if, label %exit
+
+if:
+ %add = fadd contract float %mul, %c
+ br label %exit
+
+exit:
+ %r = phi float [ %add, %if ], [ 0.000000e+00, %entry ]
+ ret float %r
+}
+
+define float @sink_fmul_fsub_f32(i1 %cond, float %a, float %b, float %c) {
+; CHECK-LABEL: define float @sink_fmul_fsub_f32(
+; CHECK-SAME: i1 [[COND:%.*]], float [[A:%.*]], float [[B:%.*]], float [[C:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[MUL:%.*]] = fmul contract float [[A]], [[B]]
+; CHECK-NEXT: br i1 [[COND]], label %[[IF:.*]], label %[[EXIT:.*]]
+; CHECK: [[IF]]:
+; CHECK-NEXT: [[SUB:%.*]] = fsub contract float [[MUL]], [[C]]
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[R:%.*]] = phi float [ [[SUB]], %[[IF]] ], [ 0.000000e+00, %[[ENTRY]] ]
+; CHECK-NEXT: ret float [[R]]
+;
+entry:
+ %mul = fmul contract float %a, %b
+ br i1 %cond, label %if, label %exit
+
+if:
+ %sub = fsub contract float %mul, %c
+ br label %exit
+
+exit:
+ %r = phi float [ %sub, %if ], [ 0.000000e+00, %entry ]
+ ret float %r
+}
+
+; The fmul may be the second operand of the fsub as well.
+define float @sink_fmul_fsub_rev_f32(i1 %cond, float %a, float %b, float %c) {
+; CHECK-LABEL: define float @sink_fmul_fsub_rev_f32(
+; CHECK-SAME: i1 [[COND:%.*]], float [[A:%.*]], float [[B:%.*]], float [[C:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[MUL:%.*]] = fmul contract float [[A]], [[B]]
+; CHECK-NEXT: br i1 [[COND]], label %[[IF:.*]], label %[[EXIT:.*]]
+; CHECK: [[IF]]:
+; CHECK-NEXT: [[SUB:%.*]] = fsub contract float [[C]], [[MUL]]
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[R:%.*]] = phi float [ [[SUB]], %[[IF]] ], [ 0.000000e+00, %[[ENTRY]] ]
+; CHECK-NEXT: ret float [[R]]
+;
+entry:
+ %mul = fmul contract float %a, %b
+ br i1 %cond, label %if, label %exit
+
+if:
+ %sub = fsub contract float %c, %mul
+ br label %exit
+
+exit:
+ %r = phi float [ %sub, %if ], [ 0.000000e+00, %entry ]
+ ret float %r
+}
+
+define half @sink_fmul_fadd_f16(i1 %cond, half %a, half %b, half %c) {
+; CHECK-LABEL: define half @sink_fmul_fadd_f16(
+; CHECK-SAME: i1 [[COND:%.*]], half [[A:%.*]], half [[B:%.*]], half [[C:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = fmul contract half [[A]], [[B]]
+; CHECK-NEXT: br i1 [[COND]], label %[[IF:.*]], label %[[EXIT:.*]]
+; CHECK: [[IF]]:
+; CHECK-NEXT: [[ADD:%.*]] = fadd contract half [[TMP0]], [[C]]
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[R:%.*]] = phi half [ [[ADD]], %[[IF]] ], [ 0.000000e+00, %[[ENTRY]] ]
+; CHECK-NEXT: ret half [[R]]
+;
+entry:
+ %mul = fmul contract half %a, %b
+ br i1 %cond, label %if, label %exit
+
+if:
+ %add = fadd contract half %mul, %c
+ br label %exit
+
+exit:
+ %r = phi half [ %add, %if ], [ 0.000000e+00, %entry ]
+ ret half %r
+}
+
+define double @sink_fmul_fadd_f64(i1 %cond, double %a, double %b, double %c) {
+; CHECK-LABEL: define double @sink_fmul_fadd_f64(
+; CHECK-SAME: i1 [[COND:%.*]], double [[A:%.*]], double [[B:%.*]], double [[C:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = fmul contract double [[A]], [[B]]
+; CHECK-NEXT: br i1 [[COND]], label %[[IF:.*]], label %[[EXIT:.*]]
+; CHECK: [[IF]]:
+; CHECK-NEXT: [[ADD:%.*]] = fadd contract double [[TMP0]], [[C]]
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[R:%.*]] = phi double [ [[ADD]], %[[IF]] ], [ 0.000000e+00, %[[ENTRY]] ]
+; CHECK-NEXT: ret double [[R]]
+;
+entry:
+ %mul = fmul contract double %a, %b
+ br i1 %cond, label %if, label %exit
+
+if:
+ %add = fadd contract double %mul, %c
+ br label %exit
+
+exit:
+ %r = phi double [ %add, %if ], [ 0.000000e+00, %entry ]
+ ret double %r
+}
+
+define <2 x float> @sink_fmul_fadd_v2f32(i1 %cond, <2 x float> %a, <2 x float> %b, <2 x float> %c) {
+; CHECK-LABEL: define <2 x float> @sink_fmul_fadd_v2f32(
+; CHECK-SAME: i1 [[COND:%.*]], <2 x float> [[A:%.*]], <2 x float> [[B:%.*]], <2 x float> [[C:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[MUL:%.*]] = fmul contract <2 x float> [[A]], [[B]]
+; CHECK-NEXT: br i1 [[COND]], label %[[IF:.*]], label %[[EXIT:.*]]
+; CHECK: [[IF]]:
+; CHECK-NEXT: [[ADD:%.*]] = fadd contract <2 x float> [[MUL]], [[C]]
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[R:%.*]] = phi <2 x float> [ [[ADD]], %[[IF]] ], [ zeroinitializer, %[[ENTRY]] ]
+; CHECK-NEXT: ret <2 x float> [[R]]
+;
+entry:
+ %mul = fmul contract <2 x float> %a, %b
+ br i1 %cond, label %if, label %exit
+
+if:
+ %add = fadd contract <2 x float> %mul, %c
+ br label %exit
+
+exit:
+ %r = phi <2 x float> [ %add, %if ], [ zeroinitializer, %entry ]
+ ret <2 x float> %r
+}
+
+; Only the operand DAGCombiner picks is a candidate.
+define float @sink_only_fusable_fmul(i1 %cond, float %a, float %b, float %c, float %d) {
+; CHECK-LABEL: define float @sink_only_fusable_fmul(
+; CHECK-SAME: i1 [[COND:%.*]], float [[A:%.*]], float [[B:%.*]], float [[C:%.*]], float [[D:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[MUL0:%.*]] = fmul contract float [[A]], [[B]]
+; CHECK-NEXT: [[MUL1:%.*]] = fmul contract float [[C]], [[D]]
+; CHECK-NEXT: br i1 [[COND]], label %[[IF:.*]], label %[[EXIT:.*]]
+; CHECK: [[IF]]:
+; CHECK-NEXT: [[ADD:%.*]] = fadd contract float [[MUL0]], [[MUL1]]
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[R:%.*]] = phi float [ [[ADD]], %[[IF]] ], [ 0.000000e+00, %[[ENTRY]] ]
+; CHECK-NEXT: ret float [[R]]
+;
+entry:
+ %mul0 = fmul contract float %a, %b
+ %mul1 = fmul contract float %c, %d
+ br i1 %cond, label %if, label %exit
+
+if:
+ %add = fadd contract float %mul0, %mul1
+ br label %exit
+
+exit:
+ %r = phi float [ %add, %if ], [ 0.000000e+00, %entry ]
+ ret float %r
+}
+
+; The first operand is already local, so it is the one that fuses.
+define float @no_sink_second_fmul(i1 %cond, float %a, float %b, float %c, float %d) {
+; CHECK-LABEL: define float @no_sink_second_fmul(
+; CHECK-SAME: i1 [[COND:%.*]], float [[A:%.*]], float [[B:%.*]], float [[C:%.*]], float [[D:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[MUL1:%.*]] = fmul contract float [[C]], [[D]]
+; CHECK-NEXT: br i1 [[COND]], label %[[IF:.*]], label %[[EXIT:.*]]
+; CHECK: [[IF]]:
+; CHECK-NEXT: [[MUL0:%.*]] = fmul contract float [[A]], [[B]]
+; CHECK-NEXT: [[ADD:%.*]] = fadd contract float [[MUL0]], [[MUL1]]
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[R:%.*]] = phi float [ [[ADD]], %[[IF]] ], [ 0.000000e+00, %[[ENTRY]] ]
+; CHECK-NEXT: ret float [[R]]
+;
+entry:
+ %mul1 = fmul contract float %c, %d
+ br i1 %cond, label %if, label %exit
+
+if:
+ %mul0 = fmul contract float %a, %b
+ %add = fadd contract float %mul0, %mul1
+ br label %exit
+
+exit:
+ %r = phi float [ %add, %if ], [ 0.000000e+00, %entry ]
+ ret float %r
+}
+
+; v_mad_f32 is exact, so with denormals flushed no fast-math flags are needed.
+define float @sink_fmul_fadd_f32_no_denormals(i1 %cond, float %a, float %b, float %c) #0 {
+; CHECK-LABEL: define float @sink_fmul_fadd_f32_no_denormals(
+; CHECK-SAME: i1 [[COND:%.*]], float [[A:%.*]], float [[B:%.*]], float [[C:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[MUL:%.*]] = fmul float [[A]], [[B]]
+; CHECK-NEXT: br i1 [[COND]], label %[[IF:.*]], label %[[EXIT:.*]]
+; CHECK: [[IF]]:
+; CHECK-NEXT: [[ADD:%.*]] = fadd float [[MUL]], [[C]]
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[R:%.*]] = phi float [ [[ADD]], %[[IF]] ], [ 0.000000e+00, %[[ENTRY]] ]
+; CHECK-NEXT: ret float [[R]]
+;
+entry:
+ %mul = fmul float %a, %b
+ br i1 %cond, label %if, label %exit
+
+if:
+ %add = fadd float %mul, %c
+ br label %exit
+
+exit:
+ %r = phi float [ %add, %if ], [ 0.000000e+00, %entry ]
+ ret float %r
+}
+
+; Same for v_mad_f16.
+define half @sink_fmul_fadd_f16_no_denormals(i1 %cond, half %a, half %b, half %c) #0 {
+; CHECK-LABEL: define half @sink_fmul_fadd_f16_no_denormals(
+; CHECK-SAME: i1 [[COND:%.*]], half [[A:%.*]], half [[B:%.*]], half [[C:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[MUL:%.*]] = fmul half [[A]], [[B]]
+; CHECK-NEXT: br i1 [[COND]], label %[[IF:.*]], label %[[EXIT:.*]]
+; CHECK: [[IF]]:
+; CHECK-NEXT: [[ADD:%.*]] = fadd half [[MUL]], [[C]]
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[R:%.*]] = phi half [ [[ADD]], %[[IF]] ], [ 0.000000e+00, %[[ENTRY]] ]
+; CHECK-NEXT: ret half [[R]]
+;
+entry:
+ %mul = fmul half %a, %b
+ br i1 %cond, label %if, label %exit
+
+if:
+ %add = fadd half %mul, %c
+ br label %exit
+
+exit:
+ %r = phi half [ %add, %if ], [ 0.000000e+00, %entry ]
+ ret half %r
+}
+
+; With denormals enabled and no contract flags nothing fuses.
+define float @no_sink_fmul_fadd_f32_denormals(i1 %cond, float %a, float %b, float %c) {
+; CHECK-LABEL: define float @no_sink_fmul_fadd_f32_denormals(
+; CHECK-SAME: i1 [[COND:%.*]], float [[A:%.*]], float [[B:%.*]], float [[C:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[MUL:%.*]] = fmul float [[A]], [[B]]
+; CHECK-NEXT: br i1 [[COND]], label %[[IF:.*]], label %[[EXIT:.*]]
+; CHECK: [[IF]]:
+; CHECK-NEXT: [[ADD:%.*]] = fadd float [[MUL]], [[C]]
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[R:%.*]] = phi float [ [[ADD]], %[[IF]] ], [ 0.000000e+00, %[[ENTRY]] ]
+; CHECK-NEXT: ret float [[R]]
+;
+entry:
+ %mul = fmul float %a, %b
+ br i1 %cond, label %if, label %exit
+
+if:
+ %add = fadd float %mul, %c
+ br label %exit
+
+exit:
+ %r = phi float [ %add, %if ], [ 0.000000e+00, %entry ]
+ ret float %r
+}
+
+; Contract on the fmul alone is not enough.
+define float @no_sink_fmul_not_contract_fadd(i1 %cond, float %a, float %b, float %c) {
+; CHECK-LABEL: define float @no_sink_fmul_not_contract_fadd(
+; CHECK-SAME: i1 [[COND:%.*]], float [[A:%.*]], float [[B:%.*]], float [[C:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[MUL:%.*]] = fmul contract float [[A]], [[B]]
+; CHECK-NEXT: br i1 [[COND]], label %[[IF:.*]], label %[[EXIT:.*]]
+; CHECK: [[IF]]:
+; CHECK-NEXT: [[ADD:%.*]] = fadd float [[MUL]], [[C]]
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[R:%.*]] = phi float [ [[ADD]], %[[IF]] ], [ 0.000000e+00, %[[ENTRY]] ]
+; CHECK-NEXT: ret float [[R]]
+;
+entry:
+ %mul = fmul contract float %a, %b
+ br i1 %cond, label %if, label %exit
+
+if:
+ %add = fadd float %mul, %c
+ br label %exit
+
+exit:
+ %r = phi float [ %add, %if ], [ 0.000000e+00, %entry ]
+ ret float %r
+}
+
+; Contract on the fadd alone is not enough.
+define float @no_sink_fadd_not_contract_fmul(i1 %cond, float %a, float %b, float %c) {
+; CHECK-LABEL: define float @no_sink_fadd_not_contract_fmul(
+; CHECK-SAME: i1 [[COND:%.*]], float [[A:%.*]], float [[B:%.*]], float [[C:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[MUL:%.*]] = fmul float [[A]], [[B]]
+; CHECK-NEXT: br i1 [[COND]], label %[[IF:.*]], label %[[EXIT:.*]]
+; CHECK: [[IF]]:
+; CHECK-NEXT: [[ADD:%.*]] = fadd contract float [[MUL]], [[C]]
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[R:%.*]] = phi float [ [[ADD]], %[[IF]] ], [ 0.000000e+00, %[[ENTRY]] ]
+; CHECK-NEXT: ret float [[R]]
+;
+entry:
+ %mul = fmul float %a, %b
+ br i1 %cond, label %if, label %exit
+
+if:
+ %add = fadd contract float %mul, %c
+ br label %exit
+
+exit:
+ %r = phi float [ %add, %if ], [ 0.000000e+00, %entry ]
+ ret float %r
+}
+
+; The fmul is live out of the branch, so it would be duplicated.
+define float @no_sink_fmul_multi_use(i1 %cond, float %a, float %b, float %c) {
+; CHECK-LABEL: define float @no_sink_fmul_multi_use(
+; CHECK-SAME: i1 [[COND:%.*]], float [[A:%.*]], float [[B:%.*]], float [[C:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[MUL:%.*]] = fmul contract float [[A]], [[B]]
+; CHECK-NEXT: br i1 [[COND]], label %[[IF:.*]], label %[[EXIT:.*]]
+; CHECK: [[IF]]:
+; CHECK-NEXT: [[ADD:%.*]] = fadd contract float [[MUL]], [[C]]
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[R:%.*]] = phi float [ [[ADD]], %[[IF]] ], [ [[MUL]], %[[ENTRY]] ]
+; CHECK-NEXT: ret float [[R]]
+;
+entry:
+ %mul = fmul contract float %a, %b
+ br i1 %cond, label %if, label %exit
+
+if:
+ %add = fadd contract float %mul, %c
+ br label %exit
+
+exit:
+ %r = phi float [ %add, %if ], [ %mul, %entry ]
+ ret float %r
+}
+
+; Nothing to do when the fmul is already in the same block as the fadd.
+define float @no_sink_same_block(float %a, float %b, float %c) {
+; CHECK-LABEL: define float @no_sink_same_block(
+; CHECK-SAME: float [[A:%.*]], float [[B:%.*]], float [[C:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[MUL:%.*]] = fmul contract float [[A]], [[B]]
+; CHECK-NEXT: [[ADD:%.*]] = fadd contract float [[MUL]], [[C]]
+; CHECK-NEXT: ret float [[ADD]]
+;
+entry:
+ %mul = fmul contract float %a, %b
+ %add = fadd contract float %mul, %c
+ ret float %add
+}
+
+; The multiplied value is not an fmul.
+define float @no_sink_not_fmul(i1 %cond, float %a, float %b, float %c) {
+; CHECK-LABEL: define float @no_sink_not_fmul(
+; CHECK-SAME: i1 [[COND:%.*]], float [[A:%.*]], float [[B:%.*]], float [[C:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[DIV:%.*]] = fdiv contract float [[A]], [[B]]
+; CHECK-NEXT: br i1 [[COND]], label %[[IF:.*]], label %[[EXIT:.*]]
+; CHECK: [[IF]]:
+; CHECK-NEXT: [[ADD:%.*]] = fadd contract float [[DIV]], [[C]]
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[R:%.*]] = phi float [ [[ADD]], %[[IF]] ], [ 0.000000e+00, %[[ENTRY]] ]
+; CHECK-NEXT: ret float [[R]]
+;
+entry:
+ %div = fdiv contract float %a, %b
+ br i1 %cond, label %if, label %exit
+
+if:
+ %add = fadd contract float %div, %c
+ br label %exit
+
+exit:
+ %r = phi float [ %add, %if ], [ 0.000000e+00, %entry ]
+ ret float %r
+}
+
+; The motivating shape, a loop-invariant fmul hoisted by LICM.
+define float @sink_fmul_into_loop(float %a, float %b, i32 %n) {
+; CHECK-LABEL: define float @sink_fmul_into_loop(
+; CHECK-SAME: float [[A:%.*]], float [[B:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[MUL:%.*]] = fmul contract float [[A]], [[B]]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[ACC:%.*]] = phi float [ 0.000000e+00, %[[ENTRY]] ], [ [[ADD:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[ADD]] = fadd contract float [[ACC]], [[MUL]]
+; CHECK-NEXT: [[I_NEXT]] = add i32 [[I]], 1
+; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[CMP]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret float [[ADD]]
+;
+entry:
+ %mul = fmul contract float %a, %b
+ br label %loop
+
+loop:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+ %acc = phi float [ 0.000000e+00, %entry ], [ %add, %loop ]
+ %add = fadd contract float %acc, %mul
+ %i.next = add i32 %i, 1
+ %cmp = icmp slt i32 %i.next, %n
+ br i1 %cmp, label %loop, label %exit
+
+exit:
+ ret float %add
+}
+
+; A legal v2f16 has no packed mad, and v_pk_fma_f16 needs denormals enabled.
+define <2 x half> @sink_fmul_fadd_v2f16_no_denormals(i1 %cond, <2 x half> %a, <2 x half> %b, <2 x half> %c) #0 {
+; CHECK-LABEL: define <2 x half> @sink_fmul_fadd_v2f16_no_denormals(
+; CHECK-SAME: i1 [[COND:%.*]], <2 x half> [[A:%.*]], <2 x half> [[B:%.*]], <2 x half> [[C:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[MUL:%.*]] = fmul contract <2 x half> [[A]], [[B]]
+; CHECK-NEXT: br i1 [[COND]], label %[[IF:.*]], label %[[EXIT:.*]]
+; CHECK: [[IF]]:
+; CHECK-NEXT: [[ADD:%.*]] = fadd contract <2 x half> [[MUL]], [[C]]
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[R:%.*]] = phi <2 x half> [ [[ADD]], %[[IF]] ], [ zeroinitializer, %[[ENTRY]] ]
+; CHECK-NEXT: ret <2 x half> [[R]]
+;
+entry:
+ %mul = fmul contract <2 x half> %a, %b
+ br i1 %cond, label %if, label %exit
+
+if:
+ %add = fadd contract <2 x half> %mul, %c
+ br label %exit
+
+exit:
+ %r = phi <2 x half> [ %add, %if ], [ zeroinitializer, %entry ]
+ ret <2 x half> %r
+}
+
+; Same without the contract flags.
+define <2 x half> @sink_fmul_fadd_v2f16_no_denormals_no_contract(i1 %cond, <2 x half> %a, <2 x half> %b, <2 x half> %c) #0 {
+; CHECK-LABEL: define <2 x half> @sink_fmul_fadd_v2f16_no_denormals_no_contract(
+; CHECK-SAME: i1 [[COND:%.*]], <2 x half> [[A:%.*]], <2 x half> [[B:%.*]], <2 x half> [[C:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[MUL:%.*]] = fmul <2 x half> [[A]], [[B]]
+; CHECK-NEXT: br i1 [[COND]], label %[[IF:.*]], label %[[EXIT:.*]]
+; CHECK: [[IF]]:
+; CHECK-NEXT: [[ADD:%.*]] = fadd <2 x half> [[MUL]], [[C]]
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[R:%.*]] = phi <2 x half> [ [[ADD]], %[[IF]] ], [ zeroinitializer, %[[ENTRY]] ]
+; CHECK-NEXT: ret <2 x half> [[R]]
+;
+entry:
+ %mul = fmul <2 x half> %a, %b
+ br i1 %cond, label %if, label %exit
+
+if:
+ %add = fadd <2 x half> %mul, %c
+ br label %exit
+
+exit:
+ %r = phi <2 x half> [ %add, %if ], [ zeroinitializer, %entry ]
+ ret <2 x half> %r
+}
+
+; With denormals enabled the pair really is selected as v_pk_fma_f16.
+define <2 x half> @sink_fmul_fadd_v2f16_denormals(i1 %cond, <2 x half> %a, <2 x half> %b, <2 x half> %c) {
+; CHECK-LABEL: define <2 x half> @sink_fmul_fadd_v2f16_denormals(
+; CHECK-SAME: i1 [[COND:%.*]], <2 x half> [[A:%.*]], <2 x half> [[B:%.*]], <2 x half> [[C:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = fmul contract <2 x half> [[A]], [[B]]
+; CHECK-NEXT: br i1 [[COND]], label %[[IF:.*]], label %[[EXIT:.*]]
+; CHECK: [[IF]]:
+; CHECK-NEXT: [[ADD:%.*]] = fadd contract <2 x half> [[TMP0]], [[C]]
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[R:%.*]] = phi <2 x half> [ [[ADD]], %[[IF]] ], [ zeroinitializer, %[[ENTRY]] ]
+; CHECK-NEXT: ret <2 x half> [[R]]
+;
+entry:
+ %mul = fmul contract <2 x half> %a, %b
+ br i1 %cond, label %if, label %exit
+
+if:
+ %add = fadd contract <2 x half> %mul, %c
+ br label %exit
+
+exit:
+ %r = phi <2 x half> [ %add, %if ], [ zeroinitializer, %entry ]
+ ret <2 x half> %r
+}
+
+; bf16 is promoted to f32, and the fused form drops the intermediate rounding.
+define bfloat @sink_fmul_fadd_bf16(i1 %cond, bfloat %a, bfloat %b, bfloat %c) {
+; CHECK-LABEL: define bfloat @sink_fmul_fadd_bf16(
+; CHECK-SAME: i1 [[COND:%.*]], bfloat [[A:%.*]], bfloat [[B:%.*]], bfloat [[C:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = fmul contract bfloat [[A]], [[B]]
+; CHECK-NEXT: br i1 [[COND]], label %[[IF:.*]], label %[[EXIT:.*]]
+; CHECK: [[IF]]:
+; CHECK-NEXT: [[ADD:%.*]] = fadd contract bfloat [[TMP0]], [[C]]
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[R:%.*]] = phi bfloat [ [[ADD]], %[[IF]] ], [ 0.000000e+00, %[[ENTRY]] ]
+; CHECK-NEXT: ret bfloat [[R]]
+;
+entry:
+ %mul = fmul contract bfloat %a, %b
+ br i1 %cond, label %if, label %exit
+
+if:
+ %add = fadd contract bfloat %mul, %c
+ br label %exit
+
+exit:
+ %r = phi bfloat [ %add, %if ], [ 0.000000e+00, %entry ]
+ ret bfloat %r
+}
+
+attributes #0 = { denormal_fpenv(preservesign) }
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; GFX10: {{.*}}
+; GFX8: {{.*}}
+; GFX9: {{.*}}
More information about the llvm-commits
mailing list