[llvm] 1fb1fcf - Add mad support for v_pk_* 16 bit integer (#95104)
via llvm-commits
llvm-commits at lists.llvm.org
Thu Jun 13 08:46:44 PDT 2024
Author: David Stuttard
Date: 2024-06-13T16:46:40+01:00
New Revision: 1fb1fcf34396808e5e88d9d6a3fe6b207188f671
URL: https://github.com/llvm/llvm-project/commit/1fb1fcf34396808e5e88d9d6a3fe6b207188f671
DIFF: https://github.com/llvm/llvm-project/commit/1fb1fcf34396808e5e88d9d6a3fe6b207188f671.diff
LOG: Add mad support for v_pk_* 16 bit integer (#95104)
Added:
Modified:
llvm/lib/Target/AMDGPU/VOP3PInstructions.td
llvm/test/CodeGen/AMDGPU/integer-mad-patterns.ll
llvm/test/CodeGen/AMDGPU/reassoc-mul-add-1-to-mad.ll
Removed:
################################################################################
diff --git a/llvm/lib/Target/AMDGPU/VOP3PInstructions.td b/llvm/lib/Target/AMDGPU/VOP3PInstructions.td
index 2ede7ec29dddf..4cab154351998 100644
--- a/llvm/lib/Target/AMDGPU/VOP3PInstructions.td
+++ b/llvm/lib/Target/AMDGPU/VOP3PInstructions.td
@@ -90,7 +90,7 @@ multiclass VOP3_VOP3PInst<string OpName, VOP3P_Mix_Profile P> {
let isReMaterializable = 1 in {
let isCommutable = 1 in {
defm V_PK_MAD_I16 : VOP3PInst<"v_pk_mad_i16", VOP3P_Profile<VOP_V2I16_V2I16_V2I16_V2I16>>;
-defm V_PK_MAD_U16 : VOP3PInst<"v_pk_mad_u16", VOP3P_Profile<VOP_V2I16_V2I16_V2I16_V2I16>>;
+defm V_PK_MAD_U16 : VOP3PInst<"v_pk_mad_u16", VOP3P_Profile<VOP_V2I16_V2I16_V2I16_V2I16>, imad>;
let FPDPRounding = 1 in {
defm V_PK_FMA_F16 : VOP3PInst<"v_pk_fma_f16", VOP3P_Profile<VOP_V2F16_V2F16_V2F16_V2F16>, any_fma>;
diff --git a/llvm/test/CodeGen/AMDGPU/integer-mad-patterns.ll b/llvm/test/CodeGen/AMDGPU/integer-mad-patterns.ll
index 526ee5a51745d..3dea5cb5c1423 100644
--- a/llvm/test/CodeGen/AMDGPU/integer-mad-patterns.ll
+++ b/llvm/test/CodeGen/AMDGPU/integer-mad-patterns.ll
@@ -477,42 +477,38 @@ define <2 x i16> @clpeak_imad_pat_v2i16(<2 x i16> %x, <2 x i16> %y) {
; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_pk_add_u16 v0, v0, 1 op_sel_hi:[1,0]
-; GFX9-NEXT: v_pk_mul_lo_u16 v2, v0, v1
-; GFX9-NEXT: v_pk_add_u16 v0, v2, v0
+; GFX9-NEXT: v_pk_mad_u16 v2, v0, v1, v0
+; GFX9-NEXT: v_pk_mul_lo_u16 v3, v2, v1
+; GFX9-NEXT: v_pk_mad_u16 v0, v0, v1, 1 op_sel_hi:[1,1,0]
+; GFX9-NEXT: v_pk_mad_u16 v1, v2, v1, 1 op_sel_hi:[1,1,0]
+; GFX9-NEXT: v_pk_mul_lo_u16 v0, v3, v0
; GFX9-NEXT: v_pk_mul_lo_u16 v0, v0, v1
-; GFX9-NEXT: v_pk_add_u16 v1, v2, 1 op_sel_hi:[1,0]
-; GFX9-NEXT: v_pk_add_u16 v2, v0, 1 op_sel_hi:[1,0]
-; GFX9-NEXT: v_pk_mul_lo_u16 v0, v0, v1
-; GFX9-NEXT: v_pk_mul_lo_u16 v0, v0, v2
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: clpeak_imad_pat_v2i16:
; GFX10: ; %bb.0: ; %entry
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_pk_add_u16 v0, v0, 1 op_sel_hi:[1,0]
-; GFX10-NEXT: v_pk_mul_lo_u16 v2, v0, v1
-; GFX10-NEXT: v_pk_add_u16 v0, v2, v0
-; GFX10-NEXT: v_pk_mul_lo_u16 v0, v0, v1
-; GFX10-NEXT: v_pk_add_u16 v1, v2, 1 op_sel_hi:[1,0]
-; GFX10-NEXT: v_pk_add_u16 v2, v0, 1 op_sel_hi:[1,0]
+; GFX10-NEXT: v_pk_mad_u16 v2, v0, v1, v0
+; GFX10-NEXT: v_pk_mad_u16 v0, v0, v1, 1 op_sel_hi:[1,1,0]
+; GFX10-NEXT: v_pk_mul_lo_u16 v3, v2, v1
+; GFX10-NEXT: v_pk_mad_u16 v1, v2, v1, 1 op_sel_hi:[1,1,0]
+; GFX10-NEXT: v_pk_mul_lo_u16 v0, v3, v0
; GFX10-NEXT: v_pk_mul_lo_u16 v0, v0, v1
-; GFX10-NEXT: v_pk_mul_lo_u16 v0, v0, v2
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: clpeak_imad_pat_v2i16:
; GFX11: ; %bb.0: ; %entry
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_pk_add_u16 v0, v0, 1 op_sel_hi:[1,0]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_pk_mul_lo_u16 v2, v0, v1
-; GFX11-NEXT: v_pk_add_u16 v0, v2, v0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_pk_mul_lo_u16 v0, v0, v1
-; GFX11-NEXT: v_pk_add_u16 v1, v2, 1 op_sel_hi:[1,0]
-; GFX11-NEXT: v_pk_add_u16 v2, v0, 1 op_sel_hi:[1,0]
+; GFX11-NEXT: v_pk_mad_u16 v2, v0, v1, v0
+; GFX11-NEXT: v_pk_mad_u16 v0, v0, v1, 1 op_sel_hi:[1,1,0]
+; GFX11-NEXT: v_pk_mul_lo_u16 v3, v2, v1
+; GFX11-NEXT: v_pk_mad_u16 v1, v2, v1, 1 op_sel_hi:[1,1,0]
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_pk_mul_lo_u16 v0, v3, v0
; GFX11-NEXT: v_pk_mul_lo_u16 v0, v0, v1
-; GFX11-NEXT: v_pk_mul_lo_u16 v0, v0, v2
; GFX11-NEXT: s_setpc_b64 s[30:31]
entry:
%y18 = add <2 x i16> %x, <i16 1, i16 1>
@@ -691,22 +687,20 @@ define <3 x i16> @clpeak_imad_pat_v3i16(<3 x i16> %x, <3 x i16> %y) {
; GFX9-SDAG-LABEL: clpeak_imad_pat_v3i16:
; GFX9-SDAG: ; %bb.0: ; %entry
; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-SDAG-NEXT: v_pk_add_u16 v1, v1, 1
; GFX9-SDAG-NEXT: v_pk_add_u16 v0, v0, 1 op_sel_hi:[1,0]
-; GFX9-SDAG-NEXT: v_pk_mul_lo_u16 v4, v0, v2
-; GFX9-SDAG-NEXT: v_pk_mul_lo_u16 v5, v1, v3
-; GFX9-SDAG-NEXT: v_pk_add_u16 v1, v5, v1
-; GFX9-SDAG-NEXT: v_pk_add_u16 v0, v4, v0
+; GFX9-SDAG-NEXT: v_pk_add_u16 v1, v1, 1
+; GFX9-SDAG-NEXT: v_pk_mad_u16 v4, v1, v3, v1
+; GFX9-SDAG-NEXT: v_pk_mad_u16 v5, v0, v2, v0
+; GFX9-SDAG-NEXT: v_pk_mul_lo_u16 v6, v5, v2
+; GFX9-SDAG-NEXT: v_pk_mul_lo_u16 v7, v4, v3
+; GFX9-SDAG-NEXT: v_pk_mad_u16 v0, v0, v2, 1 op_sel_hi:[1,1,0]
+; GFX9-SDAG-NEXT: v_pk_mad_u16 v1, v1, v3, 1
+; GFX9-SDAG-NEXT: v_pk_mad_u16 v3, v4, v3, 1
+; GFX9-SDAG-NEXT: v_pk_mad_u16 v2, v5, v2, 1 op_sel_hi:[1,1,0]
+; GFX9-SDAG-NEXT: v_pk_mul_lo_u16 v1, v7, v1
+; GFX9-SDAG-NEXT: v_pk_mul_lo_u16 v0, v6, v0
; GFX9-SDAG-NEXT: v_pk_mul_lo_u16 v0, v0, v2
; GFX9-SDAG-NEXT: v_pk_mul_lo_u16 v1, v1, v3
-; GFX9-SDAG-NEXT: v_pk_add_u16 v2, v4, 1 op_sel_hi:[1,0]
-; GFX9-SDAG-NEXT: v_pk_add_u16 v3, v5, 1
-; GFX9-SDAG-NEXT: v_pk_add_u16 v4, v1, 1
-; GFX9-SDAG-NEXT: v_pk_add_u16 v5, v0, 1 op_sel_hi:[1,0]
-; GFX9-SDAG-NEXT: v_pk_mul_lo_u16 v1, v1, v3
-; GFX9-SDAG-NEXT: v_pk_mul_lo_u16 v0, v0, v2
-; GFX9-SDAG-NEXT: v_pk_mul_lo_u16 v0, v0, v5
-; GFX9-SDAG-NEXT: v_pk_mul_lo_u16 v1, v1, v4
; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-GISEL-LABEL: clpeak_imad_pat_v3i16:
@@ -714,41 +708,37 @@ define <3 x i16> @clpeak_imad_pat_v3i16(<3 x i16> %x, <3 x i16> %y) {
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-GISEL-NEXT: v_pk_add_u16 v0, v0, 1 op_sel_hi:[1,0]
; GFX9-GISEL-NEXT: v_pk_add_u16 v1, v1, 1
-; GFX9-GISEL-NEXT: v_pk_mul_lo_u16 v4, v0, v2
-; GFX9-GISEL-NEXT: v_pk_mul_lo_u16 v5, v1, v3
-; GFX9-GISEL-NEXT: v_pk_add_u16 v0, v4, v0
-; GFX9-GISEL-NEXT: v_pk_add_u16 v1, v5, v1
+; GFX9-GISEL-NEXT: v_pk_mad_u16 v4, v0, v2, v0
+; GFX9-GISEL-NEXT: v_pk_mad_u16 v5, v1, v3, v1
+; GFX9-GISEL-NEXT: v_pk_mul_lo_u16 v6, v4, v2
+; GFX9-GISEL-NEXT: v_pk_mul_lo_u16 v7, v5, v3
+; GFX9-GISEL-NEXT: v_pk_mad_u16 v0, v0, v2, 1 op_sel_hi:[1,1,0]
+; GFX9-GISEL-NEXT: v_pk_mad_u16 v1, v1, v3, 1
+; GFX9-GISEL-NEXT: v_pk_mad_u16 v2, v4, v2, 1 op_sel_hi:[1,1,0]
+; GFX9-GISEL-NEXT: v_pk_mad_u16 v3, v5, v3, 1
+; GFX9-GISEL-NEXT: v_pk_mul_lo_u16 v0, v6, v0
+; GFX9-GISEL-NEXT: v_pk_mul_lo_u16 v1, v7, v1
; GFX9-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v2
; GFX9-GISEL-NEXT: v_pk_mul_lo_u16 v1, v1, v3
-; GFX9-GISEL-NEXT: v_pk_add_u16 v2, v4, 1 op_sel_hi:[1,0]
-; GFX9-GISEL-NEXT: v_pk_add_u16 v3, v5, 1
-; GFX9-GISEL-NEXT: v_pk_add_u16 v4, v0, 1 op_sel_hi:[1,0]
-; GFX9-GISEL-NEXT: v_pk_add_u16 v5, v1, 1
-; GFX9-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v2
-; GFX9-GISEL-NEXT: v_pk_mul_lo_u16 v1, v1, v3
-; GFX9-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v4
-; GFX9-GISEL-NEXT: v_pk_mul_lo_u16 v1, v1, v5
; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-SDAG-LABEL: clpeak_imad_pat_v3i16:
; GFX10-SDAG: ; %bb.0: ; %entry
; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_pk_add_u16 v1, v1, 1
; GFX10-SDAG-NEXT: v_pk_add_u16 v0, v0, 1 op_sel_hi:[1,0]
-; GFX10-SDAG-NEXT: v_pk_mul_lo_u16 v4, v1, v3
-; GFX10-SDAG-NEXT: v_pk_mul_lo_u16 v5, v0, v2
-; GFX10-SDAG-NEXT: v_pk_add_u16 v1, v4, v1
-; GFX10-SDAG-NEXT: v_pk_add_u16 v0, v5, v0
-; GFX10-SDAG-NEXT: v_pk_mul_lo_u16 v1, v1, v3
-; GFX10-SDAG-NEXT: v_pk_mul_lo_u16 v0, v0, v2
-; GFX10-SDAG-NEXT: v_pk_add_u16 v2, v5, 1 op_sel_hi:[1,0]
-; GFX10-SDAG-NEXT: v_pk_add_u16 v3, v4, 1
-; GFX10-SDAG-NEXT: v_pk_add_u16 v4, v1, 1
-; GFX10-SDAG-NEXT: v_pk_add_u16 v5, v0, 1 op_sel_hi:[1,0]
+; GFX10-SDAG-NEXT: v_pk_add_u16 v1, v1, 1
+; GFX10-SDAG-NEXT: v_pk_mad_u16 v4, v0, v2, v0
+; GFX10-SDAG-NEXT: v_pk_mad_u16 v5, v1, v3, v1
+; GFX10-SDAG-NEXT: v_pk_mad_u16 v0, v0, v2, 1 op_sel_hi:[1,1,0]
+; GFX10-SDAG-NEXT: v_pk_mad_u16 v1, v1, v3, 1
+; GFX10-SDAG-NEXT: v_pk_mul_lo_u16 v6, v4, v2
+; GFX10-SDAG-NEXT: v_pk_mul_lo_u16 v7, v5, v3
+; GFX10-SDAG-NEXT: v_pk_mad_u16 v3, v5, v3, 1
+; GFX10-SDAG-NEXT: v_pk_mad_u16 v2, v4, v2, 1 op_sel_hi:[1,1,0]
+; GFX10-SDAG-NEXT: v_pk_mul_lo_u16 v0, v6, v0
+; GFX10-SDAG-NEXT: v_pk_mul_lo_u16 v1, v7, v1
; GFX10-SDAG-NEXT: v_pk_mul_lo_u16 v0, v0, v2
; GFX10-SDAG-NEXT: v_pk_mul_lo_u16 v1, v1, v3
-; GFX10-SDAG-NEXT: v_pk_mul_lo_u16 v0, v0, v5
-; GFX10-SDAG-NEXT: v_pk_mul_lo_u16 v1, v1, v4
; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-GISEL-LABEL: clpeak_imad_pat_v3i16:
@@ -756,47 +746,41 @@ define <3 x i16> @clpeak_imad_pat_v3i16(<3 x i16> %x, <3 x i16> %y) {
; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-GISEL-NEXT: v_pk_add_u16 v0, v0, 1 op_sel_hi:[1,0]
; GFX10-GISEL-NEXT: v_pk_add_u16 v1, v1, 1
-; GFX10-GISEL-NEXT: v_pk_mul_lo_u16 v4, v0, v2
-; GFX10-GISEL-NEXT: v_pk_mul_lo_u16 v5, v1, v3
-; GFX10-GISEL-NEXT: v_pk_add_u16 v0, v4, v0
-; GFX10-GISEL-NEXT: v_pk_add_u16 v1, v5, v1
-; GFX10-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v2
-; GFX10-GISEL-NEXT: v_pk_mul_lo_u16 v1, v1, v3
-; GFX10-GISEL-NEXT: v_pk_add_u16 v2, v4, 1 op_sel_hi:[1,0]
-; GFX10-GISEL-NEXT: v_pk_add_u16 v3, v5, 1
-; GFX10-GISEL-NEXT: v_pk_add_u16 v4, v0, 1 op_sel_hi:[1,0]
-; GFX10-GISEL-NEXT: v_pk_add_u16 v5, v1, 1
+; GFX10-GISEL-NEXT: v_pk_mad_u16 v4, v0, v2, v0
+; GFX10-GISEL-NEXT: v_pk_mad_u16 v5, v1, v3, v1
+; GFX10-GISEL-NEXT: v_pk_mad_u16 v0, v0, v2, 1 op_sel_hi:[1,1,0]
+; GFX10-GISEL-NEXT: v_pk_mad_u16 v1, v1, v3, 1
+; GFX10-GISEL-NEXT: v_pk_mul_lo_u16 v6, v4, v2
+; GFX10-GISEL-NEXT: v_pk_mul_lo_u16 v7, v5, v3
+; GFX10-GISEL-NEXT: v_pk_mad_u16 v2, v4, v2, 1 op_sel_hi:[1,1,0]
+; GFX10-GISEL-NEXT: v_pk_mad_u16 v3, v5, v3, 1
+; GFX10-GISEL-NEXT: v_pk_mul_lo_u16 v0, v6, v0
+; GFX10-GISEL-NEXT: v_pk_mul_lo_u16 v1, v7, v1
; GFX10-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v2
; GFX10-GISEL-NEXT: v_pk_mul_lo_u16 v1, v1, v3
-; GFX10-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v4
-; GFX10-GISEL-NEXT: v_pk_mul_lo_u16 v1, v1, v5
; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-LABEL: clpeak_imad_pat_v3i16:
; GFX11-SDAG: ; %bb.0: ; %entry
; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-NEXT: v_pk_add_u16 v1, v1, 1
; GFX11-SDAG-NEXT: v_pk_add_u16 v0, v0, 1 op_sel_hi:[1,0]
+; GFX11-SDAG-NEXT: v_pk_add_u16 v1, v1, 1
; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v4, v1, v3
-; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v5, v0, v2
-; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-NEXT: v_pk_add_u16 v1, v4, v1
-; GFX11-SDAG-NEXT: v_pk_add_u16 v0, v5, v0
-; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v1, v1, v3
-; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v0, v0, v2
-; GFX11-SDAG-NEXT: v_pk_add_u16 v2, v5, 1 op_sel_hi:[1,0]
-; GFX11-SDAG-NEXT: v_pk_add_u16 v3, v4, 1
+; GFX11-SDAG-NEXT: v_pk_mad_u16 v4, v0, v2, v0
+; GFX11-SDAG-NEXT: v_pk_mad_u16 v5, v1, v3, v1
+; GFX11-SDAG-NEXT: v_pk_mad_u16 v0, v0, v2, 1 op_sel_hi:[1,1,0]
+; GFX11-SDAG-NEXT: v_pk_mad_u16 v1, v1, v3, 1
; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-SDAG-NEXT: v_pk_add_u16 v4, v1, 1
-; GFX11-SDAG-NEXT: v_pk_add_u16 v5, v0, 1 op_sel_hi:[1,0]
+; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v6, v4, v2
+; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v7, v5, v3
+; GFX11-SDAG-NEXT: v_pk_mad_u16 v3, v5, v3, 1
+; GFX11-SDAG-NEXT: v_pk_mad_u16 v2, v4, v2, 1 op_sel_hi:[1,1,0]
; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v0, v6, v0
+; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v1, v7, v1
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v0, v0, v2
; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v1, v1, v3
-; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v0, v0, v5
-; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v1, v1, v4
; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-GISEL-LABEL: clpeak_imad_pat_v3i16:
@@ -805,25 +789,21 @@ define <3 x i16> @clpeak_imad_pat_v3i16(<3 x i16> %x, <3 x i16> %y) {
; GFX11-GISEL-NEXT: v_pk_add_u16 v0, v0, 1 op_sel_hi:[1,0]
; GFX11-GISEL-NEXT: v_pk_add_u16 v1, v1, 1
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v4, v0, v2
-; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v5, v1, v3
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-GISEL-NEXT: v_pk_add_u16 v0, v4, v0
-; GFX11-GISEL-NEXT: v_pk_add_u16 v1, v5, v1
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v2
-; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v1, v1, v3
-; GFX11-GISEL-NEXT: v_pk_add_u16 v2, v4, 1 op_sel_hi:[1,0]
-; GFX11-GISEL-NEXT: v_pk_add_u16 v3, v5, 1
+; GFX11-GISEL-NEXT: v_pk_mad_u16 v4, v0, v2, v0
+; GFX11-GISEL-NEXT: v_pk_mad_u16 v5, v1, v3, v1
+; GFX11-GISEL-NEXT: v_pk_mad_u16 v0, v0, v2, 1 op_sel_hi:[1,1,0]
+; GFX11-GISEL-NEXT: v_pk_mad_u16 v1, v1, v3, 1
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-GISEL-NEXT: v_pk_add_u16 v4, v0, 1 op_sel_hi:[1,0]
-; GFX11-GISEL-NEXT: v_pk_add_u16 v5, v1, 1
+; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v6, v4, v2
+; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v7, v5, v3
+; GFX11-GISEL-NEXT: v_pk_mad_u16 v2, v4, v2, 1 op_sel_hi:[1,1,0]
+; GFX11-GISEL-NEXT: v_pk_mad_u16 v3, v5, v3, 1
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v0, v6, v0
+; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v1, v7, v1
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v2
; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v1, v1, v3
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v4
-; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v1, v1, v5
; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
entry:
%y48 = add <3 x i16> %x, <i16 1, i16 1, i16 1>
@@ -1088,22 +1068,20 @@ define <4 x i16> @clpeak_imad_pat_v4i16(<4 x i16> %x, <4 x i16> %y) {
; GFX9-SDAG-LABEL: clpeak_imad_pat_v4i16:
; GFX9-SDAG: ; %bb.0: ; %entry
; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-SDAG-NEXT: v_pk_add_u16 v1, v1, 1 op_sel_hi:[1,0]
; GFX9-SDAG-NEXT: v_pk_add_u16 v0, v0, 1 op_sel_hi:[1,0]
-; GFX9-SDAG-NEXT: v_pk_mul_lo_u16 v4, v0, v2
-; GFX9-SDAG-NEXT: v_pk_mul_lo_u16 v5, v1, v3
-; GFX9-SDAG-NEXT: v_pk_add_u16 v1, v5, v1
-; GFX9-SDAG-NEXT: v_pk_add_u16 v0, v4, v0
+; GFX9-SDAG-NEXT: v_pk_add_u16 v1, v1, 1 op_sel_hi:[1,0]
+; GFX9-SDAG-NEXT: v_pk_mad_u16 v4, v1, v3, v1
+; GFX9-SDAG-NEXT: v_pk_mad_u16 v5, v0, v2, v0
+; GFX9-SDAG-NEXT: v_pk_mul_lo_u16 v6, v5, v2
+; GFX9-SDAG-NEXT: v_pk_mul_lo_u16 v7, v4, v3
+; GFX9-SDAG-NEXT: v_pk_mad_u16 v0, v0, v2, 1 op_sel_hi:[1,1,0]
+; GFX9-SDAG-NEXT: v_pk_mad_u16 v1, v1, v3, 1 op_sel_hi:[1,1,0]
+; GFX9-SDAG-NEXT: v_pk_mad_u16 v3, v4, v3, 1 op_sel_hi:[1,1,0]
+; GFX9-SDAG-NEXT: v_pk_mad_u16 v2, v5, v2, 1 op_sel_hi:[1,1,0]
+; GFX9-SDAG-NEXT: v_pk_mul_lo_u16 v1, v7, v1
+; GFX9-SDAG-NEXT: v_pk_mul_lo_u16 v0, v6, v0
; GFX9-SDAG-NEXT: v_pk_mul_lo_u16 v0, v0, v2
; GFX9-SDAG-NEXT: v_pk_mul_lo_u16 v1, v1, v3
-; GFX9-SDAG-NEXT: v_pk_add_u16 v2, v4, 1 op_sel_hi:[1,0]
-; GFX9-SDAG-NEXT: v_pk_add_u16 v3, v5, 1 op_sel_hi:[1,0]
-; GFX9-SDAG-NEXT: v_pk_add_u16 v4, v1, 1 op_sel_hi:[1,0]
-; GFX9-SDAG-NEXT: v_pk_add_u16 v5, v0, 1 op_sel_hi:[1,0]
-; GFX9-SDAG-NEXT: v_pk_mul_lo_u16 v1, v1, v3
-; GFX9-SDAG-NEXT: v_pk_mul_lo_u16 v0, v0, v2
-; GFX9-SDAG-NEXT: v_pk_mul_lo_u16 v0, v0, v5
-; GFX9-SDAG-NEXT: v_pk_mul_lo_u16 v1, v1, v4
; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-GISEL-LABEL: clpeak_imad_pat_v4i16:
@@ -1111,41 +1089,37 @@ define <4 x i16> @clpeak_imad_pat_v4i16(<4 x i16> %x, <4 x i16> %y) {
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-GISEL-NEXT: v_pk_add_u16 v0, v0, 1 op_sel_hi:[1,0]
; GFX9-GISEL-NEXT: v_pk_add_u16 v1, v1, 1 op_sel_hi:[1,0]
-; GFX9-GISEL-NEXT: v_pk_mul_lo_u16 v4, v0, v2
-; GFX9-GISEL-NEXT: v_pk_mul_lo_u16 v5, v1, v3
-; GFX9-GISEL-NEXT: v_pk_add_u16 v0, v4, v0
-; GFX9-GISEL-NEXT: v_pk_add_u16 v1, v5, v1
-; GFX9-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v2
-; GFX9-GISEL-NEXT: v_pk_mul_lo_u16 v1, v1, v3
-; GFX9-GISEL-NEXT: v_pk_add_u16 v2, v4, 1 op_sel_hi:[1,0]
-; GFX9-GISEL-NEXT: v_pk_add_u16 v3, v5, 1 op_sel_hi:[1,0]
-; GFX9-GISEL-NEXT: v_pk_add_u16 v4, v0, 1 op_sel_hi:[1,0]
-; GFX9-GISEL-NEXT: v_pk_add_u16 v5, v1, 1 op_sel_hi:[1,0]
+; GFX9-GISEL-NEXT: v_pk_mad_u16 v4, v0, v2, v0
+; GFX9-GISEL-NEXT: v_pk_mad_u16 v5, v1, v3, v1
+; GFX9-GISEL-NEXT: v_pk_mul_lo_u16 v6, v4, v2
+; GFX9-GISEL-NEXT: v_pk_mul_lo_u16 v7, v5, v3
+; GFX9-GISEL-NEXT: v_pk_mad_u16 v0, v0, v2, 1 op_sel_hi:[1,1,0]
+; GFX9-GISEL-NEXT: v_pk_mad_u16 v1, v1, v3, 1 op_sel_hi:[1,1,0]
+; GFX9-GISEL-NEXT: v_pk_mad_u16 v2, v4, v2, 1 op_sel_hi:[1,1,0]
+; GFX9-GISEL-NEXT: v_pk_mad_u16 v3, v5, v3, 1 op_sel_hi:[1,1,0]
+; GFX9-GISEL-NEXT: v_pk_mul_lo_u16 v0, v6, v0
+; GFX9-GISEL-NEXT: v_pk_mul_lo_u16 v1, v7, v1
; GFX9-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v2
; GFX9-GISEL-NEXT: v_pk_mul_lo_u16 v1, v1, v3
-; GFX9-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v4
-; GFX9-GISEL-NEXT: v_pk_mul_lo_u16 v1, v1, v5
; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-SDAG-LABEL: clpeak_imad_pat_v4i16:
; GFX10-SDAG: ; %bb.0: ; %entry
; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_pk_add_u16 v1, v1, 1 op_sel_hi:[1,0]
; GFX10-SDAG-NEXT: v_pk_add_u16 v0, v0, 1 op_sel_hi:[1,0]
-; GFX10-SDAG-NEXT: v_pk_mul_lo_u16 v4, v1, v3
-; GFX10-SDAG-NEXT: v_pk_mul_lo_u16 v5, v0, v2
-; GFX10-SDAG-NEXT: v_pk_add_u16 v1, v4, v1
-; GFX10-SDAG-NEXT: v_pk_add_u16 v0, v5, v0
-; GFX10-SDAG-NEXT: v_pk_mul_lo_u16 v1, v1, v3
-; GFX10-SDAG-NEXT: v_pk_mul_lo_u16 v0, v0, v2
-; GFX10-SDAG-NEXT: v_pk_add_u16 v2, v5, 1 op_sel_hi:[1,0]
-; GFX10-SDAG-NEXT: v_pk_add_u16 v3, v4, 1 op_sel_hi:[1,0]
-; GFX10-SDAG-NEXT: v_pk_add_u16 v4, v1, 1 op_sel_hi:[1,0]
-; GFX10-SDAG-NEXT: v_pk_add_u16 v5, v0, 1 op_sel_hi:[1,0]
+; GFX10-SDAG-NEXT: v_pk_add_u16 v1, v1, 1 op_sel_hi:[1,0]
+; GFX10-SDAG-NEXT: v_pk_mad_u16 v4, v0, v2, v0
+; GFX10-SDAG-NEXT: v_pk_mad_u16 v5, v1, v3, v1
+; GFX10-SDAG-NEXT: v_pk_mad_u16 v0, v0, v2, 1 op_sel_hi:[1,1,0]
+; GFX10-SDAG-NEXT: v_pk_mad_u16 v1, v1, v3, 1 op_sel_hi:[1,1,0]
+; GFX10-SDAG-NEXT: v_pk_mul_lo_u16 v6, v4, v2
+; GFX10-SDAG-NEXT: v_pk_mul_lo_u16 v7, v5, v3
+; GFX10-SDAG-NEXT: v_pk_mad_u16 v3, v5, v3, 1 op_sel_hi:[1,1,0]
+; GFX10-SDAG-NEXT: v_pk_mad_u16 v2, v4, v2, 1 op_sel_hi:[1,1,0]
+; GFX10-SDAG-NEXT: v_pk_mul_lo_u16 v0, v6, v0
+; GFX10-SDAG-NEXT: v_pk_mul_lo_u16 v1, v7, v1
; GFX10-SDAG-NEXT: v_pk_mul_lo_u16 v0, v0, v2
; GFX10-SDAG-NEXT: v_pk_mul_lo_u16 v1, v1, v3
-; GFX10-SDAG-NEXT: v_pk_mul_lo_u16 v0, v0, v5
-; GFX10-SDAG-NEXT: v_pk_mul_lo_u16 v1, v1, v4
; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-GISEL-LABEL: clpeak_imad_pat_v4i16:
@@ -1153,47 +1127,41 @@ define <4 x i16> @clpeak_imad_pat_v4i16(<4 x i16> %x, <4 x i16> %y) {
; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-GISEL-NEXT: v_pk_add_u16 v0, v0, 1 op_sel_hi:[1,0]
; GFX10-GISEL-NEXT: v_pk_add_u16 v1, v1, 1 op_sel_hi:[1,0]
-; GFX10-GISEL-NEXT: v_pk_mul_lo_u16 v4, v0, v2
-; GFX10-GISEL-NEXT: v_pk_mul_lo_u16 v5, v1, v3
-; GFX10-GISEL-NEXT: v_pk_add_u16 v0, v4, v0
-; GFX10-GISEL-NEXT: v_pk_add_u16 v1, v5, v1
-; GFX10-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v2
-; GFX10-GISEL-NEXT: v_pk_mul_lo_u16 v1, v1, v3
-; GFX10-GISEL-NEXT: v_pk_add_u16 v2, v4, 1 op_sel_hi:[1,0]
-; GFX10-GISEL-NEXT: v_pk_add_u16 v3, v5, 1 op_sel_hi:[1,0]
-; GFX10-GISEL-NEXT: v_pk_add_u16 v4, v0, 1 op_sel_hi:[1,0]
-; GFX10-GISEL-NEXT: v_pk_add_u16 v5, v1, 1 op_sel_hi:[1,0]
+; GFX10-GISEL-NEXT: v_pk_mad_u16 v4, v0, v2, v0
+; GFX10-GISEL-NEXT: v_pk_mad_u16 v5, v1, v3, v1
+; GFX10-GISEL-NEXT: v_pk_mad_u16 v0, v0, v2, 1 op_sel_hi:[1,1,0]
+; GFX10-GISEL-NEXT: v_pk_mad_u16 v1, v1, v3, 1 op_sel_hi:[1,1,0]
+; GFX10-GISEL-NEXT: v_pk_mul_lo_u16 v6, v4, v2
+; GFX10-GISEL-NEXT: v_pk_mul_lo_u16 v7, v5, v3
+; GFX10-GISEL-NEXT: v_pk_mad_u16 v2, v4, v2, 1 op_sel_hi:[1,1,0]
+; GFX10-GISEL-NEXT: v_pk_mad_u16 v3, v5, v3, 1 op_sel_hi:[1,1,0]
+; GFX10-GISEL-NEXT: v_pk_mul_lo_u16 v0, v6, v0
+; GFX10-GISEL-NEXT: v_pk_mul_lo_u16 v1, v7, v1
; GFX10-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v2
; GFX10-GISEL-NEXT: v_pk_mul_lo_u16 v1, v1, v3
-; GFX10-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v4
-; GFX10-GISEL-NEXT: v_pk_mul_lo_u16 v1, v1, v5
; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-LABEL: clpeak_imad_pat_v4i16:
; GFX11-SDAG: ; %bb.0: ; %entry
; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-NEXT: v_pk_add_u16 v1, v1, 1 op_sel_hi:[1,0]
; GFX11-SDAG-NEXT: v_pk_add_u16 v0, v0, 1 op_sel_hi:[1,0]
+; GFX11-SDAG-NEXT: v_pk_add_u16 v1, v1, 1 op_sel_hi:[1,0]
; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v4, v1, v3
-; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v5, v0, v2
-; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-NEXT: v_pk_add_u16 v1, v4, v1
-; GFX11-SDAG-NEXT: v_pk_add_u16 v0, v5, v0
-; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v1, v1, v3
-; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v0, v0, v2
-; GFX11-SDAG-NEXT: v_pk_add_u16 v2, v5, 1 op_sel_hi:[1,0]
-; GFX11-SDAG-NEXT: v_pk_add_u16 v3, v4, 1 op_sel_hi:[1,0]
+; GFX11-SDAG-NEXT: v_pk_mad_u16 v4, v0, v2, v0
+; GFX11-SDAG-NEXT: v_pk_mad_u16 v5, v1, v3, v1
+; GFX11-SDAG-NEXT: v_pk_mad_u16 v0, v0, v2, 1 op_sel_hi:[1,1,0]
+; GFX11-SDAG-NEXT: v_pk_mad_u16 v1, v1, v3, 1 op_sel_hi:[1,1,0]
; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-SDAG-NEXT: v_pk_add_u16 v4, v1, 1 op_sel_hi:[1,0]
-; GFX11-SDAG-NEXT: v_pk_add_u16 v5, v0, 1 op_sel_hi:[1,0]
+; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v6, v4, v2
+; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v7, v5, v3
+; GFX11-SDAG-NEXT: v_pk_mad_u16 v3, v5, v3, 1 op_sel_hi:[1,1,0]
+; GFX11-SDAG-NEXT: v_pk_mad_u16 v2, v4, v2, 1 op_sel_hi:[1,1,0]
; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v0, v6, v0
+; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v1, v7, v1
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v0, v0, v2
; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v1, v1, v3
-; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v0, v0, v5
-; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v1, v1, v4
; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-GISEL-LABEL: clpeak_imad_pat_v4i16:
@@ -1202,25 +1170,21 @@ define <4 x i16> @clpeak_imad_pat_v4i16(<4 x i16> %x, <4 x i16> %y) {
; GFX11-GISEL-NEXT: v_pk_add_u16 v0, v0, 1 op_sel_hi:[1,0]
; GFX11-GISEL-NEXT: v_pk_add_u16 v1, v1, 1 op_sel_hi:[1,0]
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v4, v0, v2
-; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v5, v1, v3
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-GISEL-NEXT: v_pk_add_u16 v0, v4, v0
-; GFX11-GISEL-NEXT: v_pk_add_u16 v1, v5, v1
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v2
-; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v1, v1, v3
-; GFX11-GISEL-NEXT: v_pk_add_u16 v2, v4, 1 op_sel_hi:[1,0]
-; GFX11-GISEL-NEXT: v_pk_add_u16 v3, v5, 1 op_sel_hi:[1,0]
+; GFX11-GISEL-NEXT: v_pk_mad_u16 v4, v0, v2, v0
+; GFX11-GISEL-NEXT: v_pk_mad_u16 v5, v1, v3, v1
+; GFX11-GISEL-NEXT: v_pk_mad_u16 v0, v0, v2, 1 op_sel_hi:[1,1,0]
+; GFX11-GISEL-NEXT: v_pk_mad_u16 v1, v1, v3, 1 op_sel_hi:[1,1,0]
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-GISEL-NEXT: v_pk_add_u16 v4, v0, 1 op_sel_hi:[1,0]
-; GFX11-GISEL-NEXT: v_pk_add_u16 v5, v1, 1 op_sel_hi:[1,0]
+; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v6, v4, v2
+; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v7, v5, v3
+; GFX11-GISEL-NEXT: v_pk_mad_u16 v2, v4, v2, 1 op_sel_hi:[1,1,0]
+; GFX11-GISEL-NEXT: v_pk_mad_u16 v3, v5, v3, 1 op_sel_hi:[1,1,0]
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v0, v6, v0
+; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v1, v7, v1
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v2
; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v1, v1, v3
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v4
-; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v1, v1, v5
; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
entry:
%y18 = add <4 x i16> %x, <i16 1, i16 1, i16 1, i16 1>
@@ -1517,42 +1481,38 @@ define <2 x i16> @clpeak_umad_pat_v2i16(<2 x i16> %x, <2 x i16> %y) {
; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_pk_add_u16 v0, v0, 1 op_sel_hi:[1,0]
-; GFX9-NEXT: v_pk_mul_lo_u16 v2, v0, v1
-; GFX9-NEXT: v_pk_add_u16 v0, v2, v0
+; GFX9-NEXT: v_pk_mad_u16 v2, v0, v1, v0
+; GFX9-NEXT: v_pk_mul_lo_u16 v3, v2, v1
+; GFX9-NEXT: v_pk_mad_u16 v0, v0, v1, 1 op_sel_hi:[1,1,0]
+; GFX9-NEXT: v_pk_mad_u16 v1, v2, v1, 1 op_sel_hi:[1,1,0]
+; GFX9-NEXT: v_pk_mul_lo_u16 v0, v3, v0
; GFX9-NEXT: v_pk_mul_lo_u16 v0, v0, v1
-; GFX9-NEXT: v_pk_add_u16 v1, v2, 1 op_sel_hi:[1,0]
-; GFX9-NEXT: v_pk_add_u16 v2, v0, 1 op_sel_hi:[1,0]
-; GFX9-NEXT: v_pk_mul_lo_u16 v0, v0, v1
-; GFX9-NEXT: v_pk_mul_lo_u16 v0, v0, v2
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: clpeak_umad_pat_v2i16:
; GFX10: ; %bb.0: ; %entry
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_pk_add_u16 v0, v0, 1 op_sel_hi:[1,0]
-; GFX10-NEXT: v_pk_mul_lo_u16 v2, v0, v1
-; GFX10-NEXT: v_pk_add_u16 v0, v2, v0
-; GFX10-NEXT: v_pk_mul_lo_u16 v0, v0, v1
-; GFX10-NEXT: v_pk_add_u16 v1, v2, 1 op_sel_hi:[1,0]
-; GFX10-NEXT: v_pk_add_u16 v2, v0, 1 op_sel_hi:[1,0]
+; GFX10-NEXT: v_pk_mad_u16 v2, v0, v1, v0
+; GFX10-NEXT: v_pk_mad_u16 v0, v0, v1, 1 op_sel_hi:[1,1,0]
+; GFX10-NEXT: v_pk_mul_lo_u16 v3, v2, v1
+; GFX10-NEXT: v_pk_mad_u16 v1, v2, v1, 1 op_sel_hi:[1,1,0]
+; GFX10-NEXT: v_pk_mul_lo_u16 v0, v3, v0
; GFX10-NEXT: v_pk_mul_lo_u16 v0, v0, v1
-; GFX10-NEXT: v_pk_mul_lo_u16 v0, v0, v2
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: clpeak_umad_pat_v2i16:
; GFX11: ; %bb.0: ; %entry
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_pk_add_u16 v0, v0, 1 op_sel_hi:[1,0]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_pk_mul_lo_u16 v2, v0, v1
-; GFX11-NEXT: v_pk_add_u16 v0, v2, v0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_pk_mul_lo_u16 v0, v0, v1
-; GFX11-NEXT: v_pk_add_u16 v1, v2, 1 op_sel_hi:[1,0]
-; GFX11-NEXT: v_pk_add_u16 v2, v0, 1 op_sel_hi:[1,0]
+; GFX11-NEXT: v_pk_mad_u16 v2, v0, v1, v0
+; GFX11-NEXT: v_pk_mad_u16 v0, v0, v1, 1 op_sel_hi:[1,1,0]
+; GFX11-NEXT: v_pk_mul_lo_u16 v3, v2, v1
+; GFX11-NEXT: v_pk_mad_u16 v1, v2, v1, 1 op_sel_hi:[1,1,0]
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_pk_mul_lo_u16 v0, v3, v0
; GFX11-NEXT: v_pk_mul_lo_u16 v0, v0, v1
-; GFX11-NEXT: v_pk_mul_lo_u16 v0, v0, v2
; GFX11-NEXT: s_setpc_b64 s[30:31]
entry:
%y18 = add <2 x i16> %x, <i16 1, i16 1>
@@ -1731,22 +1691,20 @@ define <3 x i16> @clpeak_umad_pat_v3i16(<3 x i16> %x, <3 x i16> %y) {
; GFX9-SDAG-LABEL: clpeak_umad_pat_v3i16:
; GFX9-SDAG: ; %bb.0: ; %entry
; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-SDAG-NEXT: v_pk_add_u16 v1, v1, 1
; GFX9-SDAG-NEXT: v_pk_add_u16 v0, v0, 1 op_sel_hi:[1,0]
-; GFX9-SDAG-NEXT: v_pk_mul_lo_u16 v4, v0, v2
-; GFX9-SDAG-NEXT: v_pk_mul_lo_u16 v5, v1, v3
-; GFX9-SDAG-NEXT: v_pk_add_u16 v1, v5, v1
-; GFX9-SDAG-NEXT: v_pk_add_u16 v0, v4, v0
+; GFX9-SDAG-NEXT: v_pk_add_u16 v1, v1, 1
+; GFX9-SDAG-NEXT: v_pk_mad_u16 v4, v1, v3, v1
+; GFX9-SDAG-NEXT: v_pk_mad_u16 v5, v0, v2, v0
+; GFX9-SDAG-NEXT: v_pk_mul_lo_u16 v6, v5, v2
+; GFX9-SDAG-NEXT: v_pk_mul_lo_u16 v7, v4, v3
+; GFX9-SDAG-NEXT: v_pk_mad_u16 v0, v0, v2, 1 op_sel_hi:[1,1,0]
+; GFX9-SDAG-NEXT: v_pk_mad_u16 v1, v1, v3, 1
+; GFX9-SDAG-NEXT: v_pk_mad_u16 v3, v4, v3, 1
+; GFX9-SDAG-NEXT: v_pk_mad_u16 v2, v5, v2, 1 op_sel_hi:[1,1,0]
+; GFX9-SDAG-NEXT: v_pk_mul_lo_u16 v1, v7, v1
+; GFX9-SDAG-NEXT: v_pk_mul_lo_u16 v0, v6, v0
; GFX9-SDAG-NEXT: v_pk_mul_lo_u16 v0, v0, v2
; GFX9-SDAG-NEXT: v_pk_mul_lo_u16 v1, v1, v3
-; GFX9-SDAG-NEXT: v_pk_add_u16 v2, v4, 1 op_sel_hi:[1,0]
-; GFX9-SDAG-NEXT: v_pk_add_u16 v3, v5, 1
-; GFX9-SDAG-NEXT: v_pk_add_u16 v4, v1, 1
-; GFX9-SDAG-NEXT: v_pk_add_u16 v5, v0, 1 op_sel_hi:[1,0]
-; GFX9-SDAG-NEXT: v_pk_mul_lo_u16 v1, v1, v3
-; GFX9-SDAG-NEXT: v_pk_mul_lo_u16 v0, v0, v2
-; GFX9-SDAG-NEXT: v_pk_mul_lo_u16 v0, v0, v5
-; GFX9-SDAG-NEXT: v_pk_mul_lo_u16 v1, v1, v4
; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-GISEL-LABEL: clpeak_umad_pat_v3i16:
@@ -1754,41 +1712,37 @@ define <3 x i16> @clpeak_umad_pat_v3i16(<3 x i16> %x, <3 x i16> %y) {
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-GISEL-NEXT: v_pk_add_u16 v0, v0, 1 op_sel_hi:[1,0]
; GFX9-GISEL-NEXT: v_pk_add_u16 v1, v1, 1
-; GFX9-GISEL-NEXT: v_pk_mul_lo_u16 v4, v0, v2
-; GFX9-GISEL-NEXT: v_pk_mul_lo_u16 v5, v1, v3
-; GFX9-GISEL-NEXT: v_pk_add_u16 v0, v4, v0
-; GFX9-GISEL-NEXT: v_pk_add_u16 v1, v5, v1
-; GFX9-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v2
-; GFX9-GISEL-NEXT: v_pk_mul_lo_u16 v1, v1, v3
-; GFX9-GISEL-NEXT: v_pk_add_u16 v2, v4, 1 op_sel_hi:[1,0]
-; GFX9-GISEL-NEXT: v_pk_add_u16 v3, v5, 1
-; GFX9-GISEL-NEXT: v_pk_add_u16 v4, v0, 1 op_sel_hi:[1,0]
-; GFX9-GISEL-NEXT: v_pk_add_u16 v5, v1, 1
+; GFX9-GISEL-NEXT: v_pk_mad_u16 v4, v0, v2, v0
+; GFX9-GISEL-NEXT: v_pk_mad_u16 v5, v1, v3, v1
+; GFX9-GISEL-NEXT: v_pk_mul_lo_u16 v6, v4, v2
+; GFX9-GISEL-NEXT: v_pk_mul_lo_u16 v7, v5, v3
+; GFX9-GISEL-NEXT: v_pk_mad_u16 v0, v0, v2, 1 op_sel_hi:[1,1,0]
+; GFX9-GISEL-NEXT: v_pk_mad_u16 v1, v1, v3, 1
+; GFX9-GISEL-NEXT: v_pk_mad_u16 v2, v4, v2, 1 op_sel_hi:[1,1,0]
+; GFX9-GISEL-NEXT: v_pk_mad_u16 v3, v5, v3, 1
+; GFX9-GISEL-NEXT: v_pk_mul_lo_u16 v0, v6, v0
+; GFX9-GISEL-NEXT: v_pk_mul_lo_u16 v1, v7, v1
; GFX9-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v2
; GFX9-GISEL-NEXT: v_pk_mul_lo_u16 v1, v1, v3
-; GFX9-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v4
-; GFX9-GISEL-NEXT: v_pk_mul_lo_u16 v1, v1, v5
; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-SDAG-LABEL: clpeak_umad_pat_v3i16:
; GFX10-SDAG: ; %bb.0: ; %entry
; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_pk_add_u16 v1, v1, 1
; GFX10-SDAG-NEXT: v_pk_add_u16 v0, v0, 1 op_sel_hi:[1,0]
-; GFX10-SDAG-NEXT: v_pk_mul_lo_u16 v4, v1, v3
-; GFX10-SDAG-NEXT: v_pk_mul_lo_u16 v5, v0, v2
-; GFX10-SDAG-NEXT: v_pk_add_u16 v1, v4, v1
-; GFX10-SDAG-NEXT: v_pk_add_u16 v0, v5, v0
-; GFX10-SDAG-NEXT: v_pk_mul_lo_u16 v1, v1, v3
-; GFX10-SDAG-NEXT: v_pk_mul_lo_u16 v0, v0, v2
-; GFX10-SDAG-NEXT: v_pk_add_u16 v2, v5, 1 op_sel_hi:[1,0]
-; GFX10-SDAG-NEXT: v_pk_add_u16 v3, v4, 1
-; GFX10-SDAG-NEXT: v_pk_add_u16 v4, v1, 1
-; GFX10-SDAG-NEXT: v_pk_add_u16 v5, v0, 1 op_sel_hi:[1,0]
+; GFX10-SDAG-NEXT: v_pk_add_u16 v1, v1, 1
+; GFX10-SDAG-NEXT: v_pk_mad_u16 v4, v0, v2, v0
+; GFX10-SDAG-NEXT: v_pk_mad_u16 v5, v1, v3, v1
+; GFX10-SDAG-NEXT: v_pk_mad_u16 v0, v0, v2, 1 op_sel_hi:[1,1,0]
+; GFX10-SDAG-NEXT: v_pk_mad_u16 v1, v1, v3, 1
+; GFX10-SDAG-NEXT: v_pk_mul_lo_u16 v6, v4, v2
+; GFX10-SDAG-NEXT: v_pk_mul_lo_u16 v7, v5, v3
+; GFX10-SDAG-NEXT: v_pk_mad_u16 v3, v5, v3, 1
+; GFX10-SDAG-NEXT: v_pk_mad_u16 v2, v4, v2, 1 op_sel_hi:[1,1,0]
+; GFX10-SDAG-NEXT: v_pk_mul_lo_u16 v0, v6, v0
+; GFX10-SDAG-NEXT: v_pk_mul_lo_u16 v1, v7, v1
; GFX10-SDAG-NEXT: v_pk_mul_lo_u16 v0, v0, v2
; GFX10-SDAG-NEXT: v_pk_mul_lo_u16 v1, v1, v3
-; GFX10-SDAG-NEXT: v_pk_mul_lo_u16 v0, v0, v5
-; GFX10-SDAG-NEXT: v_pk_mul_lo_u16 v1, v1, v4
; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-GISEL-LABEL: clpeak_umad_pat_v3i16:
@@ -1796,47 +1750,41 @@ define <3 x i16> @clpeak_umad_pat_v3i16(<3 x i16> %x, <3 x i16> %y) {
; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-GISEL-NEXT: v_pk_add_u16 v0, v0, 1 op_sel_hi:[1,0]
; GFX10-GISEL-NEXT: v_pk_add_u16 v1, v1, 1
-; GFX10-GISEL-NEXT: v_pk_mul_lo_u16 v4, v0, v2
-; GFX10-GISEL-NEXT: v_pk_mul_lo_u16 v5, v1, v3
-; GFX10-GISEL-NEXT: v_pk_add_u16 v0, v4, v0
-; GFX10-GISEL-NEXT: v_pk_add_u16 v1, v5, v1
+; GFX10-GISEL-NEXT: v_pk_mad_u16 v4, v0, v2, v0
+; GFX10-GISEL-NEXT: v_pk_mad_u16 v5, v1, v3, v1
+; GFX10-GISEL-NEXT: v_pk_mad_u16 v0, v0, v2, 1 op_sel_hi:[1,1,0]
+; GFX10-GISEL-NEXT: v_pk_mad_u16 v1, v1, v3, 1
+; GFX10-GISEL-NEXT: v_pk_mul_lo_u16 v6, v4, v2
+; GFX10-GISEL-NEXT: v_pk_mul_lo_u16 v7, v5, v3
+; GFX10-GISEL-NEXT: v_pk_mad_u16 v2, v4, v2, 1 op_sel_hi:[1,1,0]
+; GFX10-GISEL-NEXT: v_pk_mad_u16 v3, v5, v3, 1
+; GFX10-GISEL-NEXT: v_pk_mul_lo_u16 v0, v6, v0
+; GFX10-GISEL-NEXT: v_pk_mul_lo_u16 v1, v7, v1
; GFX10-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v2
; GFX10-GISEL-NEXT: v_pk_mul_lo_u16 v1, v1, v3
-; GFX10-GISEL-NEXT: v_pk_add_u16 v2, v4, 1 op_sel_hi:[1,0]
-; GFX10-GISEL-NEXT: v_pk_add_u16 v3, v5, 1
-; GFX10-GISEL-NEXT: v_pk_add_u16 v4, v0, 1 op_sel_hi:[1,0]
-; GFX10-GISEL-NEXT: v_pk_add_u16 v5, v1, 1
-; GFX10-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v2
-; GFX10-GISEL-NEXT: v_pk_mul_lo_u16 v1, v1, v3
-; GFX10-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v4
-; GFX10-GISEL-NEXT: v_pk_mul_lo_u16 v1, v1, v5
; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-LABEL: clpeak_umad_pat_v3i16:
; GFX11-SDAG: ; %bb.0: ; %entry
; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-NEXT: v_pk_add_u16 v1, v1, 1
; GFX11-SDAG-NEXT: v_pk_add_u16 v0, v0, 1 op_sel_hi:[1,0]
+; GFX11-SDAG-NEXT: v_pk_add_u16 v1, v1, 1
; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v4, v1, v3
-; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v5, v0, v2
-; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-NEXT: v_pk_add_u16 v1, v4, v1
-; GFX11-SDAG-NEXT: v_pk_add_u16 v0, v5, v0
-; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v1, v1, v3
-; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v0, v0, v2
-; GFX11-SDAG-NEXT: v_pk_add_u16 v2, v5, 1 op_sel_hi:[1,0]
-; GFX11-SDAG-NEXT: v_pk_add_u16 v3, v4, 1
+; GFX11-SDAG-NEXT: v_pk_mad_u16 v4, v0, v2, v0
+; GFX11-SDAG-NEXT: v_pk_mad_u16 v5, v1, v3, v1
+; GFX11-SDAG-NEXT: v_pk_mad_u16 v0, v0, v2, 1 op_sel_hi:[1,1,0]
+; GFX11-SDAG-NEXT: v_pk_mad_u16 v1, v1, v3, 1
; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-SDAG-NEXT: v_pk_add_u16 v4, v1, 1
-; GFX11-SDAG-NEXT: v_pk_add_u16 v5, v0, 1 op_sel_hi:[1,0]
+; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v6, v4, v2
+; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v7, v5, v3
+; GFX11-SDAG-NEXT: v_pk_mad_u16 v3, v5, v3, 1
+; GFX11-SDAG-NEXT: v_pk_mad_u16 v2, v4, v2, 1 op_sel_hi:[1,1,0]
; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v0, v6, v0
+; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v1, v7, v1
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v0, v0, v2
; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v1, v1, v3
-; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v0, v0, v5
-; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v1, v1, v4
; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-GISEL-LABEL: clpeak_umad_pat_v3i16:
@@ -1845,25 +1793,21 @@ define <3 x i16> @clpeak_umad_pat_v3i16(<3 x i16> %x, <3 x i16> %y) {
; GFX11-GISEL-NEXT: v_pk_add_u16 v0, v0, 1 op_sel_hi:[1,0]
; GFX11-GISEL-NEXT: v_pk_add_u16 v1, v1, 1
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v4, v0, v2
-; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v5, v1, v3
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-GISEL-NEXT: v_pk_add_u16 v0, v4, v0
-; GFX11-GISEL-NEXT: v_pk_add_u16 v1, v5, v1
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v2
-; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v1, v1, v3
-; GFX11-GISEL-NEXT: v_pk_add_u16 v2, v4, 1 op_sel_hi:[1,0]
-; GFX11-GISEL-NEXT: v_pk_add_u16 v3, v5, 1
+; GFX11-GISEL-NEXT: v_pk_mad_u16 v4, v0, v2, v0
+; GFX11-GISEL-NEXT: v_pk_mad_u16 v5, v1, v3, v1
+; GFX11-GISEL-NEXT: v_pk_mad_u16 v0, v0, v2, 1 op_sel_hi:[1,1,0]
+; GFX11-GISEL-NEXT: v_pk_mad_u16 v1, v1, v3, 1
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-GISEL-NEXT: v_pk_add_u16 v4, v0, 1 op_sel_hi:[1,0]
-; GFX11-GISEL-NEXT: v_pk_add_u16 v5, v1, 1
+; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v6, v4, v2
+; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v7, v5, v3
+; GFX11-GISEL-NEXT: v_pk_mad_u16 v2, v4, v2, 1 op_sel_hi:[1,1,0]
+; GFX11-GISEL-NEXT: v_pk_mad_u16 v3, v5, v3, 1
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v0, v6, v0
+; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v1, v7, v1
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v2
; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v1, v1, v3
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v4
-; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v1, v1, v5
; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
entry:
%y48 = add <3 x i16> %x, <i16 1, i16 1, i16 1>
@@ -2128,22 +2072,20 @@ define <4 x i16> @clpeak_umad_pat_v4i16(<4 x i16> %x, <4 x i16> %y) {
; GFX9-SDAG-LABEL: clpeak_umad_pat_v4i16:
; GFX9-SDAG: ; %bb.0: ; %entry
; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-SDAG-NEXT: v_pk_add_u16 v1, v1, 1 op_sel_hi:[1,0]
; GFX9-SDAG-NEXT: v_pk_add_u16 v0, v0, 1 op_sel_hi:[1,0]
-; GFX9-SDAG-NEXT: v_pk_mul_lo_u16 v4, v0, v2
-; GFX9-SDAG-NEXT: v_pk_mul_lo_u16 v5, v1, v3
-; GFX9-SDAG-NEXT: v_pk_add_u16 v1, v5, v1
-; GFX9-SDAG-NEXT: v_pk_add_u16 v0, v4, v0
+; GFX9-SDAG-NEXT: v_pk_add_u16 v1, v1, 1 op_sel_hi:[1,0]
+; GFX9-SDAG-NEXT: v_pk_mad_u16 v4, v1, v3, v1
+; GFX9-SDAG-NEXT: v_pk_mad_u16 v5, v0, v2, v0
+; GFX9-SDAG-NEXT: v_pk_mul_lo_u16 v6, v5, v2
+; GFX9-SDAG-NEXT: v_pk_mul_lo_u16 v7, v4, v3
+; GFX9-SDAG-NEXT: v_pk_mad_u16 v0, v0, v2, 1 op_sel_hi:[1,1,0]
+; GFX9-SDAG-NEXT: v_pk_mad_u16 v1, v1, v3, 1 op_sel_hi:[1,1,0]
+; GFX9-SDAG-NEXT: v_pk_mad_u16 v3, v4, v3, 1 op_sel_hi:[1,1,0]
+; GFX9-SDAG-NEXT: v_pk_mad_u16 v2, v5, v2, 1 op_sel_hi:[1,1,0]
+; GFX9-SDAG-NEXT: v_pk_mul_lo_u16 v1, v7, v1
+; GFX9-SDAG-NEXT: v_pk_mul_lo_u16 v0, v6, v0
; GFX9-SDAG-NEXT: v_pk_mul_lo_u16 v0, v0, v2
; GFX9-SDAG-NEXT: v_pk_mul_lo_u16 v1, v1, v3
-; GFX9-SDAG-NEXT: v_pk_add_u16 v2, v4, 1 op_sel_hi:[1,0]
-; GFX9-SDAG-NEXT: v_pk_add_u16 v3, v5, 1 op_sel_hi:[1,0]
-; GFX9-SDAG-NEXT: v_pk_add_u16 v4, v1, 1 op_sel_hi:[1,0]
-; GFX9-SDAG-NEXT: v_pk_add_u16 v5, v0, 1 op_sel_hi:[1,0]
-; GFX9-SDAG-NEXT: v_pk_mul_lo_u16 v1, v1, v3
-; GFX9-SDAG-NEXT: v_pk_mul_lo_u16 v0, v0, v2
-; GFX9-SDAG-NEXT: v_pk_mul_lo_u16 v0, v0, v5
-; GFX9-SDAG-NEXT: v_pk_mul_lo_u16 v1, v1, v4
; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-GISEL-LABEL: clpeak_umad_pat_v4i16:
@@ -2151,41 +2093,37 @@ define <4 x i16> @clpeak_umad_pat_v4i16(<4 x i16> %x, <4 x i16> %y) {
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-GISEL-NEXT: v_pk_add_u16 v0, v0, 1 op_sel_hi:[1,0]
; GFX9-GISEL-NEXT: v_pk_add_u16 v1, v1, 1 op_sel_hi:[1,0]
-; GFX9-GISEL-NEXT: v_pk_mul_lo_u16 v4, v0, v2
-; GFX9-GISEL-NEXT: v_pk_mul_lo_u16 v5, v1, v3
-; GFX9-GISEL-NEXT: v_pk_add_u16 v0, v4, v0
-; GFX9-GISEL-NEXT: v_pk_add_u16 v1, v5, v1
-; GFX9-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v2
-; GFX9-GISEL-NEXT: v_pk_mul_lo_u16 v1, v1, v3
-; GFX9-GISEL-NEXT: v_pk_add_u16 v2, v4, 1 op_sel_hi:[1,0]
-; GFX9-GISEL-NEXT: v_pk_add_u16 v3, v5, 1 op_sel_hi:[1,0]
-; GFX9-GISEL-NEXT: v_pk_add_u16 v4, v0, 1 op_sel_hi:[1,0]
-; GFX9-GISEL-NEXT: v_pk_add_u16 v5, v1, 1 op_sel_hi:[1,0]
+; GFX9-GISEL-NEXT: v_pk_mad_u16 v4, v0, v2, v0
+; GFX9-GISEL-NEXT: v_pk_mad_u16 v5, v1, v3, v1
+; GFX9-GISEL-NEXT: v_pk_mul_lo_u16 v6, v4, v2
+; GFX9-GISEL-NEXT: v_pk_mul_lo_u16 v7, v5, v3
+; GFX9-GISEL-NEXT: v_pk_mad_u16 v0, v0, v2, 1 op_sel_hi:[1,1,0]
+; GFX9-GISEL-NEXT: v_pk_mad_u16 v1, v1, v3, 1 op_sel_hi:[1,1,0]
+; GFX9-GISEL-NEXT: v_pk_mad_u16 v2, v4, v2, 1 op_sel_hi:[1,1,0]
+; GFX9-GISEL-NEXT: v_pk_mad_u16 v3, v5, v3, 1 op_sel_hi:[1,1,0]
+; GFX9-GISEL-NEXT: v_pk_mul_lo_u16 v0, v6, v0
+; GFX9-GISEL-NEXT: v_pk_mul_lo_u16 v1, v7, v1
; GFX9-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v2
; GFX9-GISEL-NEXT: v_pk_mul_lo_u16 v1, v1, v3
-; GFX9-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v4
-; GFX9-GISEL-NEXT: v_pk_mul_lo_u16 v1, v1, v5
; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-SDAG-LABEL: clpeak_umad_pat_v4i16:
; GFX10-SDAG: ; %bb.0: ; %entry
; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_pk_add_u16 v1, v1, 1 op_sel_hi:[1,0]
; GFX10-SDAG-NEXT: v_pk_add_u16 v0, v0, 1 op_sel_hi:[1,0]
-; GFX10-SDAG-NEXT: v_pk_mul_lo_u16 v4, v1, v3
-; GFX10-SDAG-NEXT: v_pk_mul_lo_u16 v5, v0, v2
-; GFX10-SDAG-NEXT: v_pk_add_u16 v1, v4, v1
-; GFX10-SDAG-NEXT: v_pk_add_u16 v0, v5, v0
-; GFX10-SDAG-NEXT: v_pk_mul_lo_u16 v1, v1, v3
-; GFX10-SDAG-NEXT: v_pk_mul_lo_u16 v0, v0, v2
-; GFX10-SDAG-NEXT: v_pk_add_u16 v2, v5, 1 op_sel_hi:[1,0]
-; GFX10-SDAG-NEXT: v_pk_add_u16 v3, v4, 1 op_sel_hi:[1,0]
-; GFX10-SDAG-NEXT: v_pk_add_u16 v4, v1, 1 op_sel_hi:[1,0]
-; GFX10-SDAG-NEXT: v_pk_add_u16 v5, v0, 1 op_sel_hi:[1,0]
+; GFX10-SDAG-NEXT: v_pk_add_u16 v1, v1, 1 op_sel_hi:[1,0]
+; GFX10-SDAG-NEXT: v_pk_mad_u16 v4, v0, v2, v0
+; GFX10-SDAG-NEXT: v_pk_mad_u16 v5, v1, v3, v1
+; GFX10-SDAG-NEXT: v_pk_mad_u16 v0, v0, v2, 1 op_sel_hi:[1,1,0]
+; GFX10-SDAG-NEXT: v_pk_mad_u16 v1, v1, v3, 1 op_sel_hi:[1,1,0]
+; GFX10-SDAG-NEXT: v_pk_mul_lo_u16 v6, v4, v2
+; GFX10-SDAG-NEXT: v_pk_mul_lo_u16 v7, v5, v3
+; GFX10-SDAG-NEXT: v_pk_mad_u16 v3, v5, v3, 1 op_sel_hi:[1,1,0]
+; GFX10-SDAG-NEXT: v_pk_mad_u16 v2, v4, v2, 1 op_sel_hi:[1,1,0]
+; GFX10-SDAG-NEXT: v_pk_mul_lo_u16 v0, v6, v0
+; GFX10-SDAG-NEXT: v_pk_mul_lo_u16 v1, v7, v1
; GFX10-SDAG-NEXT: v_pk_mul_lo_u16 v0, v0, v2
; GFX10-SDAG-NEXT: v_pk_mul_lo_u16 v1, v1, v3
-; GFX10-SDAG-NEXT: v_pk_mul_lo_u16 v0, v0, v5
-; GFX10-SDAG-NEXT: v_pk_mul_lo_u16 v1, v1, v4
; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-GISEL-LABEL: clpeak_umad_pat_v4i16:
@@ -2193,47 +2131,41 @@ define <4 x i16> @clpeak_umad_pat_v4i16(<4 x i16> %x, <4 x i16> %y) {
; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-GISEL-NEXT: v_pk_add_u16 v0, v0, 1 op_sel_hi:[1,0]
; GFX10-GISEL-NEXT: v_pk_add_u16 v1, v1, 1 op_sel_hi:[1,0]
-; GFX10-GISEL-NEXT: v_pk_mul_lo_u16 v4, v0, v2
-; GFX10-GISEL-NEXT: v_pk_mul_lo_u16 v5, v1, v3
-; GFX10-GISEL-NEXT: v_pk_add_u16 v0, v4, v0
-; GFX10-GISEL-NEXT: v_pk_add_u16 v1, v5, v1
-; GFX10-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v2
-; GFX10-GISEL-NEXT: v_pk_mul_lo_u16 v1, v1, v3
-; GFX10-GISEL-NEXT: v_pk_add_u16 v2, v4, 1 op_sel_hi:[1,0]
-; GFX10-GISEL-NEXT: v_pk_add_u16 v3, v5, 1 op_sel_hi:[1,0]
-; GFX10-GISEL-NEXT: v_pk_add_u16 v4, v0, 1 op_sel_hi:[1,0]
-; GFX10-GISEL-NEXT: v_pk_add_u16 v5, v1, 1 op_sel_hi:[1,0]
+; GFX10-GISEL-NEXT: v_pk_mad_u16 v4, v0, v2, v0
+; GFX10-GISEL-NEXT: v_pk_mad_u16 v5, v1, v3, v1
+; GFX10-GISEL-NEXT: v_pk_mad_u16 v0, v0, v2, 1 op_sel_hi:[1,1,0]
+; GFX10-GISEL-NEXT: v_pk_mad_u16 v1, v1, v3, 1 op_sel_hi:[1,1,0]
+; GFX10-GISEL-NEXT: v_pk_mul_lo_u16 v6, v4, v2
+; GFX10-GISEL-NEXT: v_pk_mul_lo_u16 v7, v5, v3
+; GFX10-GISEL-NEXT: v_pk_mad_u16 v2, v4, v2, 1 op_sel_hi:[1,1,0]
+; GFX10-GISEL-NEXT: v_pk_mad_u16 v3, v5, v3, 1 op_sel_hi:[1,1,0]
+; GFX10-GISEL-NEXT: v_pk_mul_lo_u16 v0, v6, v0
+; GFX10-GISEL-NEXT: v_pk_mul_lo_u16 v1, v7, v1
; GFX10-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v2
; GFX10-GISEL-NEXT: v_pk_mul_lo_u16 v1, v1, v3
-; GFX10-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v4
-; GFX10-GISEL-NEXT: v_pk_mul_lo_u16 v1, v1, v5
; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-LABEL: clpeak_umad_pat_v4i16:
; GFX11-SDAG: ; %bb.0: ; %entry
; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-NEXT: v_pk_add_u16 v1, v1, 1 op_sel_hi:[1,0]
; GFX11-SDAG-NEXT: v_pk_add_u16 v0, v0, 1 op_sel_hi:[1,0]
+; GFX11-SDAG-NEXT: v_pk_add_u16 v1, v1, 1 op_sel_hi:[1,0]
; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v4, v1, v3
-; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v5, v0, v2
-; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-NEXT: v_pk_add_u16 v1, v4, v1
-; GFX11-SDAG-NEXT: v_pk_add_u16 v0, v5, v0
-; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v1, v1, v3
-; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v0, v0, v2
-; GFX11-SDAG-NEXT: v_pk_add_u16 v2, v5, 1 op_sel_hi:[1,0]
-; GFX11-SDAG-NEXT: v_pk_add_u16 v3, v4, 1 op_sel_hi:[1,0]
+; GFX11-SDAG-NEXT: v_pk_mad_u16 v4, v0, v2, v0
+; GFX11-SDAG-NEXT: v_pk_mad_u16 v5, v1, v3, v1
+; GFX11-SDAG-NEXT: v_pk_mad_u16 v0, v0, v2, 1 op_sel_hi:[1,1,0]
+; GFX11-SDAG-NEXT: v_pk_mad_u16 v1, v1, v3, 1 op_sel_hi:[1,1,0]
; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-SDAG-NEXT: v_pk_add_u16 v4, v1, 1 op_sel_hi:[1,0]
-; GFX11-SDAG-NEXT: v_pk_add_u16 v5, v0, 1 op_sel_hi:[1,0]
+; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v6, v4, v2
+; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v7, v5, v3
+; GFX11-SDAG-NEXT: v_pk_mad_u16 v3, v5, v3, 1 op_sel_hi:[1,1,0]
+; GFX11-SDAG-NEXT: v_pk_mad_u16 v2, v4, v2, 1 op_sel_hi:[1,1,0]
; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v0, v6, v0
+; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v1, v7, v1
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v0, v0, v2
; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v1, v1, v3
-; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v0, v0, v5
-; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v1, v1, v4
; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-GISEL-LABEL: clpeak_umad_pat_v4i16:
@@ -2242,25 +2174,21 @@ define <4 x i16> @clpeak_umad_pat_v4i16(<4 x i16> %x, <4 x i16> %y) {
; GFX11-GISEL-NEXT: v_pk_add_u16 v0, v0, 1 op_sel_hi:[1,0]
; GFX11-GISEL-NEXT: v_pk_add_u16 v1, v1, 1 op_sel_hi:[1,0]
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v4, v0, v2
-; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v5, v1, v3
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-GISEL-NEXT: v_pk_add_u16 v0, v4, v0
-; GFX11-GISEL-NEXT: v_pk_add_u16 v1, v5, v1
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v2
-; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v1, v1, v3
-; GFX11-GISEL-NEXT: v_pk_add_u16 v2, v4, 1 op_sel_hi:[1,0]
-; GFX11-GISEL-NEXT: v_pk_add_u16 v3, v5, 1 op_sel_hi:[1,0]
+; GFX11-GISEL-NEXT: v_pk_mad_u16 v4, v0, v2, v0
+; GFX11-GISEL-NEXT: v_pk_mad_u16 v5, v1, v3, v1
+; GFX11-GISEL-NEXT: v_pk_mad_u16 v0, v0, v2, 1 op_sel_hi:[1,1,0]
+; GFX11-GISEL-NEXT: v_pk_mad_u16 v1, v1, v3, 1 op_sel_hi:[1,1,0]
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-GISEL-NEXT: v_pk_add_u16 v4, v0, 1 op_sel_hi:[1,0]
-; GFX11-GISEL-NEXT: v_pk_add_u16 v5, v1, 1 op_sel_hi:[1,0]
+; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v6, v4, v2
+; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v7, v5, v3
+; GFX11-GISEL-NEXT: v_pk_mad_u16 v2, v4, v2, 1 op_sel_hi:[1,1,0]
+; GFX11-GISEL-NEXT: v_pk_mad_u16 v3, v5, v3, 1 op_sel_hi:[1,1,0]
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v0, v6, v0
+; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v1, v7, v1
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v2
; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v1, v1, v3
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v4
-; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v1, v1, v5
; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
entry:
%y18 = add <4 x i16> %x, <i16 1, i16 1, i16 1, i16 1>
@@ -7192,69 +7120,58 @@ define <2 x i16> @clpeak_imad_pat_v2i16_x2(<2 x i16> %x, <2 x i16> %y) {
; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_pk_add_u16 v0, v0, 1 op_sel_hi:[1,0]
-; GFX9-NEXT: v_pk_mul_lo_u16 v2, v0, v1
-; GFX9-NEXT: v_pk_add_u16 v0, v2, v0
-; GFX9-NEXT: v_pk_mul_lo_u16 v0, v0, v1
-; GFX9-NEXT: v_pk_add_u16 v1, v2, 1 op_sel_hi:[1,0]
-; GFX9-NEXT: v_pk_mul_lo_u16 v2, v0, v1
-; GFX9-NEXT: v_pk_add_u16 v1, v2, v1
-; GFX9-NEXT: v_pk_mul_lo_u16 v0, v1, v0
-; GFX9-NEXT: v_pk_add_u16 v1, v2, 1 op_sel_hi:[1,0]
-; GFX9-NEXT: v_pk_mul_lo_u16 v2, v0, v1
-; GFX9-NEXT: v_pk_add_u16 v1, v2, v1
-; GFX9-NEXT: v_pk_mul_lo_u16 v0, v1, v0
-; GFX9-NEXT: v_pk_add_u16 v1, v2, 1 op_sel_hi:[1,0]
-; GFX9-NEXT: v_pk_add_u16 v2, v0, 1 op_sel_hi:[1,0]
+; GFX9-NEXT: v_pk_mad_u16 v2, v0, v1, v0
+; GFX9-NEXT: v_pk_mul_lo_u16 v2, v2, v1
+; GFX9-NEXT: v_pk_mad_u16 v0, v0, v1, 1 op_sel_hi:[1,1,0]
+; GFX9-NEXT: v_pk_mad_u16 v1, v2, v0, v0
+; GFX9-NEXT: v_pk_mul_lo_u16 v1, v1, v2
+; GFX9-NEXT: v_pk_mad_u16 v0, v2, v0, 1 op_sel_hi:[1,1,0]
+; GFX9-NEXT: v_pk_mad_u16 v2, v1, v0, v0
+; GFX9-NEXT: v_pk_mul_lo_u16 v3, v2, v1
+; GFX9-NEXT: v_pk_mad_u16 v0, v1, v0, 1 op_sel_hi:[1,1,0]
+; GFX9-NEXT: v_pk_mad_u16 v1, v2, v1, 1 op_sel_hi:[1,1,0]
+; GFX9-NEXT: v_pk_mul_lo_u16 v0, v3, v0
; GFX9-NEXT: v_pk_mul_lo_u16 v0, v0, v1
-; GFX9-NEXT: v_pk_mul_lo_u16 v0, v0, v2
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: clpeak_imad_pat_v2i16_x2:
; GFX10: ; %bb.0: ; %entry
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_pk_add_u16 v0, v0, 1 op_sel_hi:[1,0]
-; GFX10-NEXT: v_pk_mul_lo_u16 v2, v0, v1
-; GFX10-NEXT: v_pk_add_u16 v0, v2, v0
-; GFX10-NEXT: v_pk_add_u16 v2, v2, 1 op_sel_hi:[1,0]
+; GFX10-NEXT: v_pk_mad_u16 v2, v0, v1, v0
+; GFX10-NEXT: v_pk_mad_u16 v0, v0, v1, 1 op_sel_hi:[1,1,0]
+; GFX10-NEXT: v_pk_mul_lo_u16 v1, v2, v1
+; GFX10-NEXT: v_pk_mad_u16 v2, v1, v0, v0
+; GFX10-NEXT: v_pk_mad_u16 v0, v1, v0, 1 op_sel_hi:[1,1,0]
+; GFX10-NEXT: v_pk_mul_lo_u16 v1, v2, v1
+; GFX10-NEXT: v_pk_mad_u16 v2, v1, v0, v0
+; GFX10-NEXT: v_pk_mad_u16 v0, v1, v0, 1 op_sel_hi:[1,1,0]
+; GFX10-NEXT: v_pk_mul_lo_u16 v3, v2, v1
+; GFX10-NEXT: v_pk_mad_u16 v1, v2, v1, 1 op_sel_hi:[1,1,0]
+; GFX10-NEXT: v_pk_mul_lo_u16 v0, v3, v0
; GFX10-NEXT: v_pk_mul_lo_u16 v0, v0, v1
-; GFX10-NEXT: v_pk_mul_lo_u16 v1, v0, v2
-; GFX10-NEXT: v_pk_add_u16 v2, v1, v2
-; GFX10-NEXT: v_pk_add_u16 v1, v1, 1 op_sel_hi:[1,0]
-; GFX10-NEXT: v_pk_mul_lo_u16 v0, v2, v0
-; GFX10-NEXT: v_pk_mul_lo_u16 v2, v0, v1
-; GFX10-NEXT: v_pk_add_u16 v1, v2, v1
-; GFX10-NEXT: v_pk_mul_lo_u16 v0, v1, v0
-; GFX10-NEXT: v_pk_add_u16 v1, v2, 1 op_sel_hi:[1,0]
-; GFX10-NEXT: v_pk_add_u16 v2, v0, 1 op_sel_hi:[1,0]
-; GFX10-NEXT: v_pk_mul_lo_u16 v0, v0, v1
-; GFX10-NEXT: v_pk_mul_lo_u16 v0, v0, v2
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: clpeak_imad_pat_v2i16_x2:
; GFX11: ; %bb.0: ; %entry
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_pk_add_u16 v0, v0, 1 op_sel_hi:[1,0]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_pk_mul_lo_u16 v2, v0, v1
-; GFX11-NEXT: v_pk_add_u16 v0, v2, v0
-; GFX11-NEXT: v_pk_add_u16 v2, v2, 1 op_sel_hi:[1,0]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_pk_mul_lo_u16 v0, v0, v1
-; GFX11-NEXT: v_pk_mul_lo_u16 v1, v0, v2
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_pk_add_u16 v2, v1, v2
-; GFX11-NEXT: v_pk_add_u16 v1, v1, 1 op_sel_hi:[1,0]
-; GFX11-NEXT: v_pk_mul_lo_u16 v0, v2, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_pk_mul_lo_u16 v2, v0, v1
-; GFX11-NEXT: v_pk_add_u16 v1, v2, v1
+; GFX11-NEXT: v_pk_mad_u16 v2, v0, v1, v0
+; GFX11-NEXT: v_pk_mad_u16 v0, v0, v1, 1 op_sel_hi:[1,1,0]
+; GFX11-NEXT: v_pk_mul_lo_u16 v1, v2, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_pk_mad_u16 v2, v1, v0, v0
+; GFX11-NEXT: v_pk_mad_u16 v0, v1, v0, 1 op_sel_hi:[1,1,0]
+; GFX11-NEXT: v_pk_mul_lo_u16 v1, v2, v1
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_pk_mul_lo_u16 v0, v1, v0
-; GFX11-NEXT: v_pk_add_u16 v1, v2, 1 op_sel_hi:[1,0]
-; GFX11-NEXT: v_pk_add_u16 v2, v0, 1 op_sel_hi:[1,0]
+; GFX11-NEXT: v_pk_mad_u16 v2, v1, v0, v0
+; GFX11-NEXT: v_pk_mad_u16 v0, v1, v0, 1 op_sel_hi:[1,1,0]
+; GFX11-NEXT: v_pk_mul_lo_u16 v3, v2, v1
+; GFX11-NEXT: v_pk_mad_u16 v1, v2, v1, 1 op_sel_hi:[1,1,0]
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_pk_mul_lo_u16 v0, v3, v0
; GFX11-NEXT: v_pk_mul_lo_u16 v0, v0, v1
-; GFX11-NEXT: v_pk_mul_lo_u16 v0, v0, v2
; GFX11-NEXT: s_setpc_b64 s[30:31]
entry:
%y38 = add <2 x i16> %x, <i16 1, i16 1>
@@ -7500,69 +7417,58 @@ define <2 x i16> @clpeak_umad_pat_v2i16_x2(<2 x i16> %x, <2 x i16> %y) {
; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_pk_add_u16 v0, v0, 1 op_sel_hi:[1,0]
-; GFX9-NEXT: v_pk_mul_lo_u16 v2, v0, v1
-; GFX9-NEXT: v_pk_add_u16 v0, v2, v0
+; GFX9-NEXT: v_pk_mad_u16 v2, v0, v1, v0
+; GFX9-NEXT: v_pk_mul_lo_u16 v2, v2, v1
+; GFX9-NEXT: v_pk_mad_u16 v0, v0, v1, 1 op_sel_hi:[1,1,0]
+; GFX9-NEXT: v_pk_mad_u16 v1, v2, v0, v0
+; GFX9-NEXT: v_pk_mul_lo_u16 v1, v1, v2
+; GFX9-NEXT: v_pk_mad_u16 v0, v2, v0, 1 op_sel_hi:[1,1,0]
+; GFX9-NEXT: v_pk_mad_u16 v2, v1, v0, v0
+; GFX9-NEXT: v_pk_mul_lo_u16 v3, v2, v1
+; GFX9-NEXT: v_pk_mad_u16 v0, v1, v0, 1 op_sel_hi:[1,1,0]
+; GFX9-NEXT: v_pk_mad_u16 v1, v2, v1, 1 op_sel_hi:[1,1,0]
+; GFX9-NEXT: v_pk_mul_lo_u16 v0, v3, v0
; GFX9-NEXT: v_pk_mul_lo_u16 v0, v0, v1
-; GFX9-NEXT: v_pk_add_u16 v1, v2, 1 op_sel_hi:[1,0]
-; GFX9-NEXT: v_pk_mul_lo_u16 v2, v0, v1
-; GFX9-NEXT: v_pk_add_u16 v1, v2, v1
-; GFX9-NEXT: v_pk_mul_lo_u16 v0, v1, v0
-; GFX9-NEXT: v_pk_add_u16 v1, v2, 1 op_sel_hi:[1,0]
-; GFX9-NEXT: v_pk_mul_lo_u16 v2, v0, v1
-; GFX9-NEXT: v_pk_add_u16 v1, v2, v1
-; GFX9-NEXT: v_pk_mul_lo_u16 v0, v1, v0
-; GFX9-NEXT: v_pk_add_u16 v1, v2, 1 op_sel_hi:[1,0]
-; GFX9-NEXT: v_pk_add_u16 v2, v0, 1 op_sel_hi:[1,0]
-; GFX9-NEXT: v_pk_mul_lo_u16 v0, v0, v1
-; GFX9-NEXT: v_pk_mul_lo_u16 v0, v0, v2
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: clpeak_umad_pat_v2i16_x2:
; GFX10: ; %bb.0: ; %entry
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_pk_add_u16 v0, v0, 1 op_sel_hi:[1,0]
-; GFX10-NEXT: v_pk_mul_lo_u16 v2, v0, v1
-; GFX10-NEXT: v_pk_add_u16 v0, v2, v0
-; GFX10-NEXT: v_pk_add_u16 v2, v2, 1 op_sel_hi:[1,0]
-; GFX10-NEXT: v_pk_mul_lo_u16 v0, v0, v1
-; GFX10-NEXT: v_pk_mul_lo_u16 v1, v0, v2
-; GFX10-NEXT: v_pk_add_u16 v2, v1, v2
-; GFX10-NEXT: v_pk_add_u16 v1, v1, 1 op_sel_hi:[1,0]
-; GFX10-NEXT: v_pk_mul_lo_u16 v0, v2, v0
-; GFX10-NEXT: v_pk_mul_lo_u16 v2, v0, v1
-; GFX10-NEXT: v_pk_add_u16 v1, v2, v1
-; GFX10-NEXT: v_pk_mul_lo_u16 v0, v1, v0
-; GFX10-NEXT: v_pk_add_u16 v1, v2, 1 op_sel_hi:[1,0]
-; GFX10-NEXT: v_pk_add_u16 v2, v0, 1 op_sel_hi:[1,0]
+; GFX10-NEXT: v_pk_mad_u16 v2, v0, v1, v0
+; GFX10-NEXT: v_pk_mad_u16 v0, v0, v1, 1 op_sel_hi:[1,1,0]
+; GFX10-NEXT: v_pk_mul_lo_u16 v1, v2, v1
+; GFX10-NEXT: v_pk_mad_u16 v2, v1, v0, v0
+; GFX10-NEXT: v_pk_mad_u16 v0, v1, v0, 1 op_sel_hi:[1,1,0]
+; GFX10-NEXT: v_pk_mul_lo_u16 v1, v2, v1
+; GFX10-NEXT: v_pk_mad_u16 v2, v1, v0, v0
+; GFX10-NEXT: v_pk_mad_u16 v0, v1, v0, 1 op_sel_hi:[1,1,0]
+; GFX10-NEXT: v_pk_mul_lo_u16 v3, v2, v1
+; GFX10-NEXT: v_pk_mad_u16 v1, v2, v1, 1 op_sel_hi:[1,1,0]
+; GFX10-NEXT: v_pk_mul_lo_u16 v0, v3, v0
; GFX10-NEXT: v_pk_mul_lo_u16 v0, v0, v1
-; GFX10-NEXT: v_pk_mul_lo_u16 v0, v0, v2
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: clpeak_umad_pat_v2i16_x2:
; GFX11: ; %bb.0: ; %entry
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_pk_add_u16 v0, v0, 1 op_sel_hi:[1,0]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_pk_mul_lo_u16 v2, v0, v1
-; GFX11-NEXT: v_pk_add_u16 v0, v2, v0
-; GFX11-NEXT: v_pk_add_u16 v2, v2, 1 op_sel_hi:[1,0]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_pk_mul_lo_u16 v0, v0, v1
-; GFX11-NEXT: v_pk_mul_lo_u16 v1, v0, v2
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_pk_add_u16 v2, v1, v2
-; GFX11-NEXT: v_pk_add_u16 v1, v1, 1 op_sel_hi:[1,0]
-; GFX11-NEXT: v_pk_mul_lo_u16 v0, v2, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_pk_mul_lo_u16 v2, v0, v1
-; GFX11-NEXT: v_pk_add_u16 v1, v2, v1
+; GFX11-NEXT: v_pk_mad_u16 v2, v0, v1, v0
+; GFX11-NEXT: v_pk_mad_u16 v0, v0, v1, 1 op_sel_hi:[1,1,0]
+; GFX11-NEXT: v_pk_mul_lo_u16 v1, v2, v1
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_pk_mul_lo_u16 v0, v1, v0
-; GFX11-NEXT: v_pk_add_u16 v1, v2, 1 op_sel_hi:[1,0]
-; GFX11-NEXT: v_pk_add_u16 v2, v0, 1 op_sel_hi:[1,0]
+; GFX11-NEXT: v_pk_mad_u16 v2, v1, v0, v0
+; GFX11-NEXT: v_pk_mad_u16 v0, v1, v0, 1 op_sel_hi:[1,1,0]
+; GFX11-NEXT: v_pk_mul_lo_u16 v1, v2, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_pk_mad_u16 v2, v1, v0, v0
+; GFX11-NEXT: v_pk_mad_u16 v0, v1, v0, 1 op_sel_hi:[1,1,0]
+; GFX11-NEXT: v_pk_mul_lo_u16 v3, v2, v1
+; GFX11-NEXT: v_pk_mad_u16 v1, v2, v1, 1 op_sel_hi:[1,1,0]
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_pk_mul_lo_u16 v0, v3, v0
; GFX11-NEXT: v_pk_mul_lo_u16 v0, v0, v1
-; GFX11-NEXT: v_pk_mul_lo_u16 v0, v0, v2
; GFX11-NEXT: s_setpc_b64 s[30:31]
entry:
%y38 = add <2 x i16> %x, <i16 1, i16 1>
@@ -7955,26 +7861,26 @@ define <4 x i16> @multi_use_mul_mad_v2i16_var(<2 x i16> %x, <2 x i16> %y, <2 x i
; GFX9-LABEL: multi_use_mul_mad_v2i16_var:
; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_pk_mul_lo_u16 v1, v0, v1
-; GFX9-NEXT: v_pk_add_u16 v0, v1, v2
-; GFX9-NEXT: v_pk_add_u16 v1, v1, v3
+; GFX9-NEXT: v_pk_mad_u16 v2, v0, v1, v2
+; GFX9-NEXT: v_pk_mad_u16 v1, v0, v1, v3
+; GFX9-NEXT: v_mov_b32_e32 v0, v2
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: multi_use_mul_mad_v2i16_var:
; GFX10: ; %bb.0: ; %entry
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_pk_mul_lo_u16 v1, v0, v1
-; GFX10-NEXT: v_pk_add_u16 v0, v1, v2
-; GFX10-NEXT: v_pk_add_u16 v1, v1, v3
+; GFX10-NEXT: v_pk_mad_u16 v2, v0, v1, v2
+; GFX10-NEXT: v_pk_mad_u16 v1, v0, v1, v3
+; GFX10-NEXT: v_mov_b32_e32 v0, v2
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: multi_use_mul_mad_v2i16_var:
; GFX11: ; %bb.0: ; %entry
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_pk_mul_lo_u16 v1, v0, v1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_pk_add_u16 v0, v1, v2
-; GFX11-NEXT: v_pk_add_u16 v1, v1, v3
+; GFX11-NEXT: v_pk_mad_u16 v2, v0, v1, v2
+; GFX11-NEXT: v_pk_mad_u16 v1, v0, v1, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-NEXT: v_mov_b32_e32 v0, v2
; GFX11-NEXT: s_setpc_b64 s[30:31]
entry:
%mul = mul <2 x i16> %x, %y
@@ -8067,28 +7973,27 @@ define <2 x i16> @other_use_mul_mad_v2i16_var(<2 x i16> %x, <2 x i16> %y, <2 x i
; GFX9-LABEL: other_use_mul_mad_v2i16_var:
; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_pk_mul_lo_u16 v1, v0, v1
-; GFX9-NEXT: v_pk_add_u16 v0, v1, v2
-; GFX9-NEXT: ds_write_b32 v3, v1
+; GFX9-NEXT: v_pk_mul_lo_u16 v4, v0, v1
+; GFX9-NEXT: v_pk_mad_u16 v0, v0, v1, v2
+; GFX9-NEXT: ds_write_b32 v3, v4
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: other_use_mul_mad_v2i16_var:
; GFX10: ; %bb.0: ; %entry
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_pk_mul_lo_u16 v1, v0, v1
-; GFX10-NEXT: v_pk_add_u16 v0, v1, v2
-; GFX10-NEXT: ds_write_b32 v3, v1
+; GFX10-NEXT: v_pk_mul_lo_u16 v4, v0, v1
+; GFX10-NEXT: v_pk_mad_u16 v0, v0, v1, v2
+; GFX10-NEXT: ds_write_b32 v3, v4
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: other_use_mul_mad_v2i16_var:
; GFX11: ; %bb.0: ; %entry
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_pk_mul_lo_u16 v1, v0, v1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_pk_add_u16 v0, v1, v2
-; GFX11-NEXT: ds_store_b32 v3, v1
+; GFX11-NEXT: v_pk_mul_lo_u16 v4, v0, v1
+; GFX11-NEXT: v_pk_mad_u16 v0, v0, v1, v2
+; GFX11-NEXT: ds_store_b32 v3, v4
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_setpc_b64 s[30:31]
entry:
diff --git a/llvm/test/CodeGen/AMDGPU/reassoc-mul-add-1-to-mad.ll b/llvm/test/CodeGen/AMDGPU/reassoc-mul-add-1-to-mad.ll
index 77e1694dbe7e1..b62d6ee59a854 100644
--- a/llvm/test/CodeGen/AMDGPU/reassoc-mul-add-1-to-mad.ll
+++ b/llvm/test/CodeGen/AMDGPU/reassoc-mul-add-1-to-mad.ll
@@ -1718,15 +1718,13 @@ define <2 x i16> @v_mul_add_x_v2i16(<2 x i16> %x, <2 x i16> %y) {
; GFX9-LABEL: v_mul_add_x_v2i16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_pk_mul_lo_u16 v1, v0, v1
-; GFX9-NEXT: v_pk_add_u16 v0, v0, v1
+; GFX9-NEXT: v_pk_mad_u16 v0, v0, v1, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_mul_add_x_v2i16:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_pk_mul_lo_u16 v1, v0, v1
-; GFX10-NEXT: v_pk_add_u16 v0, v0, v1
+; GFX10-NEXT: v_pk_mad_u16 v0, v0, v1, v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
%mul = mul <2 x i16> %x, %y
%add = add <2 x i16> %x, %mul
@@ -2718,15 +2716,13 @@ define <2 x i16> @v_mul_9_add_52_v2i16(<2 x i16> %arg) {
; GFX9-LABEL: v_mul_9_add_52_v2i16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_pk_mul_lo_u16 v0, v0, 9 op_sel_hi:[1,0]
-; GFX9-NEXT: v_pk_add_u16 v0, v0, 52 op_sel_hi:[1,0]
+; GFX9-NEXT: v_pk_mad_u16 v0, v0, 9, 52 op_sel_hi:[1,0,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_mul_9_add_52_v2i16:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_pk_mul_lo_u16 v0, v0, 9 op_sel_hi:[1,0]
-; GFX10-NEXT: v_pk_add_u16 v0, v0, 52 op_sel_hi:[1,0]
+; GFX10-NEXT: v_pk_mad_u16 v0, v0, 9, 52 op_sel_hi:[1,0,0]
; GFX10-NEXT: s_setpc_b64 s[30:31]
%mul = mul <2 x i16> %arg, <i16 9, i16 9>
%add = add <2 x i16> %mul, <i16 52, i16 52>
@@ -2952,15 +2948,13 @@ define <2 x i16> @v_mul_5_add_1_v2i16(<2 x i16> %arg) {
; GFX9-LABEL: v_mul_5_add_1_v2i16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_pk_mul_lo_u16 v0, v0, 5 op_sel_hi:[1,0]
-; GFX9-NEXT: v_pk_add_u16 v0, v0, 1 op_sel_hi:[1,0]
+; GFX9-NEXT: v_pk_mad_u16 v0, v0, 5, 1 op_sel_hi:[1,0,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_mul_5_add_1_v2i16:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_pk_mul_lo_u16 v0, v0, 5 op_sel_hi:[1,0]
-; GFX10-NEXT: v_pk_add_u16 v0, v0, 1 op_sel_hi:[1,0]
+; GFX10-NEXT: v_pk_mad_u16 v0, v0, 5, 1 op_sel_hi:[1,0,0]
; GFX10-NEXT: s_setpc_b64 s[30:31]
%mul = mul <2 x i16> %arg, <i16 5, i16 5>
%add = add <2 x i16> %mul, <i16 1, i16 1>
@@ -2999,16 +2993,15 @@ define <2 x i16> @v_mul_284_add_82_v2i16(<2 x i16> %arg) {
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_movk_i32 s4, 0x11c
-; GFX9-NEXT: v_pk_mul_lo_u16 v0, v0, s4 op_sel_hi:[1,0]
-; GFX9-NEXT: s_movk_i32 s4, 0x52
-; GFX9-NEXT: v_pk_add_u16 v0, v0, s4 op_sel_hi:[1,0]
+; GFX9-NEXT: v_mov_b32_e32 v1, 0x52
+; GFX9-NEXT: v_pk_mad_u16 v0, v0, s4, v1 op_sel_hi:[1,0,0]
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_mul_284_add_82_v2i16:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_pk_mul_lo_u16 v0, 0x11c, v0 op_sel_hi:[0,1]
-; GFX10-NEXT: v_pk_add_u16 v0, 0x52, v0 op_sel_hi:[0,1]
+; GFX10-NEXT: s_movk_i32 s4, 0x11c
+; GFX10-NEXT: v_pk_mad_u16 v0, v0, s4, 0x52 op_sel_hi:[1,0,0]
; GFX10-NEXT: s_setpc_b64 s[30:31]
%mul = mul <2 x i16> %arg, <i16 284, i16 284>
%add = add <2 x i16> %mul, <i16 82, i16 82>
More information about the llvm-commits
mailing list