[llvm] [AMDGPU][True16][MC] VOP3 profile in True16 format (PR #109031)
Brox Chen via llvm-commits
llvm-commits at lists.llvm.org
Wed Sep 18 08:09:16 PDT 2024
https://github.com/broxigarchen updated https://github.com/llvm/llvm-project/pull/109031
>From cce56efc68653b615fca2f9d51a75942402c0d94 Mon Sep 17 00:00:00 2001
From: guochen2 <guochen2 at amd.com>
Date: Tue, 17 Sep 2024 15:06:04 -0400
Subject: [PATCH] [AMDGPU][True16][MC] VOP3 profile in True16 format
---
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 35 +-
llvm/lib/Target/AMDGPU/SIInstrInfo.td | 1 +
llvm/lib/Target/AMDGPU/SIInstructions.td | 48 +-
llvm/lib/Target/AMDGPU/VOP2Instructions.td | 4 +-
llvm/lib/Target/AMDGPU/VOP3Instructions.td | 43 +-
llvm/lib/Target/AMDGPU/VOPInstructions.td | 288 ++++++++++--
llvm/test/CodeGen/AMDGPU/bf16.ll | 502 +++++++++++----------
7 files changed, 604 insertions(+), 317 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index e4a679f6a3ef8f..7752acd0925502 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -5441,8 +5441,12 @@ unsigned SIInstrInfo::getVALUOp(const MachineInstr &MI) const {
case AMDGPU::S_SUB_F16: return AMDGPU::V_SUB_F16_fake16_e64;
case AMDGPU::S_MIN_F16: return AMDGPU::V_MIN_F16_fake16_e64;
case AMDGPU::S_MAX_F16: return AMDGPU::V_MAX_F16_fake16_e64;
- case AMDGPU::S_MINIMUM_F16: return AMDGPU::V_MINIMUM_F16_e64;
- case AMDGPU::S_MAXIMUM_F16: return AMDGPU::V_MAXIMUM_F16_e64;
+ case AMDGPU::S_MINIMUM_F16:
+ return ST.useRealTrue16Insts() ? AMDGPU::V_MINIMUM_F16_t16_e64
+ : AMDGPU::V_MINIMUM_F16_fake16_e64;
+ case AMDGPU::S_MAXIMUM_F16:
+ return ST.useRealTrue16Insts() ? AMDGPU::V_MAXIMUM_F16_t16_e64
+ : AMDGPU::V_MAXIMUM_F16_fake16_e64;
case AMDGPU::S_MUL_F16: return AMDGPU::V_MUL_F16_fake16_e64;
case AMDGPU::S_CVT_PK_RTZ_F16_F32: return AMDGPU::V_CVT_PKRTZ_F16_F32_e64;
case AMDGPU::S_FMAC_F32: return AMDGPU::V_FMAC_F32_e64;
@@ -7336,9 +7340,7 @@ void SIInstrInfo::moveToVALUImpl(SIInstrWorklist &Worklist,
return;
}
case AMDGPU::S_MINIMUM_F32:
- case AMDGPU::S_MAXIMUM_F32:
- case AMDGPU::S_MINIMUM_F16:
- case AMDGPU::S_MAXIMUM_F16: {
+ case AMDGPU::S_MAXIMUM_F32: {
const DebugLoc &DL = Inst.getDebugLoc();
Register NewDst = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
MachineInstr *NewInstr = BuildMI(*MBB, Inst, DL, get(NewOpcode), NewDst)
@@ -7349,7 +7351,28 @@ void SIInstrInfo::moveToVALUImpl(SIInstrWorklist &Worklist,
.addImm(0) // clamp
.addImm(0); // omod
MRI.replaceRegWith(Inst.getOperand(0).getReg(), NewDst);
-
+ legalizeOperands(*NewInstr, MDT);
+ addUsersToMoveToVALUWorklist(NewDst, MRI, Worklist);
+ Inst.eraseFromParent();
+ return;
+ }
+ case AMDGPU::S_MINIMUM_F16:
+ case AMDGPU::S_MAXIMUM_F16: {
+ const DebugLoc &DL = Inst.getDebugLoc();
+ Register NewDst;
+ if (ST.useRealTrue16Insts())
+ NewDst = MRI.createVirtualRegister(&AMDGPU::VGPR_16RegClass);
+ else
+ NewDst = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
+ MachineInstr *NewInstr = BuildMI(*MBB, Inst, DL, get(NewOpcode), NewDst)
+ .addImm(0) // src0_modifiers
+ .add(Inst.getOperand(1))
+ .addImm(0) // src1_modifiers
+ .add(Inst.getOperand(2))
+ .addImm(0) // clamp
+ .addImm(0) // omod
+ .addImm(0); // opsel0
+ MRI.replaceRegWith(Inst.getOperand(0).getReg(), NewDst);
legalizeOperands(*NewInstr, MDT);
addUsersToMoveToVALUWorklist(NewDst, MRI, Worklist);
Inst.eraseFromParent();
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.td b/llvm/lib/Target/AMDGPU/SIInstrInfo.td
index c016be2fc6c0fb..9c9653ce4747f3 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.td
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.td
@@ -2735,6 +2735,7 @@ def VOP_F32_F32_F16_F16 : VOPProfile <[f32, f32, f16, f16]>;
def VOP_F32_F32_F32_F32 : VOPProfile <[f32, f32, f32, f32]>;
def VOP_F64_F64_F64_F64 : VOPProfile <[f64, f64, f64, f64]>;
def VOP_I32_I32_I32_I32 : VOPProfile <[i32, i32, i32, i32]>;
+def VOP_I32_I32_I32_I16 : VOPProfile <[i32, i32, i32, i16]>;
def VOP_I64_I32_I32_I64 : VOPProfile <[i64, i32, i32, i64]>;
def VOP_I32_F32_I32_I32 : VOPProfile <[i32, f32, i32, i32]>;
def VOP_I64_I64_I32_I64 : VOPProfile <[i64, i64, i32, i64]>;
diff --git a/llvm/lib/Target/AMDGPU/SIInstructions.td b/llvm/lib/Target/AMDGPU/SIInstructions.td
index 9afb29d95abd7d..5a94ee7025e43f 100644
--- a/llvm/lib/Target/AMDGPU/SIInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SIInstructions.td
@@ -2444,6 +2444,7 @@ def : AMDGPUPat <
$src1), sub1)
>;
+let True16Predicate = NotHasTrue16BitInsts in {
def : ROTRPattern <V_ALIGNBIT_B32_e64>;
def : GCNPat<(i32 (trunc (srl i64:$src0, (and i32:$src1, (i32 31))))),
@@ -2453,6 +2454,30 @@ def : GCNPat<(i32 (trunc (srl i64:$src0, (and i32:$src1, (i32 31))))),
def : GCNPat<(i32 (trunc (srl i64:$src0, (i32 ShiftAmt32Imm:$src1)))),
(V_ALIGNBIT_B32_e64 (i32 (EXTRACT_SUBREG (i64 $src0), sub1)),
(i32 (EXTRACT_SUBREG (i64 $src0), sub0)), $src1)>;
+} // end OtherPredicates = [NotHasTrue16BitInsts]
+
+let True16Predicate = UseFakeTrue16Insts in {
+def ROTRPattern_fake16 : GCNPat <
+ (rotr i32:$src0, i32:$src1),
+ (V_ALIGNBIT_B32_fake16_e64 /* src0_modifiers */ 0, $src0,
+ /* src1_modifiers */ 0, $src0,
+ /* src2_modifiers */ 0,
+ $src1, /* clamp */ 0, /* op_sel */ 0)
+>;
+def : GCNPat<(i32 (trunc (srl i64:$src0, (i32 ShiftAmt32Imm:$src1)))),
+ (V_ALIGNBIT_B32_fake16_e64 0, /* src0_modifiers */
+ (i32 (EXTRACT_SUBREG (i64 $src0), sub1)),
+ 0, /* src1_modifiers */
+ (i32 (EXTRACT_SUBREG (i64 $src0), sub0)),
+ 0, /* src2_modifiers */
+ $src1, /* clamp */ 0, /* op_sel */ 0)>;
+
+def : GCNPat<(fshr i32:$src0, i32:$src1, i32:$src2),
+ (V_ALIGNBIT_B32_fake16_e64 /* src0_modifiers */ 0, $src0,
+ /* src1_modifiers */ 0, $src1,
+ /* src2_modifiers */ 0,
+ $src2, /* clamp */ 0, /* op_sel */ 0)>;
+} // end OtherPredicates = [UseFakeTrue16Insts]
/********** ====================== **********/
/********** Indirect addressing **********/
@@ -2932,6 +2957,7 @@ def : GCNPat <
(i32 (EXTRACT_SUBREG $a, sub0))), (i32 1))
>;
+let True16Predicate = NotHasTrue16BitInsts in
def : GCNPat <
(i32 (bswap i32:$a)),
(V_BFI_B32_e64 (S_MOV_B32 (i32 0x00ff00ff)),
@@ -2941,6 +2967,7 @@ def : GCNPat <
// FIXME: This should have been narrowed to i32 during legalization.
// This pattern should also be skipped for GlobalISel
+let True16Predicate = NotHasTrue16BitInsts in
def : GCNPat <
(i64 (bswap i64:$a)),
(REG_SEQUENCE VReg_64,
@@ -3312,6 +3339,7 @@ def : GCNPat <
// Take the upper 16 bits from V[0] and the lower 16 bits from V[1]
// Special case, can use V_ALIGNBIT (always uses encoded literal)
+let True16Predicate = NotHasTrue16BitInsts in
def : GCNPat <
(vecTy (DivergentBinFrag<build_vector>
(Ty !if(!eq(Ty, i16),
@@ -3321,6 +3349,16 @@ def : GCNPat <
(V_ALIGNBIT_B32_e64 VGPR_32:$b, VGPR_32:$a, (i32 16))
>;
+let True16Predicate = UseFakeTrue16Insts in
+def : GCNPat <
+ (vecTy (DivergentBinFrag<build_vector>
+ (Ty !if(!eq(Ty, i16),
+ (Ty (trunc (srl VGPR_32:$a, (i32 16)))),
+ (Ty (bitconvert (i16 (trunc (srl VGPR_32:$a, (i32 16)))))))),
+ (Ty VGPR_32:$b))),
+ (V_ALIGNBIT_B32_fake16_e64 0, VGPR_32:$b, 0, VGPR_32:$a, 0, (i16 16), 0, 0)
+>;
+
// Take the upper 16 bits from each VGPR_32 and concat them
def : GCNPat <
(vecTy (DivergentBinFrag<build_vector>
@@ -3682,12 +3720,14 @@ defm : Int16Med3Pat<V_MED3_U16_e64, umin, umax>;
let OtherPredicates = [isGFX12Plus] in {
def : FPMinMaxPat<V_MINIMUMMAXIMUM_F32_e64, f32, DivergentBinFrag<fmaximum>, fminimum_oneuse>;
def : FPMinMaxPat<V_MAXIMUMMINIMUM_F32_e64, f32, DivergentBinFrag<fminimum>, fmaximum_oneuse>;
-def : FPMinMaxPat<V_MINIMUMMAXIMUM_F16_e64, f16, DivergentBinFrag<fmaximum>, fminimum_oneuse>;
-def : FPMinMaxPat<V_MAXIMUMMINIMUM_F16_e64, f16, DivergentBinFrag<fminimum>, fmaximum_oneuse>;
def : FPMinCanonMaxPat<V_MINIMUMMAXIMUM_F32_e64, f32, DivergentBinFrag<fmaximum>, fminimum_oneuse>;
def : FPMinCanonMaxPat<V_MAXIMUMMINIMUM_F32_e64, f32, DivergentBinFrag<fminimum>, fmaximum_oneuse>;
-def : FPMinCanonMaxPat<V_MINIMUMMAXIMUM_F16_e64, f16, DivergentBinFrag<fmaximum>, fminimum_oneuse>;
-def : FPMinCanonMaxPat<V_MAXIMUMMINIMUM_F16_e64, f16, DivergentBinFrag<fminimum>, fmaximum_oneuse>;
+}
+let True16Predicate = UseFakeTrue16Insts, OtherPredicates = [isGFX12Plus] in {
+def : FPMinMaxPat<V_MINIMUMMAXIMUM_F16_fake16_e64, f16, DivergentBinFrag<fmaximum>, fminimum_oneuse>;
+def : FPMinMaxPat<V_MAXIMUMMINIMUM_F16_fake16_e64, f16, DivergentBinFrag<fminimum>, fmaximum_oneuse>;
+def : FPMinCanonMaxPat<V_MINIMUMMAXIMUM_F16_fake16_e64, f16, DivergentBinFrag<fmaximum>, fminimum_oneuse>;
+def : FPMinCanonMaxPat<V_MAXIMUMMINIMUM_F16_fake16_e64, f16, DivergentBinFrag<fminimum>, fmaximum_oneuse>;
}
// Convert a floating-point power of 2 to the integer exponent.
diff --git a/llvm/lib/Target/AMDGPU/VOP2Instructions.td b/llvm/lib/Target/AMDGPU/VOP2Instructions.td
index dd48607402eb0b..ce7dd0f705ed6f 100644
--- a/llvm/lib/Target/AMDGPU/VOP2Instructions.td
+++ b/llvm/lib/Target/AMDGPU/VOP2Instructions.td
@@ -1682,8 +1682,8 @@ multiclass VOP3Only_Realtriple_gfx11_gfx12<bits<10> op> :
VOP3Only_Realtriple<GFX11Gen, op>, VOP3Only_Realtriple<GFX12Gen, op>;
multiclass VOP3Only_Realtriple_t16_gfx11_gfx12<bits<10> op, string asmName> :
- VOP3Only_Realtriple_t16<GFX11Gen, op, asmName>,
- VOP3Only_Realtriple_t16<GFX12Gen, op, asmName>;
+ VOP3_Realtriple_t16_gfx11<op, asmName, NAME, "", 1>,
+ VOP3_Realtriple_t16_gfx12<op, asmName, NAME, "", 1>;
multiclass VOP3beOnly_Realtriple_gfx11_gfx12<bits<10> op> :
VOP3beOnly_Realtriple<GFX11Gen, op>, VOP3beOnly_Realtriple<GFX12Gen, op>;
diff --git a/llvm/lib/Target/AMDGPU/VOP3Instructions.td b/llvm/lib/Target/AMDGPU/VOP3Instructions.td
index 466114b95f9f90..b18437d612fe03 100644
--- a/llvm/lib/Target/AMDGPU/VOP3Instructions.td
+++ b/llvm/lib/Target/AMDGPU/VOP3Instructions.td
@@ -167,8 +167,8 @@ defm V_MUL_HI_I32 : VOP3Inst <"v_mul_hi_i32", V_MUL_PROF<VOP_I32_I32_I32>, mulhs
let SubtargetPredicate = isGFX12Plus, ReadsModeReg = 0 in {
defm V_MINIMUM_F32 : VOP3Inst <"v_minimum_f32", VOP3_Profile<VOP_F32_F32_F32>, DivergentBinFrag<fminimum>>;
defm V_MAXIMUM_F32 : VOP3Inst <"v_maximum_f32", VOP3_Profile<VOP_F32_F32_F32>, DivergentBinFrag<fmaximum>>;
-defm V_MINIMUM_F16 : VOP3Inst <"v_minimum_f16", VOP3_Profile<VOP_F16_F16_F16>, DivergentBinFrag<fminimum>>;
-defm V_MAXIMUM_F16 : VOP3Inst <"v_maximum_f16", VOP3_Profile<VOP_F16_F16_F16>, DivergentBinFrag<fmaximum>>;
+defm V_MINIMUM_F16 : VOP3Inst_t16 <"v_minimum_f16", VOP_F16_F16_F16, DivergentBinFrag<fminimum>>;
+defm V_MAXIMUM_F16 : VOP3Inst_t16 <"v_maximum_f16", VOP_F16_F16_F16, DivergentBinFrag<fmaximum>>;
let SchedRW = [WriteDoubleAdd] in {
defm V_MINIMUM_F64 : VOP3Inst <"v_minimum_f64", VOP3_Profile<VOP_F64_F64_F64>, fminimum>;
@@ -208,7 +208,11 @@ defm V_CUBEMA_F32 : VOP3Inst <"v_cubema_f32", VOP3_Profile<VOP_F32_F32_F32_F32>,
defm V_BFE_U32 : VOP3Inst <"v_bfe_u32", VOP3_Profile<VOP_I32_I32_I32_I32>, AMDGPUbfe_u32>;
defm V_BFE_I32 : VOP3Inst <"v_bfe_i32", VOP3_Profile<VOP_I32_I32_I32_I32>, AMDGPUbfe_i32>;
defm V_BFI_B32 : VOP3Inst <"v_bfi_b32", VOP3_Profile<VOP_I32_I32_I32_I32>, AMDGPUbfi>;
-defm V_ALIGNBIT_B32 : VOP3Inst <"v_alignbit_b32", VOP3_Profile<VOP_I32_I32_I32_I32>, fshr>;
+
+defm V_ALIGNBIT_B32 : VOP3Inst_t16_with_profiles <"v_alignbit_b32", VOP3_Profile<VOP_I32_I32_I32_I32>,
+ VOP3_Profile_True16<VOP_I32_I32_I32_I16, VOP3_OPSEL>, VOP3_Profile_Fake16<VOP_I32_I32_I32_I16, VOP3_OPSEL>,
+ fshr, null_frag>;
+
defm V_ALIGNBYTE_B32 : VOP3Inst <"v_alignbyte_b32", VOP3_Profile<VOP_I32_I32_I32_I32>, int_amdgcn_alignbyte>;
// XXX - No FPException seems suspect but manual doesn't say it does
@@ -636,8 +640,8 @@ defm V_MAX3_I16 : VOP3Inst <"v_max3_i16", VOP3_Profile<VOP_I16_I16_I16_I16, VOP3
defm V_MAX3_U16 : VOP3Inst <"v_max3_u16", VOP3_Profile<VOP_I16_I16_I16_I16, VOP3_OPSEL>, AMDGPUumax3>;
let SubtargetPredicate = isGFX12Plus, ReadsModeReg = 0 in {
- defm V_MINIMUM3_F16 : VOP3Inst <"v_minimum3_f16", VOP3_Profile<VOP_F16_F16_F16_F16, VOP3_OPSEL>, AMDGPUfminimum3>;
- defm V_MAXIMUM3_F16 : VOP3Inst <"v_maximum3_f16", VOP3_Profile<VOP_F16_F16_F16_F16, VOP3_OPSEL>, AMDGPUfmaximum3>;
+ defm V_MINIMUM3_F16 : VOP3Inst_t16 <"v_minimum3_f16", VOP_F16_F16_F16_F16, AMDGPUfminimum3>;
+ defm V_MAXIMUM3_F16 : VOP3Inst_t16 <"v_maximum3_f16", VOP_F16_F16_F16_F16, AMDGPUfmaximum3>;
} // End SubtargetPredicate = isGFX12Plus, ReadsModeReg = 0
defm V_ADD_I16 : VOP3Inst <"v_add_i16", VOP3_Profile<VOP_I16_I16_I16, VOP3_OPSEL>>;
@@ -940,8 +944,8 @@ let SubtargetPredicate = isGFX11Plus in {
let SubtargetPredicate = isGFX12Plus, ReadsModeReg = 0 in {
defm V_MAXIMUMMINIMUM_F32 : VOP3Inst<"v_maximumminimum_f32", VOP3_Profile<VOP_F32_F32_F32_F32>>;
defm V_MINIMUMMAXIMUM_F32 : VOP3Inst<"v_minimummaximum_f32", VOP3_Profile<VOP_F32_F32_F32_F32>>;
- defm V_MAXIMUMMINIMUM_F16 : VOP3Inst<"v_maximumminimum_f16", VOP3_Profile<VOP_F16_F16_F16_F16, VOP3_OPSEL>>;
- defm V_MINIMUMMAXIMUM_F16 : VOP3Inst<"v_minimummaximum_f16", VOP3_Profile<VOP_F16_F16_F16_F16, VOP3_OPSEL>>;
+ defm V_MAXIMUMMINIMUM_F16 : VOP3Inst_t16<"v_maximumminimum_f16", VOP_F16_F16_F16_F16>;
+ defm V_MINIMUMMAXIMUM_F16 : VOP3Inst_t16<"v_minimummaximum_f16", VOP_F16_F16_F16_F16>;
} // End SubtargetPredicate = isGFX12Plus, ReadsModeReg = 0
let OtherPredicates = [HasDot9Insts], IsDOT=1 in {
@@ -1046,8 +1050,8 @@ defm V_MIN3_NUM_F16 : VOP3_Realtriple_with_name_gfx12<0x22b, "V_MIN3_F16",
defm V_MAX3_NUM_F16 : VOP3_Realtriple_with_name_gfx12<0x22c, "V_MAX3_F16", "v_max3_num_f16">;
defm V_MINIMUM3_F32 : VOP3Only_Realtriple_gfx12<0x22d>;
defm V_MAXIMUM3_F32 : VOP3Only_Realtriple_gfx12<0x22e>;
-defm V_MINIMUM3_F16 : VOP3Only_Realtriple_t16_gfx12<0x22f>;
-defm V_MAXIMUM3_F16 : VOP3Only_Realtriple_t16_gfx12<0x230>;
+defm V_MINIMUM3_F16 : VOP3_Realtriple_t16_and_f16_gfx12<0x22f, "v_minimum3_f16">;
+defm V_MAXIMUM3_F16 : VOP3_Realtriple_t16_and_f16_gfx12<0x230, "v_maximum3_f16">;
defm V_MED3_NUM_F32 : VOP3_Realtriple_with_name_gfx12<0x231, "V_MED3_F32", "v_med3_num_f32">;
defm V_MED3_NUM_F16 : VOP3_Realtriple_with_name_gfx12<0x232, "V_MED3_F16", "v_med3_num_f16">;
defm V_MINMAX_NUM_F32 : VOP3_Realtriple_with_name_gfx12<0x268, "V_MINMAX_F32", "v_minmax_num_f32">;
@@ -1056,8 +1060,8 @@ defm V_MINMAX_NUM_F16 : VOP3_Realtriple_with_name_gfx12<0x26a, "V_MINMAX_F16
defm V_MAXMIN_NUM_F16 : VOP3_Realtriple_with_name_gfx12<0x26b, "V_MAXMIN_F16", "v_maxmin_num_f16">;
defm V_MINIMUMMAXIMUM_F32 : VOP3Only_Realtriple_gfx12<0x26c>;
defm V_MAXIMUMMINIMUM_F32 : VOP3Only_Realtriple_gfx12<0x26d>;
-defm V_MINIMUMMAXIMUM_F16 : VOP3Only_Realtriple_t16_gfx12<0x26e>;
-defm V_MAXIMUMMINIMUM_F16 : VOP3Only_Realtriple_t16_gfx12<0x26f>;
+defm V_MINIMUMMAXIMUM_F16 : VOP3_Realtriple_t16_and_f16_gfx12<0x26e, "v_minimummaximum_f16">;
+defm V_MAXIMUMMINIMUM_F16 : VOP3_Realtriple_t16_and_f16_gfx12<0x26f, "v_maximumminimum_f16">;
defm V_S_EXP_F32 : VOP3Only_Real_Base_gfx12<0x280>;
defm V_S_EXP_F16 : VOP3Only_Real_Base_gfx12<0x281>;
defm V_S_LOG_F32 : VOP3Only_Real_Base_gfx12<0x282>;
@@ -1074,8 +1078,8 @@ defm V_MINIMUM_F64 : VOP3Only_Real_Base_gfx12<0x341>;
defm V_MAXIMUM_F64 : VOP3Only_Real_Base_gfx12<0x342>;
defm V_MINIMUM_F32 : VOP3Only_Realtriple_gfx12<0x365>;
defm V_MAXIMUM_F32 : VOP3Only_Realtriple_gfx12<0x366>;
-defm V_MINIMUM_F16 : VOP3Only_Realtriple_t16_gfx12<0x367>;
-defm V_MAXIMUM_F16 : VOP3Only_Realtriple_t16_gfx12<0x368>;
+defm V_MINIMUM_F16 : VOP3_Realtriple_t16_and_f16_gfx12<0x367, "v_minimum_f16">;
+defm V_MAXIMUM_F16 : VOP3_Realtriple_t16_and_f16_gfx12<0x368, "v_maximum_f16">;
defm V_PERMLANE16_VAR_B32 : VOP3Only_Real_Base_gfx12<0x30f>;
defm V_PERMLANEX16_VAR_B32 : VOP3Only_Real_Base_gfx12<0x310>;
@@ -1108,6 +1112,17 @@ multiclass VOP3_Realtriple_with_name_gfx11_gfx12<bits<10> op, string opName,
multiclass VOP3Dot_Realtriple_gfx11_gfx12<bits<10> op> :
VOP3Dot_Realtriple<GFX11Gen, op>, VOP3Dot_Realtriple<GFX12Gen, op>;
+multiclass VOP3_Realtriple_t16_gfx11_gfx12<bits<10> op, string asmName, string opName = NAME,
+ string pseudo_mnemonic = "", bit isSingle = 0> :
+ VOP3_Realtriple_with_name<GFX11Gen, op, opName, asmName, pseudo_mnemonic, isSingle>,
+ VOP3_Realtriple_with_name<GFX12Gen, op, opName, asmName, pseudo_mnemonic, isSingle>;
+
+multiclass VOP3_Realtriple_t16_and_f16_gfx11_gfx12<bits<10> op, string asmName, string opName = NAME,
+ string pseudo_mnemonic = "", bit isSingle = 0> {
+ defm opName#"_t16": VOP3_Realtriple_t16_gfx11_gfx12<op, asmName, opName#"_t16", pseudo_mnemonic, isSingle>;
+ defm opName#"_fake16": VOP3_Realtriple_t16_gfx11_gfx12<op, asmName, opName#"_fake16", pseudo_mnemonic, isSingle>;
+}
+
multiclass VOP3be_Real_gfx11_gfx12<bits<10> op, string opName, string asmName> :
VOP3be_Real<GFX11Gen, op, opName, asmName>,
VOP3be_Real<GFX12Gen, op, opName, asmName>;
@@ -1128,7 +1143,7 @@ defm V_BFI_B32 : VOP3_Realtriple_gfx11_gfx12<0x212>;
defm V_FMA_F32 : VOP3_Realtriple_gfx11_gfx12<0x213>;
defm V_FMA_F64 : VOP3_Real_Base_gfx11_gfx12<0x214>;
defm V_LERP_U8 : VOP3_Realtriple_gfx11_gfx12<0x215>;
-defm V_ALIGNBIT_B32 : VOP3_Realtriple_gfx11_gfx12<0x216>;
+defm V_ALIGNBIT_B32 : VOP3_Realtriple_t16_and_f16_gfx11_gfx12<0x216, "v_alignbit_b32">;
defm V_ALIGNBYTE_B32 : VOP3_Realtriple_gfx11_gfx12<0x217>;
defm V_MULLIT_F32 : VOP3_Realtriple_gfx11_gfx12<0x218>;
defm V_MIN3_F32 : VOP3_Realtriple_gfx11<0x219>;
diff --git a/llvm/lib/Target/AMDGPU/VOPInstructions.td b/llvm/lib/Target/AMDGPU/VOPInstructions.td
index 5a460ef0d42320..8dc25409a6c224 100644
--- a/llvm/lib/Target/AMDGPU/VOPInstructions.td
+++ b/llvm/lib/Target/AMDGPU/VOPInstructions.td
@@ -115,9 +115,7 @@ class VOP3_Pseudo <string opName, VOPProfile P, list<dag> pattern = [],
bit HasFP8DstByteSel = P.HasFP8DstByteSel;
- let AsmOperands = !if(isVop3OpSel,
- P.AsmVOP3OpSel,
- !if(!and(isVOP3P, P.IsPacked), P.AsmVOP3P, P.Asm64));
+ let AsmOperands = !if(!and(isVOP3P, P.IsPacked), P.AsmVOP3P, P.Asm64);
let Size = 8;
let mayLoad = 0;
@@ -184,6 +182,7 @@ class VOP3_Real <VOP_Pseudo ps, int EncodingFamily, string asm_name = ps.Mnemoni
let SubtargetPredicate = ps.SubtargetPredicate;
let WaveSizePredicate = ps.WaveSizePredicate;
let OtherPredicates = ps.OtherPredicates;
+ let True16Predicate = ps.True16Predicate;
let AsmMatchConverter = ps.AsmMatchConverter;
let AsmVariantName = ps.AsmVariantName;
let Constraints = ps.Constraints;
@@ -248,6 +247,41 @@ class VOP3a<VOPProfile P> : Enc64 {
let Inst{63} = !if(P.HasSrc2Mods, src2_modifiers{0}, 0);
}
+// To avoid having different version of every type of operand depending on if
+// they are part of a True16 instruction or not, the operand encoding should be
+// the same for SGPR, imm, and VGPR_32 whether the instruction is True16 or not.
+class VOP3a_t16<VOPProfile P> : Enc64 {
+ bits<11> vdst;
+ bits<4> src0_modifiers;
+ bits<11> src0;
+ bits<3> src1_modifiers;
+ bits<11> src1;
+ bits<3> src2_modifiers;
+ bits<11> src2;
+ bits<1> clamp;
+ bits<2> omod;
+
+ let Inst{7-0} = !if(P.EmitDst, vdst{7-0}, 0);
+ let Inst{8} = !if(P.HasSrc0Mods, src0_modifiers{1}, 0);
+ let Inst{9} = !if(P.HasSrc1Mods, src1_modifiers{1}, 0);
+ let Inst{10} = !if(P.HasSrc2Mods, src2_modifiers{1}, 0);
+ // 16-bit select fields which can be interpreted as OpSel or hi/lo suffix
+ let Inst{11} = !if(P.HasSrc0Mods, src0_modifiers{2}, 0);
+ let Inst{12} = !if(P.HasSrc1Mods, src1_modifiers{2}, 0);
+ let Inst{13} = !if(P.HasSrc2Mods, src2_modifiers{2}, 0);
+ let Inst{14} = !if(!and(P.HasDst, P.HasSrc0Mods), src0_modifiers{3}, 0);
+ let Inst{15} = !if(P.HasClamp, clamp{0}, 0);
+
+ let Inst{31-26} = 0x35;
+ let Inst{40-32} = !if(P.HasSrc0, src0{8-0}, 0);
+ let Inst{49-41} = !if(P.HasSrc1, src1{8-0}, 0);
+ let Inst{58-50} = !if(P.HasSrc2, src2{8-0}, 0);
+ let Inst{60-59} = !if(P.HasOMod, omod, 0);
+ let Inst{61} = !if(P.HasSrc0Mods, src0_modifiers{0}, 0);
+ let Inst{62} = !if(P.HasSrc1Mods, src1_modifiers{0}, 0);
+ let Inst{63} = !if(P.HasSrc2Mods, src2_modifiers{0}, 0);
+}
+
class VOP3a_gfx6_gfx7<bits<9> op, VOPProfile p> : VOP3a<p> {
let Inst{11} = !if(p.HasClamp, clamp{0}, 0);
let Inst{25-17} = op;
@@ -278,6 +312,10 @@ class VOP3e_gfx10<bits<10> op, VOPProfile p> : VOP3a_gfx10<op, p> {
class VOP3e_gfx11_gfx12<bits<10> op, VOPProfile p> : VOP3e_gfx10<op, p>;
+class VOP3e_t16_gfx11_gfx12<bits<10> op, VOPProfile p> : VOP3a_t16<p> {
+ let Inst{25-16} = op;
+}
+
class VOP3e_vi <bits<10> op, VOPProfile P> : VOP3a_vi <op, P> {
bits<8> vdst;
let Inst{7-0} = !if(P.EmitDst, vdst{7-0}, 0);
@@ -742,7 +780,12 @@ class VOP3_DPPe_Fields : VOP3_DPPe_Fields_Base {
bits<8> src0;
}
+class VOP3_DPPe_Fields_t16 : VOP3_DPPe_Fields_Base {
+ bits<11> src0;
+}
+
// Common refers to common between DPP and DPP8
+// Base refers to a shared base between T16 and regular instructions
class VOP3_DPPe_Common_Base<bits<10> op, VOPProfile P> : Enc96 {
bits<4> src0_modifiers;
bits<3> src1_modifiers;
@@ -754,7 +797,7 @@ class VOP3_DPPe_Common_Base<bits<10> op, VOPProfile P> : Enc96 {
let Inst{8} = !if(P.HasSrc0Mods, src0_modifiers{1}, 0);
let Inst{9} = !if(P.HasSrc1Mods, src1_modifiers{1}, 0);
let Inst{10} = !if(P.HasSrc2Mods, src2_modifiers{1}, 0);
- // OPSEL must be set such that the low result only uses low inputs, and the high result only uses high inputs.
+ // 16-bit select fields which can be interpreted as OpSel or hi/lo suffix
let Inst{11} = !if(P.HasOpSel, !if(P.HasSrc0Mods, src0_modifiers{2}, 0),
!if(P.IsFP8SrcByteSel, byte_sel{1}, ?));
let Inst{12} = !if(P.HasOpSel, !if(P.HasSrc1Mods, src1_modifiers{2}, 0),
@@ -783,6 +826,16 @@ class VOP3_DPPe_Common<bits<10> op, VOPProfile P> : VOP3_DPPe_Common_Base<op, P>
let Inst{58-50} = !if(P.HasSrc2, src2, 0);
}
+class VOP3_DPPe_Common_t16<bits<10> op, VOPProfile P> : VOP3_DPPe_Common_Base<op, P> {
+ bits<11> vdst;
+ bits<11> src1;
+ bits<11> src2;
+
+ let Inst{7-0} = !if(P.EmitDst, vdst{7-0}, 0);
+ let Inst{49-41} = !if(P.HasSrc1, src1{8-0}, 0);
+ let Inst{58-50} = !if(P.HasSrc2, src2{8-0}, 0);
+}
+
class VOP3P_DPPe_Common_Base<bits<7> op, VOPProfile P> : Enc96 {
bits<4> src0_modifiers;
bits<4> src1_modifiers;
@@ -792,6 +845,7 @@ class VOP3P_DPPe_Common_Base<bits<7> op, VOPProfile P> : Enc96 {
let Inst{8} = !if(P.HasSrc0Mods, src0_modifiers{1}, 0); // neg_hi src0
let Inst{9} = !if(P.HasSrc1Mods, src1_modifiers{1}, 0); // neg_hi src1
let Inst{10} = !if(P.HasSrc2Mods, src2_modifiers{1}, 0); // neg_hi src2
+ // OPSEL must be set such that the low result only uses low inputs, and the high result only uses high inputs.
let Inst{11} = !if(!and(P.HasSrc0, P.HasOpSel), src0_modifiers{2}, 0); // op_sel(0)
let Inst{12} = !if(!and(P.HasSrc1, P.HasOpSel), src1_modifiers{2}, 0); // op_sel(1)
let Inst{13} = !if(!and(P.HasSrc2, P.HasOpSel), src2_modifiers{2}, 0); // op_sel(2)
@@ -816,6 +870,16 @@ class VOP3P_DPPe_Common<bits<7> op, VOPProfile P> : VOP3P_DPPe_Common_Base<op, P
let Inst{58-50} = !if(P.HasSrc2, src2, 0);
}
+class VOP3P_DPPe_Common_t16<bits<7> op, VOPProfile P> : VOP3P_DPPe_Common_Base<op, P> {
+ bits<11> vdst;
+ bits<11> src1;
+ bits<11> src2;
+
+ let Inst{7-0} = vdst{7-0};
+ let Inst{49-41} = !if(P.HasSrc1, src1{8-0}, 0);
+ let Inst{58-50} = !if(P.HasSrc2, src2{8-0}, 0);
+}
+
class VOP_DPP_Pseudo <string OpName, VOPProfile P, list<dag> pattern=[],
dag Ins = P.InsDPP, string asmOps = P.AsmDPP> :
VOP_Pseudo<OpName, "_dpp", P, P.OutsDPP, Ins, asmOps, pattern> {
@@ -879,6 +943,7 @@ class VOP_DPP_Real <VOP_DPP_Pseudo ps, int EncodingFamily> :
// Copy relevant pseudo op flags
let isConvergent = ps.isConvergent;
let SubtargetPredicate = ps.SubtargetPredicate;
+ let True16Predicate = ps.True16Predicate;
let AssemblerPredicate = ps.AssemblerPredicate;
let OtherPredicates = ps.OtherPredicates;
let AsmMatchConverter = ps.AsmMatchConverter;
@@ -937,11 +1002,9 @@ class VOP3_DPP_Base <string OpName, VOPProfile P, bit IsDPP16,
let Size = 12;
}
-class VOP3_DPP <bits<10> op, string OpName, VOPProfile P, bit IsDPP16,
- dag InsDPP = !if(IsDPP16, P.InsVOP3DPP16, P.InsVOP3DPP),
- string AsmDPP = !if(IsDPP16, P.AsmVOP3DPP16, P.AsmVOP3DPP)> :
- VOP3_DPP_Base<OpName, P, IsDPP16, InsDPP, AsmDPP>, VOP3_DPPe_Common<op, P>,
- VOP3_DPPe_Fields {
+class VOP3_DPP_Enc_t16<bits<10> op, VOPProfile P, bit IsDPP16 >
+ : VOP3_DPPe_Common_t16<op, P>,
+ VOP3_DPPe_Fields_t16 {
let Inst{40-32} = 0xfa;
let Inst{71-64} = !if(P.HasSrc0, src0{7-0}, 0);
@@ -954,6 +1017,13 @@ class VOP3_DPP <bits<10> op, string OpName, VOPProfile P, bit IsDPP16,
let Inst{95-92} = row_mask;
}
+class VOP3_DPP_t16<bits<10> op, string OpName, VOPProfile P, bit IsDPP16,
+ dag InsDPP = !if (IsDPP16, P.InsVOP3DPP16, P.InsVOP3DPP),
+ string AsmDPP = !if (IsDPP16, P.AsmVOP3DPP16, P.AsmVOP3DPP)>
+ : VOP3_DPP_Base<OpName, P, IsDPP16, InsDPP, AsmDPP>,
+ VOP3_DPP_Enc_t16<op, P, IsDPP16> {
+}
+
class VOP3P_DPP <bits<7> op, string OpName, VOPProfile P, bit IsDPP16,
dag InsDPP = !if(IsDPP16, P.InsVOP3DPP16, P.InsVOP3DPP),
string AsmDPP = !if(IsDPP16, P.AsmVOP3DPP16, P.AsmVOP3DPP)> :
@@ -988,6 +1058,12 @@ class VOP3_DPP8e_Fields {
bits<9> fi;
}
+class VOP3_DPP8e_Fields_t16 {
+ bits<11> src0;
+ bits<24> dpp8;
+ bits<9> fi;
+}
+
class VOP_DPP8_Base<string OpName, VOPProfile P, dag InsDPP8 = P.InsDPP8, string AsmDPP8 = P.AsmDPP8> :
InstSI<P.OutsDPP8, InsDPP8, OpName#AsmDPP8, []> {
@@ -1030,6 +1106,17 @@ class VOP3_DPP8<bits<10> op, string OpName, VOPProfile P> :
let Inst{95-72} = dpp8{23-0};
}
+class VOP3_DPP8_Enc_t16 <bits<10> op, VOPProfile P> :
+ VOP3_DPPe_Common_t16<op, P>,
+ VOP3_DPP8e_Fields_t16 {
+ let Inst{40-32} = fi;
+ let Inst{71-64} = !if(P.HasSrc0, src0{7-0}, 0);
+ let Inst{95-72} = dpp8{23-0};
+}
+
+class VOP3_DPP8_t16<bits<10> op, string OpName, VOPProfile P> :
+ VOP3_DPP8_Base<OpName, P>, VOP3_DPP8_Enc_t16<op, P>;
+
class VOP3P_DPP8<bits<7> op, string OpName, VOPProfile P> :
VOP3_DPP8_Base<OpName, P>, VOP3P_DPPe_Common<op, P>,
VOP3_DPP8e_Fields {
@@ -1274,7 +1361,7 @@ class VOP3_Profile_Base<VOPProfile P, VOP3Features Features = VOP3_REGULAR> : VO
let HasModifiers =
!if (Features.IsMAI, 0,
- !or(Features.IsPacked, Features.HasOpSel, P.HasModifiers));
+ !or(Features.IsPacked, P.HasModifiers));
}
class VOP3_Profile<VOPProfile P, VOP3Features Features = VOP3_REGULAR> : VOP3_Profile_Base<P, Features> {
@@ -1282,6 +1369,32 @@ class VOP3_Profile<VOPProfile P, VOP3Features Features = VOP3_REGULAR> : VOP3_Pr
}
+class VOP3_Profile_True16<VOPProfile P, VOP3Features Features = VOP3_REGULAR> : VOPProfile_True16<P> {
+
+ let HasClamp = !if(Features.HasClamp, 1, P.HasClamp);
+ let HasOpSel = !if(Features.HasOpSel, 1, P.HasOpSel);
+ let IsMAI = !if(Features.IsMAI, 1, P.IsMAI);
+ let IsPacked = !if(Features.IsPacked, 1, P.IsPacked);
+
+ let HasModifiers =
+ !if (Features.IsMAI, 0,
+ !or(Features.IsPacked, Features.HasOpSel, P.HasModifiers));
+ let IsSingle = 1;
+}
+
+class VOP3_Profile_Fake16<VOPProfile P, VOP3Features Features = VOP3_REGULAR> : VOPProfile_Fake16<P> {
+
+ let HasClamp = !if(Features.HasClamp, 1, P.HasClamp);
+ let HasOpSel = !if(Features.HasOpSel, 1, P.HasOpSel);
+ let IsMAI = !if(Features.IsMAI, 1, P.IsMAI);
+ let IsPacked = !if(Features.IsPacked, 1, P.IsPacked);
+
+ let HasModifiers =
+ !if (Features.IsMAI, 0,
+ !or(Features.IsPacked, Features.HasOpSel, P.HasModifiers));
+ let IsSingle = 1;
+}
+
// consistently gives instructions a _e64 suffix
multiclass VOP3Inst_Pseudo_Wrapper<string opName, VOPProfile P, list<dag> pattern = [], bit VOP3Only = 0> {
def _e64 : VOP3_Pseudo<opName, P, pattern, VOP3Only>;
@@ -1334,6 +1447,28 @@ multiclass VOP3PseudoScalarInst<string OpName, VOPProfile P,
i32:$omod))))]>;
}
+multiclass VOP3Inst_t16_with_profiles<string OpName, VOPProfile P, VOPProfile P_t16,
+ VOPProfile P_fake16,
+ SDPatternOperator node = null_frag,
+ SDPatternOperator node_t16 = node> {
+ let True16Predicate = NotHasTrue16BitInsts in {
+ defm NAME : VOP3Inst<OpName, P, node>;
+ }
+ let True16Predicate = UseRealTrue16Insts in {
+ defm _t16 : VOP3Inst<OpName # "_t16", P_t16, node_t16>;
+ }
+ let True16Predicate = UseFakeTrue16Insts in {
+ defm _fake16 : VOP3Inst<OpName # "_fake16", P_fake16, node>;
+ }
+}
+
+multiclass VOP3Inst_t16<string OpName, VOPProfile P,
+ SDPatternOperator node = null_frag,
+ SDPatternOperator node_t16 = node>
+ : VOP3Inst_t16_with_profiles<OpName, VOP3_Profile<P, VOP3_OPSEL>,
+ VOP3_Profile_True16<P, VOP3_OPSEL>, VOP3_Profile_Fake16<P, VOP3_OPSEL>,
+ node, node_t16>;
+
//===----------------------------------------------------------------------===//
// VOP3 DPP
//===----------------------------------------------------------------------===//
@@ -1351,17 +1486,43 @@ class Base_VOP3_DPP16<bits<10> op, VOP_DPP_Pseudo ps, string opName = ps.OpName>
let OtherPredicates = ps.OtherPredicates;
}
+class Base_VOP3_DPP16_t16<bits<10> op, VOP_DPP_Pseudo ps,
+ string opName = ps.OpName>
+ : VOP3_DPP_t16<op, opName, ps.Pfl, 1> {
+ let VOP3_OPSEL = ps.Pfl.HasOpSel;
+ let IsDOT = ps.IsDOT;
+ let hasSideEffects = ps.hasSideEffects;
+ let Defs = ps.Defs;
+ let SchedRW = ps.SchedRW;
+ let Uses = ps.Uses;
+ let AssemblerPredicate = HasDPP16;
+ let SubtargetPredicate = HasDPP16;
+ let OtherPredicates = ps.OtherPredicates;
+}
+
class VOP3_DPP16<bits<10> op, VOP_DPP_Pseudo ps, int subtarget,
string opName = ps.OpName>
: Base_VOP3_DPP16<op, ps, opName>, SIMCInstr<ps.PseudoInstr, subtarget>;
+class VOP3_DPP16_t16<bits<10> op, VOP_DPP_Pseudo ps, int subtarget,
+ string opName = ps.OpName>
+ : Base_VOP3_DPP16_t16<op, ps, opName>, SIMCInstr<ps.PseudoInstr, subtarget>;
+
class VOP3_DPP16_Gen<bits<10> op, VOP_DPP_Pseudo ps, GFXGen Gen,
string opName = ps.OpName> :
VOP3_DPP16 <op, ps, Gen.Subtarget, opName> {
let AssemblerPredicate = Gen.AssemblerPredicate;
- let True16Predicate = !if(ps.Pfl.IsRealTrue16, UseRealTrue16Insts, NoTrue16Predicate);
- let DecoderNamespace = Gen.DecoderNamespace#
- !if(ps.Pfl.IsRealTrue16, "", "_FAKE16");
+ let DecoderNamespace = Gen.DecoderNamespace;
+}
+
+class VOP3_DPP16_Gen_t16<bits<10> op, VOP_DPP_Pseudo ps, GFXGen Gen,
+ string opName = ps.OpName>
+ : VOP3_DPP16_t16<op, ps, Gen.Subtarget, opName> {
+ let True16Predicate =
+ !if (ps.Pfl.IsRealTrue16, UseRealTrue16Insts, NoTrue16Predicate);
+ let AssemblerPredicate = Gen.AssemblerPredicate;
+ let DecoderNamespace =
+ Gen.DecoderNamespace #!if (ps.Pfl.IsRealTrue16, "", "_FAKE16");
}
class Base_VOP3_DPP8<bits<10> op, VOP_Pseudo ps, string opName = ps.OpName>
@@ -1375,6 +1536,20 @@ class Base_VOP3_DPP8<bits<10> op, VOP_Pseudo ps, string opName = ps.OpName>
let SubtargetPredicate = ps.SubtargetPredicate;
let OtherPredicates = ps.OtherPredicates;
+ let True16Predicate = ps.True16Predicate;
+}
+
+class Base_VOP3_DPP8_t16<bits<10> op, VOP_Pseudo ps, string opName = ps.OpName>
+ : VOP3_DPP8_t16<op, opName, ps.Pfl> {
+ let VOP3_OPSEL = ps.Pfl.HasOpSel;
+ let IsDOT = ps.IsDOT;
+ let hasSideEffects = ps.hasSideEffects;
+ let Defs = ps.Defs;
+ let SchedRW = ps.SchedRW;
+ let Uses = ps.Uses;
+
+ let OtherPredicates = ps.OtherPredicates;
+ let True16Predicate = ps.True16Predicate;
}
class Base_VOP3b_DPP16<bits<10> op, VOP_DPP_Pseudo ps,
@@ -1390,6 +1565,12 @@ class VOP3b_DPP8_Base<bits<10> op, VOP_Pseudo ps, string opName = ps.OpName>
let Inst{14 - 8} = sdst;
}
+class VOP3b_DPP8_Base_t16<bits<10> op, VOP_Pseudo ps, string opName = ps.OpName>
+ : Base_VOP3_DPP8<op, ps, opName> {
+ bits<8> sdst;
+ let Inst{14 - 8} = sdst{7-1};
+}
+
//===----------------------------------------------------------------------===//
// VOP3 GFX11, GFX12
//===----------------------------------------------------------------------===//
@@ -1429,10 +1610,11 @@ multiclass VOP3Dot_Real_Base<GFXGen Gen, bits<10> op, string opName = NAME,
}
multiclass VOP3_Real_with_name<GFXGen Gen, bits<10> op, string opName,
- string asmName, bit isSingle = 0> {
+ string asmName, string pseudo_mnemonic = "", bit isSingle = 0> {
defvar ps = !cast<VOP_Pseudo>(opName#"_e64");
let AsmString = asmName # ps.AsmOperands,
IsSingle = !or(isSingle, ps.Pfl.IsSingle) in {
+ // FIXME-TRUE16 support FP8 instructions properly
if ps.Pfl.IsFP8SrcByteSel then {
def _e64#Gen.Suffix :
VOP3_Real_Gen<ps, Gen>,
@@ -1441,17 +1623,27 @@ multiclass VOP3_Real_with_name<GFXGen Gen, bits<10> op, string opName,
def _e64#Gen.Suffix :
VOP3_Real_Gen<ps, Gen>,
VOP3FP8OpSel_dst_bytesel_gfx11_gfx12<op, ps.Pfl>;
- } else if ps.Pfl.HasOpSel then {
- def _e64#Gen.Suffix :
- VOP3_Real_Gen<ps, Gen>,
- VOP3OpSel_gfx11_gfx12<op, ps.Pfl>;
} else {
- def _e64#Gen.Suffix :
- VOP3_Real_Gen<ps, Gen>,
- VOP3e_gfx11_gfx12<op, ps.Pfl>;
+ if ps.Pfl.IsRealTrue16 then {
+ def _e64#Gen.Suffix :
+ VOP3_Real_Gen<ps, Gen>,
+ VOP3e_t16_gfx11_gfx12<op, ps.Pfl>;
+ } else {
+ if ps.Pfl.HasOpSel then {
+ def _e64#Gen.Suffix :
+ VOP3_Real_Gen<ps, Gen>,
+ VOP3OpSel_gfx11_gfx12<op, ps.Pfl>;
+ } else {
+ def _e64#Gen.Suffix :
+ VOP3_Real_Gen<ps, Gen>,
+ VOP3e_gfx11_gfx12<op, ps.Pfl>;
+ }
+ }
}
}
- def Gen.Suffix#"_VOP3_alias" : LetDummies, AMDGPUMnemonicAlias<ps.Mnemonic, asmName> {
+ def Gen.Suffix#"_VOP3_alias" : LetDummies,
+ AMDGPUMnemonicAlias<!if(!empty(pseudo_mnemonic),
+ ps.Mnemonic, pseudo_mnemonic), asmName, ""> {
let AssemblerPredicate = Gen.AssemblerPredicate;
}
}
@@ -1465,8 +1657,13 @@ multiclass VOP3_Real_No_Suffix<GFXGen Gen, bits<10> op, string opName = NAME> {
}
multiclass VOP3_Real_dpp_Base<GFXGen Gen, bits<10> op, string opName = NAME> {
- def _e64_dpp#Gen.Suffix :
- VOP3_DPP16_Gen<op, !cast<VOP_DPP_Pseudo>(opName#"_e64"#"_dpp"), Gen>;
+ defvar ps = !cast<VOP_DPP_Pseudo>(opName#"_e64"#"_dpp");
+ if ps.Pfl.IsTrue16 then
+ def _e64_dpp#Gen.Suffix :
+ VOP3_DPP16_Gen_t16<op, ps, Gen>;
+ else
+ def _e64_dpp#Gen.Suffix :
+ VOP3_DPP16_Gen<op, ps, Gen>;
}
multiclass VOP3Dot_Real_dpp_Base<GFXGen Gen, bits<10> op, string opName = NAME> {
@@ -1561,18 +1758,14 @@ multiclass VOP3Only_Realtriple<GFXGen Gen, bits<10> op> :
VOP3_Realtriple<Gen, op, 1>;
multiclass VOP3_Realtriple_with_name<GFXGen Gen, bits<10> op, string opName,
- string asmName, bit isSingle = 0> :
- VOP3_Real_with_name<Gen, op, opName, asmName, isSingle>,
+ string asmName, string pseudo_mnemonic = "", bit isSingle = 0> :
+ VOP3_Real_with_name<Gen, op, opName, asmName, pseudo_mnemonic, isSingle>,
VOP3_Real_dpp_with_name<Gen, op, opName, asmName>,
VOP3_Real_dpp8_with_name<Gen, op, opName, asmName>;
multiclass VOP3Only_Realtriple_with_name<GFXGen Gen, bits<10> op, string opName,
string asmName> :
- VOP3_Realtriple_with_name<Gen, op, opName, asmName, 1>;
-
-multiclass VOP3Only_Realtriple_t16<GFXGen Gen, bits<10> op, string asmName,
- string opName = NAME>
- : VOP3Only_Realtriple_with_name<Gen, op, opName, asmName>;
+ VOP3_Realtriple_with_name<Gen, op, opName, asmName, "", 1>;
multiclass VOP3be_Realtriple<
GFXGen Gen, bits<10> op, bit isSingle = 0, string opName = NAME,
@@ -1588,6 +1781,16 @@ multiclass VOP3beOnly_Realtriple<GFXGen Gen, bits<10> op> :
// VOP3 GFX11
//===----------------------------------------------------------------------===//
+// VOP1 and VOP2 depend on these triple defs
+
+multiclass VOP3_Realtriple_t16_gfx11<bits<10> op, string asmName, string opName = NAME,
+ string pseudo_mnemonic = "", bit isSingle = 0> :
+ VOP3_Realtriple_with_name<GFX11Gen, op, opName, asmName, pseudo_mnemonic, isSingle>;
+
+multiclass VOP3Only_Realtriple_t16_gfx11<bits<10> op, string asmName,
+ string opName = NAME, string pseudo_mnemonic = "">
+ : VOP3_Realtriple_t16_gfx11<op, asmName, opName, pseudo_mnemonic, 1>;
+
multiclass VOP3be_Real_gfx11<bits<10> op, string opName, string asmName,
bit isSingle = 0> :
VOP3be_Real<GFX11Gen, op, opName, asmName, isSingle>;
@@ -1600,10 +1803,6 @@ multiclass VOP3_Realtriple_gfx11<bits<10> op, bit isSingle = 0,
string opName = NAME> :
VOP3_Realtriple<GFX11Gen, op, isSingle, opName>;
-multiclass VOP3Only_Realtriple_t16_gfx11<bits<10> op, string asmName,
- string opName = NAME>
- : VOP3Only_Realtriple_with_name<GFX11Gen, op, opName, asmName>;
-
//===----------------------------------------------------------------------===//
// VOP3 GFX12
//===----------------------------------------------------------------------===//
@@ -1616,8 +1815,15 @@ multiclass VOP3Only_Realtriple_gfx12<bits<10> op, bit isSingle = 0> :
multiclass VOP3Only_Real_Base_gfx12<bits<10> op> :
VOP3_Real_Base<GFX12Gen, op, NAME, 1/*IsSingle*/>;
-multiclass VOP3Only_Realtriple_t16_gfx12<bits<10> op> :
- VOP3Only_Realtriple<GFX12Gen, op>;
+multiclass VOP3_Realtriple_t16_gfx12<bits<10> op, string asmName, string opName = NAME,
+ string pseudo_mnemonic = "", bit isSingle = 0> :
+ VOP3_Realtriple_with_name<GFX12Gen, op, opName, asmName, pseudo_mnemonic, isSingle>;
+
+multiclass VOP3_Realtriple_t16_and_f16_gfx12<bits<10> op, string asmName, string opName = NAME,
+ string pseudo_mnemonic = "", bit isSingle = 0> {
+ defm opName#"_t16":VOP3_Realtriple_t16_gfx12<op, asmName, opName#"_t16", pseudo_mnemonic, isSingle>;
+ defm opName#"_fake16":VOP3_Realtriple_t16_gfx12<op, asmName, opName#"_fake16", pseudo_mnemonic, isSingle>;
+}
multiclass VOP3be_Real_with_name_gfx12<bits<10> op, string opName,
string asmName, bit isSingle = 0> {
@@ -1633,18 +1839,14 @@ multiclass VOP3be_Real_with_name_gfx12<bits<10> op, string opName,
}
multiclass VOP3_Realtriple_with_name_gfx12<bits<10> op, string opName,
- string asmName, bit isSingle = 0> :
- VOP3_Realtriple_with_name<GFX12Gen, op, opName, asmName, isSingle>;
+ string asmName, string pseudo_mnemonic = "", bit isSingle = 0> :
+ VOP3_Realtriple_with_name<GFX12Gen, op, opName, asmName, pseudo_mnemonic, isSingle>;
multiclass VOP3Only_Realtriple_with_name_gfx11_gfx12<bits<10> op, string opName,
string asmName> :
VOP3Only_Realtriple_with_name<GFX11Gen, op, opName, asmName>,
VOP3Only_Realtriple_with_name<GFX12Gen, op, opName, asmName>;
-multiclass VOP3Only_Realtriple_with_name_t16_gfx12<bits<10> op, string asmName,
- string opName = NAME>
- : VOP3Only_Realtriple_with_name<GFX12Gen, op, opName, asmName>;
-
//===----------------------------------------------------------------------===//
include "VOPCInstructions.td"
diff --git a/llvm/test/CodeGen/AMDGPU/bf16.ll b/llvm/test/CodeGen/AMDGPU/bf16.ll
index d9ce1e4efe0e50..aa6a0ea134fe22 100644
--- a/llvm/test/CodeGen/AMDGPU/bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/bf16.ll
@@ -9314,34 +9314,35 @@ define <3 x bfloat> @v_fadd_v3bf16(<3 x bfloat> %a, <3 x bfloat> %b) {
; GFX11TRUE16-LABEL: v_fadd_v3bf16:
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX11TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX11TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_dual_add_f32 v4, v5, v4 :: v_dual_lshlrev_b32 v1, 16, v1
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11TRUE16-NEXT: v_dual_add_f32 v0, v0, v2 :: v_dual_add_f32 v1, v1, v3
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_bfe_u32 v2, v4, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v4
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11TRUE16-NEXT: v_add_f32_e32 v3, v5, v4
; GFX11TRUE16-NEXT: v_bfe_u32 v5, v0, 16, 1
-; GFX11TRUE16-NEXT: v_bfe_u32 v3, v1, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v2, v2, v4, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_bfe_u32 v2, v1, 16, 1
+; GFX11TRUE16-NEXT: v_bfe_u32 v4, v3, 16, 1
; GFX11TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v1
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
+; GFX11TRUE16-NEXT: v_add3_u32 v2, v2, v1, 0x7fff
+; GFX11TRUE16-NEXT: v_add3_u32 v4, v4, v3, 0x7fff
; GFX11TRUE16-NEXT: v_add3_u32 v5, v5, v0, 0x7fff
-; GFX11TRUE16-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v7, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v2, v6, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v7, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v5, v8, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11TRUE16-NEXT: v_perm_b32 v0, v0, v2, 0x7060302
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v6, vcc_lo
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_alignbit_b32 v1, v0, v1, 16
+; GFX11TRUE16-NEXT: v_perm_b32 v0, v0, v2, 0x7060302
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_fadd_v3bf16:
@@ -13082,34 +13083,35 @@ define <3 x bfloat> @v_fsub_v3bf16(<3 x bfloat> %a, <3 x bfloat> %b) {
; GFX11TRUE16-LABEL: v_fsub_v3bf16:
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX11TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX11TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_dual_sub_f32 v4, v5, v4 :: v_dual_lshlrev_b32 v1, 16, v1
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11TRUE16-NEXT: v_dual_sub_f32 v0, v0, v2 :: v_dual_sub_f32 v1, v1, v3
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_bfe_u32 v2, v4, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v4
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11TRUE16-NEXT: v_sub_f32_e32 v3, v5, v4
; GFX11TRUE16-NEXT: v_bfe_u32 v5, v0, 16, 1
-; GFX11TRUE16-NEXT: v_bfe_u32 v3, v1, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v2, v2, v4, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_bfe_u32 v2, v1, 16, 1
+; GFX11TRUE16-NEXT: v_bfe_u32 v4, v3, 16, 1
; GFX11TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v1
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
+; GFX11TRUE16-NEXT: v_add3_u32 v2, v2, v1, 0x7fff
+; GFX11TRUE16-NEXT: v_add3_u32 v4, v4, v3, 0x7fff
; GFX11TRUE16-NEXT: v_add3_u32 v5, v5, v0, 0x7fff
-; GFX11TRUE16-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v7, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v2, v6, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v7, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v5, v8, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11TRUE16-NEXT: v_perm_b32 v0, v0, v2, 0x7060302
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v6, vcc_lo
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_alignbit_b32 v1, v0, v1, 16
+; GFX11TRUE16-NEXT: v_perm_b32 v0, v0, v2, 0x7060302
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_fsub_v3bf16:
@@ -13751,34 +13753,35 @@ define <3 x bfloat> @v_fmul_v3bf16(<3 x bfloat> %a, <3 x bfloat> %b) {
; GFX11TRUE16-LABEL: v_fmul_v3bf16:
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX11TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX11TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_dual_mul_f32 v4, v5, v4 :: v_dual_lshlrev_b32 v1, 16, v1
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11TRUE16-NEXT: v_dual_mul_f32 v0, v0, v2 :: v_dual_mul_f32 v1, v1, v3
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_bfe_u32 v2, v4, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v4
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11TRUE16-NEXT: v_mul_f32_e32 v3, v5, v4
; GFX11TRUE16-NEXT: v_bfe_u32 v5, v0, 16, 1
-; GFX11TRUE16-NEXT: v_bfe_u32 v3, v1, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v2, v2, v4, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_bfe_u32 v2, v1, 16, 1
+; GFX11TRUE16-NEXT: v_bfe_u32 v4, v3, 16, 1
; GFX11TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v1
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
+; GFX11TRUE16-NEXT: v_add3_u32 v2, v2, v1, 0x7fff
+; GFX11TRUE16-NEXT: v_add3_u32 v4, v4, v3, 0x7fff
; GFX11TRUE16-NEXT: v_add3_u32 v5, v5, v0, 0x7fff
-; GFX11TRUE16-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v7, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v2, v6, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v7, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v5, v8, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11TRUE16-NEXT: v_perm_b32 v0, v0, v2, 0x7060302
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v6, vcc_lo
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_alignbit_b32 v1, v0, v1, 16
+; GFX11TRUE16-NEXT: v_perm_b32 v0, v0, v2, 0x7060302
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_fmul_v3bf16:
@@ -17829,34 +17832,35 @@ define <3 x bfloat> @v_minnum_v3bf16(<3 x bfloat> %a, <3 x bfloat> %b) {
; GFX11TRUE16-LABEL: v_minnum_v3bf16:
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX11TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX11TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_dual_min_f32 v4, v5, v4 :: v_dual_lshlrev_b32 v1, 16, v1
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11TRUE16-NEXT: v_dual_min_f32 v0, v0, v2 :: v_dual_min_f32 v1, v1, v3
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_bfe_u32 v2, v4, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v4
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11TRUE16-NEXT: v_min_f32_e32 v3, v5, v4
; GFX11TRUE16-NEXT: v_bfe_u32 v5, v0, 16, 1
-; GFX11TRUE16-NEXT: v_bfe_u32 v3, v1, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v2, v2, v4, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_bfe_u32 v2, v1, 16, 1
+; GFX11TRUE16-NEXT: v_bfe_u32 v4, v3, 16, 1
; GFX11TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v1
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
+; GFX11TRUE16-NEXT: v_add3_u32 v2, v2, v1, 0x7fff
+; GFX11TRUE16-NEXT: v_add3_u32 v4, v4, v3, 0x7fff
; GFX11TRUE16-NEXT: v_add3_u32 v5, v5, v0, 0x7fff
-; GFX11TRUE16-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v7, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v2, v6, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v7, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v5, v8, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11TRUE16-NEXT: v_perm_b32 v0, v0, v2, 0x7060302
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v6, vcc_lo
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_alignbit_b32 v1, v0, v1, 16
+; GFX11TRUE16-NEXT: v_perm_b32 v0, v0, v2, 0x7060302
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_minnum_v3bf16:
@@ -21448,34 +21452,35 @@ define <3 x bfloat> @v_maxnum_v3bf16(<3 x bfloat> %a, <3 x bfloat> %b) {
; GFX11TRUE16-LABEL: v_maxnum_v3bf16:
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX11TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX11TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_dual_max_f32 v4, v5, v4 :: v_dual_lshlrev_b32 v1, 16, v1
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11TRUE16-NEXT: v_dual_max_f32 v0, v0, v2 :: v_dual_max_f32 v1, v1, v3
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_bfe_u32 v2, v4, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v4
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11TRUE16-NEXT: v_max_f32_e32 v3, v5, v4
; GFX11TRUE16-NEXT: v_bfe_u32 v5, v0, 16, 1
-; GFX11TRUE16-NEXT: v_bfe_u32 v3, v1, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v2, v2, v4, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_bfe_u32 v2, v1, 16, 1
+; GFX11TRUE16-NEXT: v_bfe_u32 v4, v3, 16, 1
; GFX11TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v1
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
+; GFX11TRUE16-NEXT: v_add3_u32 v2, v2, v1, 0x7fff
+; GFX11TRUE16-NEXT: v_add3_u32 v4, v4, v3, 0x7fff
; GFX11TRUE16-NEXT: v_add3_u32 v5, v5, v0, 0x7fff
-; GFX11TRUE16-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v7, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v2, v6, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v7, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v5, v8, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11TRUE16-NEXT: v_perm_b32 v0, v0, v2, 0x7060302
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v6, vcc_lo
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_alignbit_b32 v1, v0, v1, 16
+; GFX11TRUE16-NEXT: v_perm_b32 v0, v0, v2, 0x7060302
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_maxnum_v3bf16:
@@ -30454,34 +30459,33 @@ define <3 x bfloat> @v_sitofp_v3i16_to_v3bf16(<3 x i16> %x) {
; GFX11TRUE16-LABEL: v_sitofp_v3i16_to_v3bf16:
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_bfe_i32 v2, v0, 0, 16
; GFX11TRUE16-NEXT: v_bfe_i32 v1, v1, 0, 16
+; GFX11TRUE16-NEXT: v_bfe_i32 v2, v0, 0, 16
; GFX11TRUE16-NEXT: v_ashrrev_i32_e32 v0, 16, v0
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_cvt_f32_i32_e32 v2, v2
; GFX11TRUE16-NEXT: v_cvt_f32_i32_e32 v1, v1
+; GFX11TRUE16-NEXT: v_cvt_f32_i32_e32 v2, v2
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11TRUE16-NEXT: v_cvt_f32_i32_e32 v0, v0
-; GFX11TRUE16-NEXT: v_bfe_u32 v3, v2, 16, 1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_bfe_u32 v4, v1, 16, 1
+; GFX11TRUE16-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v1
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX11TRUE16-NEXT: v_bfe_u32 v5, v0, 16, 1
+; GFX11TRUE16-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
+; GFX11TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v2
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11TRUE16-NEXT: v_add3_u32 v3, v3, v2, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v1
; GFX11TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
; GFX11TRUE16-NEXT: v_add3_u32 v5, v5, v0, 0x7fff
-; GFX11TRUE16-NEXT: v_add3_u32 v4, v4, v1, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v3, v7, vcc_lo
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v6, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v7, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v5, v8, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v4, v6, vcc_lo
; GFX11TRUE16-NEXT: v_perm_b32 v0, v0, v2, 0x7060302
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_alignbit_b32 v1, v0, v1, 16
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_sitofp_v3i16_to_v3bf16:
@@ -30980,28 +30984,30 @@ define <3 x bfloat> @v_sitofp_v3i32_to_v3bf16(<3 x i32> %x) {
; GFX11TRUE16-LABEL: v_sitofp_v3i32_to_v3bf16:
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11TRUE16-NEXT: v_cvt_f32_i32_e32 v2, v2
; GFX11TRUE16-NEXT: v_cvt_f32_i32_e32 v0, v0
; GFX11TRUE16-NEXT: v_cvt_f32_i32_e32 v1, v1
-; GFX11TRUE16-NEXT: v_cvt_f32_i32_e32 v2, v2
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX11TRUE16-NEXT: v_bfe_u32 v3, v2, 16, 1
+; GFX11TRUE16-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX11TRUE16-NEXT: v_bfe_u32 v5, v1, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11TRUE16-NEXT: v_add3_u32 v3, v3, v2, 0x7fff
+; GFX11TRUE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v0
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v1
; GFX11TRUE16-NEXT: v_add3_u32 v5, v5, v1, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2
-; GFX11TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v7, vcc_lo
+; GFX11TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v1
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v3, v6, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v4, v7, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v5, v8, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11TRUE16-NEXT: v_perm_b32 v0, v1, v0, 0x7060302
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v6, vcc_lo
-; GFX11TRUE16-NEXT: v_alignbit_b32 v1, v0, v2, 16
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_sitofp_v3i32_to_v3bf16:
@@ -31858,70 +31864,68 @@ define <3 x bfloat> @v_sitofp_v3i64_to_v3bf16(<3 x i64> %x) {
; GFX11TRUE16-LABEL: v_sitofp_v3i64_to_v3bf16:
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_xor_b32_e32 v8, v0, v1
; GFX11TRUE16-NEXT: v_xor_b32_e32 v7, v4, v5
+; GFX11TRUE16-NEXT: v_cls_i32_e32 v6, v5
+; GFX11TRUE16-NEXT: v_xor_b32_e32 v8, v0, v1
; GFX11TRUE16-NEXT: v_xor_b32_e32 v9, v2, v3
; GFX11TRUE16-NEXT: v_cls_i32_e32 v10, v1
-; GFX11TRUE16-NEXT: v_cls_i32_e32 v6, v5
-; GFX11TRUE16-NEXT: v_ashrrev_i32_e32 v8, 31, v8
; GFX11TRUE16-NEXT: v_ashrrev_i32_e32 v7, 31, v7
+; GFX11TRUE16-NEXT: v_add_nc_u32_e32 v6, -1, v6
+; GFX11TRUE16-NEXT: v_ashrrev_i32_e32 v8, 31, v8
; GFX11TRUE16-NEXT: v_cls_i32_e32 v11, v3
; GFX11TRUE16-NEXT: v_ashrrev_i32_e32 v9, 31, v9
+; GFX11TRUE16-NEXT: v_add_nc_u32_e32 v7, 32, v7
; GFX11TRUE16-NEXT: v_add_nc_u32_e32 v10, -1, v10
; GFX11TRUE16-NEXT: v_add_nc_u32_e32 v8, 32, v8
-; GFX11TRUE16-NEXT: v_add_nc_u32_e32 v6, -1, v6
-; GFX11TRUE16-NEXT: v_add_nc_u32_e32 v7, 32, v7
; GFX11TRUE16-NEXT: v_add_nc_u32_e32 v11, -1, v11
; GFX11TRUE16-NEXT: v_add_nc_u32_e32 v9, 32, v9
-; GFX11TRUE16-NEXT: v_min_u32_e32 v8, v10, v8
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11TRUE16-NEXT: v_min_u32_e32 v6, v6, v7
-; GFX11TRUE16-NEXT: v_min_u32_e32 v7, v11, v9
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_min_u32_e32 v7, v10, v8
+; GFX11TRUE16-NEXT: v_min_u32_e32 v8, v11, v9
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_lshlrev_b64 v[0:1], v8, v[0:1]
; GFX11TRUE16-NEXT: v_lshlrev_b64 v[4:5], v6, v[4:5]
-; GFX11TRUE16-NEXT: v_sub_nc_u32_e32 v6, 32, v6
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_lshlrev_b64 v[2:3], v7, v[2:3]
-; GFX11TRUE16-NEXT: v_min_u32_e32 v0, 1, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_lshlrev_b64 v[0:1], v7, v[0:1]
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_lshlrev_b64 v[2:3], v8, v[2:3]
; GFX11TRUE16-NEXT: v_min_u32_e32 v4, 1, v4
-; GFX11TRUE16-NEXT: v_min_u32_e32 v2, 1, v2
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_min_u32_e32 v0, 1, v0
+; GFX11TRUE16-NEXT: v_min_u32_e32 v2, 1, v2
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_or_b32_e32 v4, v5, v4
+; GFX11TRUE16-NEXT: v_sub_nc_u32_e32 v5, 32, v8
; GFX11TRUE16-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX11TRUE16-NEXT: v_or_b32_e32 v1, v5, v4
-; GFX11TRUE16-NEXT: v_sub_nc_u32_e32 v4, 32, v7
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_sub_nc_u32_e32 v1, 32, v6
; GFX11TRUE16-NEXT: v_or_b32_e32 v2, v3, v2
-; GFX11TRUE16-NEXT: v_sub_nc_u32_e32 v3, 32, v8
+; GFX11TRUE16-NEXT: v_cvt_f32_i32_e32 v4, v4
+; GFX11TRUE16-NEXT: v_sub_nc_u32_e32 v3, 32, v7
; GFX11TRUE16-NEXT: v_cvt_f32_i32_e32 v0, v0
-; GFX11TRUE16-NEXT: v_cvt_f32_i32_e32 v1, v1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11TRUE16-NEXT: v_cvt_f32_i32_e32 v2, v2
+; GFX11TRUE16-NEXT: v_ldexp_f32 v1, v4, v1
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11TRUE16-NEXT: v_ldexp_f32 v0, v0, v3
-; GFX11TRUE16-NEXT: v_ldexp_f32 v1, v1, v6
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_ldexp_f32 v2, v2, v4
-; GFX11TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v0
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_ldexp_f32 v2, v2, v5
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v1
+; GFX11TRUE16-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX11TRUE16-NEXT: v_bfe_u32 v5, v2, 16, 1
-; GFX11TRUE16-NEXT: v_bfe_u32 v4, v1, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
+; GFX11TRUE16-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v0
+; GFX11TRUE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
; GFX11TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
-; GFX11TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v1
; GFX11TRUE16-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
-; GFX11TRUE16-NEXT: v_add3_u32 v4, v4, v1, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v7, vcc_lo
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v6, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v4, v7, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v8, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX11TRUE16-NEXT: v_perm_b32 v0, v2, v0, 0x7060302
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v4, v6, vcc_lo
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_alignbit_b32 v1, v0, v1, 16
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_sitofp_v3i64_to_v3bf16:
@@ -32727,35 +32731,35 @@ define <3 x bfloat> @v_uitofp_v3i16_to_v3bf16(<3 x i16> %x) {
; GFX11TRUE16-LABEL: v_uitofp_v3i16_to_v3bf16:
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_cvt_f32_u32_e32 v1, v1
-; GFX11TRUE16-NEXT: v_bfe_u32 v4, v1, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_add3_u32 v4, v4, v1, 0x7fff
; GFX11TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v0
; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11TRUE16-NEXT: v_cvt_f32_u32_e32 v2, v2
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_cvt_f32_u32_e32 v2, v2
; GFX11TRUE16-NEXT: v_cvt_f32_u32_e32 v0, v0
-; GFX11TRUE16-NEXT: v_bfe_u32 v3, v2, 16, 1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_bfe_u32 v5, v0, 16, 1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1
; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v2
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11TRUE16-NEXT: v_bfe_u32 v5, v0, 16, 1
; GFX11TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
-; GFX11TRUE16-NEXT: v_add3_u32 v3, v3, v2, 0x7fff
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
+; GFX11TRUE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX11TRUE16-NEXT: v_add3_u32 v5, v5, v0, 0x7fff
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v3, v7, vcc_lo
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11TRUE16-NEXT: v_cvt_f32_u32_e32 v1, v1
+; GFX11TRUE16-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v1
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11TRUE16-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v6, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v7, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v5, v8, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11TRUE16-NEXT: v_perm_b32 v0, v0, v2, 0x7060302
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v4, v6, vcc_lo
-; GFX11TRUE16-NEXT: v_alignbit_b32 v1, v0, v1, 16
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_uitofp_v3i16_to_v3bf16:
@@ -33257,28 +33261,30 @@ define <3 x bfloat> @v_uitofp_v3i32_to_v3bf16(<3 x i32> %x) {
; GFX11TRUE16-LABEL: v_uitofp_v3i32_to_v3bf16:
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11TRUE16-NEXT: v_cvt_f32_u32_e32 v2, v2
; GFX11TRUE16-NEXT: v_cvt_f32_u32_e32 v0, v0
; GFX11TRUE16-NEXT: v_cvt_f32_u32_e32 v1, v1
-; GFX11TRUE16-NEXT: v_cvt_f32_u32_e32 v2, v2
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX11TRUE16-NEXT: v_bfe_u32 v3, v2, 16, 1
+; GFX11TRUE16-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX11TRUE16-NEXT: v_bfe_u32 v5, v1, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11TRUE16-NEXT: v_add3_u32 v3, v3, v2, 0x7fff
+; GFX11TRUE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v0
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v1
; GFX11TRUE16-NEXT: v_add3_u32 v5, v5, v1, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2
-; GFX11TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v7, vcc_lo
+; GFX11TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v1
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v3, v6, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v4, v7, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v5, v8, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11TRUE16-NEXT: v_perm_b32 v0, v1, v0, 0x7060302
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v6, vcc_lo
-; GFX11TRUE16-NEXT: v_alignbit_b32 v1, v0, v2, 16
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_uitofp_v3i32_to_v3bf16:
@@ -33997,58 +34003,57 @@ define <3 x bfloat> @v_uitofp_v3i64_to_v3bf16(<3 x i64> %x) {
; GFX11TRUE16-LABEL: v_uitofp_v3i64_to_v3bf16:
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_clz_i32_u32_e32 v6, v1
-; GFX11TRUE16-NEXT: v_clz_i32_u32_e32 v7, v5
+; GFX11TRUE16-NEXT: v_clz_i32_u32_e32 v6, v5
+; GFX11TRUE16-NEXT: v_clz_i32_u32_e32 v7, v1
; GFX11TRUE16-NEXT: v_clz_i32_u32_e32 v8, v3
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11TRUE16-NEXT: v_min_u32_e32 v6, 32, v6
; GFX11TRUE16-NEXT: v_min_u32_e32 v7, 32, v7
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11TRUE16-NEXT: v_min_u32_e32 v8, 32, v8
-; GFX11TRUE16-NEXT: v_lshlrev_b64 v[0:1], v6, v[0:1]
+; GFX11TRUE16-NEXT: v_lshlrev_b64 v[4:5], v6, v[4:5]
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_lshlrev_b64 v[4:5], v7, v[4:5]
+; GFX11TRUE16-NEXT: v_lshlrev_b64 v[0:1], v7, v[0:1]
; GFX11TRUE16-NEXT: v_lshlrev_b64 v[2:3], v8, v[2:3]
-; GFX11TRUE16-NEXT: v_sub_nc_u32_e32 v7, 32, v7
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_min_u32_e32 v0, 1, v0
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11TRUE16-NEXT: v_min_u32_e32 v4, 1, v4
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_min_u32_e32 v0, 1, v0
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11TRUE16-NEXT: v_min_u32_e32 v2, 1, v2
+; GFX11TRUE16-NEXT: v_or_b32_e32 v4, v5, v4
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
; GFX11TRUE16-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_or_b32_e32 v1, v5, v4
+; GFX11TRUE16-NEXT: v_sub_nc_u32_e32 v1, 32, v6
; GFX11TRUE16-NEXT: v_or_b32_e32 v2, v3, v2
-; GFX11TRUE16-NEXT: v_sub_nc_u32_e32 v3, 32, v6
-; GFX11TRUE16-NEXT: v_sub_nc_u32_e32 v4, 32, v8
+; GFX11TRUE16-NEXT: v_sub_nc_u32_e32 v3, 32, v7
+; GFX11TRUE16-NEXT: v_cvt_f32_u32_e32 v4, v4
; GFX11TRUE16-NEXT: v_cvt_f32_u32_e32 v0, v0
-; GFX11TRUE16-NEXT: v_cvt_f32_u32_e32 v1, v1
+; GFX11TRUE16-NEXT: v_sub_nc_u32_e32 v5, 32, v8
; GFX11TRUE16-NEXT: v_cvt_f32_u32_e32 v2, v2
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_ldexp_f32 v1, v4, v1
; GFX11TRUE16-NEXT: v_ldexp_f32 v0, v0, v3
-; GFX11TRUE16-NEXT: v_ldexp_f32 v1, v1, v7
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_ldexp_f32 v2, v2, v4
-; GFX11TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_bfe_u32 v4, v1, 16, 1
+; GFX11TRUE16-NEXT: v_ldexp_f32 v2, v2, v5
+; GFX11TRUE16-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v1
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX11TRUE16-NEXT: v_bfe_u32 v5, v2, 16, 1
+; GFX11TRUE16-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
+; GFX11TRUE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v0
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v1
; GFX11TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
; GFX11TRUE16-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
-; GFX11TRUE16-NEXT: v_add3_u32 v4, v4, v1, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v7, vcc_lo
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v6, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v4, v7, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v8, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v4, v6, vcc_lo
; GFX11TRUE16-NEXT: v_perm_b32 v0, v2, v0, 0x7060302
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_alignbit_b32 v1, v0, v1, 16
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_uitofp_v3i64_to_v3bf16:
@@ -40489,40 +40494,40 @@ define <3 x bfloat> @v_fma_v3bf16(<3 x bfloat> %a, <3 x bfloat> %b, <3 x bfloat>
; GFX11TRUE16-LABEL: v_fma_v3bf16:
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v4
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v2
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v0
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; GFX11TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
; GFX11TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX11TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11TRUE16-NEXT: v_dual_fmac_f32 v6, v8, v7 :: v_dual_lshlrev_b32 v5, 16, v5
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_fmac_f32_e32 v4, v0, v2
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_bfe_u32 v2, v4, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v4
-; GFX11TRUE16-NEXT: v_add3_u32 v2, v2, v4, 0x7fff
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_fmac_f32_e32 v6, v8, v7
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_dual_fmac_f32 v4, v0, v2 :: v_dual_lshlrev_b32 v3, 16, v3
+; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v6
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_4)
; GFX11TRUE16-NEXT: v_fmac_f32_e32 v5, v1, v3
; GFX11TRUE16-NEXT: v_bfe_u32 v1, v6, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v6
+; GFX11TRUE16-NEXT: v_bfe_u32 v2, v4, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v4
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX11TRUE16-NEXT: v_bfe_u32 v0, v5, 16, 1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v5
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX11TRUE16-NEXT: v_add3_u32 v1, v1, v6, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11TRUE16-NEXT: v_add3_u32 v2, v2, v4, 0x7fff
; GFX11TRUE16-NEXT: v_add3_u32 v0, v0, v5, 0x7fff
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc_lo
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v7, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v7, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v3, v0, v8, vcc_lo
-; GFX11TRUE16-NEXT: v_perm_b32 v0, v2, v1, 0x7060302
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v8, vcc_lo
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_alignbit_b32 v1, v0, v3, 16
+; GFX11TRUE16-NEXT: v_perm_b32 v0, v2, v1, 0x7060302
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v3.l
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_fma_v3bf16:
@@ -41439,8 +41444,9 @@ define <3 x bfloat> @v_fmuladd_v3bf16(<3 x bfloat> %a, <3 x bfloat> %b, <3 x bfl
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v0
; GFX11TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX11TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_dual_mul_f32 v0, v0, v2 :: v_dual_lshlrev_b32 v3, 16, v3
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11TRUE16-NEXT: v_mul_f32_e32 v0, v0, v2
; GFX11TRUE16-NEXT: v_bfe_u32 v8, v0, 16, 1
; GFX11TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v0
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
@@ -41472,27 +41478,27 @@ define <3 x bfloat> @v_fmuladd_v3bf16(<3 x bfloat> %a, <3 x bfloat> %b, <3 x bfl
; GFX11TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX11TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v2
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11TRUE16-NEXT: v_dual_add_f32 v0, v0, v4 :: v_dual_add_f32 v1, v1, v3
-; GFX11TRUE16-NEXT: v_bfe_u32 v3, v2, 16, 1
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1
; GFX11TRUE16-NEXT: v_bfe_u32 v5, v0, 16, 1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_bfe_u32 v4, v1, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v3, v3, v2, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_bfe_u32 v3, v1, 16, 1
; GFX11TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v1
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
; GFX11TRUE16-NEXT: v_add3_u32 v5, v5, v0, 0x7fff
-; GFX11TRUE16-NEXT: v_add3_u32 v4, v4, v1, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v3, v7, vcc_lo
+; GFX11TRUE16-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v6, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v7, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v5, v8, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11TRUE16-NEXT: v_perm_b32 v0, v0, v2, 0x7060302
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v4, v6, vcc_lo
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_alignbit_b32 v1, v0, v1, 16
+; GFX11TRUE16-NEXT: v_perm_b32 v0, v0, v2, 0x7060302
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_fmuladd_v3bf16:
More information about the llvm-commits
mailing list