[llvm] [AMDGPU][True16][MC] true16 for v_alignbit_b32 (PR #119409)
Brox Chen via llvm-commits
llvm-commits at lists.llvm.org
Wed Feb 19 12:48:17 PST 2025
https://github.com/broxigarchen updated https://github.com/llvm/llvm-project/pull/119409
>From 37aa213b7abe2f23b5722b285c47114b9a1898df Mon Sep 17 00:00:00 2001
From: guochen2 <guochen2 at amd.com>
Date: Tue, 10 Dec 2024 01:20:40 -0500
Subject: [PATCH 1/4] VOP3 v_alignbit_b32 mc support true16
---
llvm/lib/Target/AMDGPU/VOP3Instructions.td | 8 ++--
llvm/test/MC/AMDGPU/gfx11_asm_vop3.s | 11 +++--
llvm/test/MC/AMDGPU/gfx11_asm_vop3_dpp16.s | 42 +++++++++++++------
llvm/test/MC/AMDGPU/gfx11_asm_vop3_dpp8.s | 17 ++++++--
llvm/test/MC/AMDGPU/gfx12_asm_vop3.s | 3 ++
llvm/test/MC/AMDGPU/gfx12_asm_vop3_dpp16.s | 3 ++
llvm/test/MC/AMDGPU/gfx12_asm_vop3_dpp8.s | 3 ++
.../Disassembler/AMDGPU/gfx11_dasm_vop3.txt | 16 ++++++-
.../AMDGPU/gfx11_dasm_vop3_dpp16.txt | 31 +++++++++++---
.../AMDGPU/gfx11_dasm_vop3_dpp8.txt | 16 ++++++-
.../Disassembler/AMDGPU/gfx12_dasm_vop3.txt | 16 ++++++-
.../AMDGPU/gfx12_dasm_vop3_dpp16.txt | 36 +++++++++++++---
.../AMDGPU/gfx12_dasm_vop3_dpp8.txt | 21 ++++++++--
13 files changed, 180 insertions(+), 43 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/VOP3Instructions.td b/llvm/lib/Target/AMDGPU/VOP3Instructions.td
index afafc2ecccfaf..aae18e492e9bf 100644
--- a/llvm/lib/Target/AMDGPU/VOP3Instructions.td
+++ b/llvm/lib/Target/AMDGPU/VOP3Instructions.td
@@ -211,9 +211,11 @@ defm V_CUBEMA_F32 : VOP3Inst <"v_cubema_f32", VOP3_Profile<VOP_F32_F32_F32_F32>,
defm V_BFE_U32 : VOP3Inst <"v_bfe_u32", VOP3_Profile<VOP_I32_I32_I32_I32>, AMDGPUbfe_u32>;
defm V_BFE_I32 : VOP3Inst <"v_bfe_i32", VOP3_Profile<VOP_I32_I32_I32_I32>, AMDGPUbfe_i32>;
defm V_BFI_B32 : VOP3Inst <"v_bfi_b32", VOP3_Profile<VOP_I32_I32_I32_I32>, AMDGPUbfi>;
-defm V_ALIGNBIT_B32 : VOP3Inst <"v_alignbit_b32", VOP3_Profile<VOP_I32_I32_I32_I32>, fshr>;
+defm V_ALIGNBIT_B32 : VOP3Inst_t16_with_profiles <"v_alignbit_b32", VOP3_Profile<VOP_I32_I32_I32_I32>,
+ VOP3_Profile_True16<VOP_I32_I32_I32_I16, VOP3_OPSEL>,
+ VOP3_Profile_Fake16<VOP_I32_I32_I32_I16, VOP3_OPSEL>,
+ fshr, null_frag>;
-let True16Predicate = NotHasTrue16BitInsts in
defm V_ALIGNBYTE_B32 : VOP3Inst <"v_alignbyte_b32", VOP3_Profile<VOP_I32_I32_I32_I32>, int_amdgcn_alignbyte>;
let True16Predicate = UseRealTrue16Insts in
defm V_ALIGNBYTE_B32_t16 : VOP3Inst <"v_alignbyte_b32_t16", VOP3_Profile_True16<VOP_I32_I32_I32_I16, VOP3_OPSEL>>;
@@ -1714,7 +1716,7 @@ defm V_BFI_B32 : VOP3_Realtriple_gfx11_gfx12<0x212>;
defm V_FMA_F32 : VOP3_Realtriple_gfx11_gfx12<0x213>;
defm V_FMA_F64 : VOP3_Real_Base_gfx11_gfx12<0x214>;
defm V_LERP_U8 : VOP3_Realtriple_gfx11_gfx12<0x215>;
-defm V_ALIGNBIT_B32 : VOP3_Realtriple_gfx11_gfx12<0x216>;
+defm V_ALIGNBIT_B32 : VOP3_Realtriple_t16_and_fake16_gfx11_gfx12<0x216, "v_alignbit_b32">;
defm V_ALIGNBYTE_B32 : VOP3_Realtriple_t16_and_fake16_gfx11_gfx12<0x217, "v_alignbyte_b32">;
defm V_MULLIT_F32 : VOP3_Realtriple_gfx11_gfx12<0x218>;
defm V_MIN3_F32 : VOP3_Realtriple_gfx11<0x219>;
diff --git a/llvm/test/MC/AMDGPU/gfx11_asm_vop3.s b/llvm/test/MC/AMDGPU/gfx11_asm_vop3.s
index 857a1359b00d9..312fe00db2e35 100644
--- a/llvm/test/MC/AMDGPU/gfx11_asm_vop3.s
+++ b/llvm/test/MC/AMDGPU/gfx11_asm_vop3.s
@@ -416,11 +416,11 @@ v_alignbit_b32 v5, s1, v255, s3
v_alignbit_b32 v5, s105, s105, s105
// GFX11: v_alignbit_b32 v5, s105, s105, s105 ; encoding: [0x05,0x00,0x16,0xd6,0x69,0xd2,0xa4,0x01]
-v_alignbit_b32 v5, vcc_lo, ttmp15, v3
-// GFX11: v_alignbit_b32 v5, vcc_lo, ttmp15, v3 ; encoding: [0x05,0x00,0x16,0xd6,0x6a,0xf6,0x0c,0x04]
+v_alignbit_b32 v5, vcc_lo, ttmp15, v3.l
+// GFX11: v_alignbit_b32 v5, vcc_lo, ttmp15, v3.l ; encoding: [0x05,0x00,0x16,0xd6,0x6a,0xf6,0x0c,0x04]
-v_alignbit_b32 v5, vcc_hi, 0xaf123456, v255
-// GFX11: v_alignbit_b32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x16,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
+v_alignbit_b32 v5, vcc_hi, 0xaf123456, v255.l
+// GFX11: v_alignbit_b32 v5, vcc_hi, 0xaf123456, v255.l ; encoding: [0x05,0x00,0x16,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
v_alignbit_b32 v5, ttmp15, src_scc, ttmp15
// GFX11: v_alignbit_b32 v5, ttmp15, src_scc, ttmp15 ; encoding: [0x05,0x00,0x16,0xd6,0x7b,0xfa,0xed,0x01]
@@ -449,6 +449,9 @@ v_alignbit_b32 v5, src_scc, vcc_lo, -1
v_alignbit_b32 v255, 0xaf123456, vcc_hi, null
// GFX11: v_alignbit_b32 v255, 0xaf123456, vcc_hi, null ; encoding: [0xff,0x00,0x16,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
+v_alignbit_b32 v5, vcc_hi, 0xaf123456, v255.h
+// GFX11: [0x05,0x20,0x16,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
+
v_alignbyte_b32 v5, v1, v2, s3
// GFX11: v_alignbyte_b32 v5, v1, v2, s3 ; encoding: [0x05,0x00,0x17,0xd6,0x01,0x05,0x0e,0x00]
diff --git a/llvm/test/MC/AMDGPU/gfx11_asm_vop3_dpp16.s b/llvm/test/MC/AMDGPU/gfx11_asm_vop3_dpp16.s
index 1864996b26028..faaf2581d9d07 100644
--- a/llvm/test/MC/AMDGPU/gfx11_asm_vop3_dpp16.s
+++ b/llvm/test/MC/AMDGPU/gfx11_asm_vop3_dpp16.s
@@ -321,22 +321,22 @@ v_add_nc_u16_e64_dpp v5.l, v1.l, v2.h row_xmask:0 row_mask:0x1 bank_mask:0x3 bou
v_add_nc_u16_e64_dpp v255.h, v255.l, v255.l clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
// GFX11: v_add_nc_u16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0xc0,0x03,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x05,0x30]
-v_alignbit_b32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0]
-// GFX11: v_alignbit_b32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
+v_alignbit_b32_e64_dpp v5, v1, v2, v3.l quad_perm:[3,2,1,0]
+// GFX11: v_alignbit_b32_e64_dpp v5, v1, v2, v3.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
-v_alignbit_b32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3]
-// GFX11: v_alignbit_b32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
+v_alignbit_b32_e64_dpp v5, v1, v2, v3.l quad_perm:[0,1,2,3]
+// GFX11: v_alignbit_b32_e64_dpp v5, v1, v2, v3.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
-v_alignbit_b32_e64_dpp v5, v1, v2, v3 row_mirror
-// GFX11: v_alignbit_b32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
+v_alignbit_b32_e64_dpp v5, v1, v2, v3.l row_mirror row_mask:0xf bank_mask:0xf
+// GFX11: v_alignbit_b32_e64_dpp v5, v1, v2, v3.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
-v_alignbit_b32_e64_dpp v5, v1, v2, v3 row_half_mirror
-// GFX11: v_alignbit_b32_e64_dpp v5, v1, v2, v3 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x41,0x01,0xff]
+v_alignbit_b32_e64_dpp v5, v1, v2, v3.l row_half_mirror row_mask:0xf bank_mask:0xf
+// GFX11: v_alignbit_b32_e64_dpp v5, v1, v2, v3.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x41,0x01,0xff]
-v_alignbit_b32_e64_dpp v5, v1, v2, v255 row_shl:1
-// GFX11: v_alignbit_b32_e64_dpp v5, v1, v2, v255 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x01,0x01,0xff]
+v_alignbit_b32_e64_dpp v5, v1, v2, v255.l row_shl:1 row_mask:0xf bank_mask:0xf
+// GFX11: v_alignbit_b32_e64_dpp v5, v1, v2, v255.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x01,0x01,0xff]
-v_alignbit_b32_e64_dpp v5, v1, v2, s105 row_shl:15
+v_alignbit_b32_e64_dpp v5, v1, v2, s105 row_shl:15 row_mask:0xf bank_mask:0xf
// GFX11: v_alignbit_b32_e64_dpp v5, v1, v2, s105 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x0f,0x01,0xff]
v_alignbit_b32_e64_dpp v5, v1, v2, vcc_hi row_shr:1
@@ -345,7 +345,7 @@ v_alignbit_b32_e64_dpp v5, v1, v2, vcc_hi row_shr:1
v_alignbit_b32_e64_dpp v5, v1, v2, vcc_lo row_shr:15
// GFX11: v_alignbit_b32_e64_dpp v5, v1, v2, vcc_lo row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x1f,0x01,0xff]
-v_alignbit_b32_e64_dpp v5, v1, v2, ttmp15 row_ror:1
+v_alignbit_b32_e64_dpp v5, v1, v2, ttmp15 row_ror:1 row_mask:0xf bank_mask:0xf
// GFX11: v_alignbit_b32_e64_dpp v5, v1, v2, ttmp15 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0xee,0x01,0x01,0x21,0x01,0xff]
v_alignbit_b32_e64_dpp v5, v1, v2, exec_hi row_ror:15
@@ -363,6 +363,24 @@ v_alignbit_b32_e64_dpp v5, v1, v2, -1 row_xmask:0 row_mask:0x1 bank_mask:0x3 bou
v_alignbit_b32_e64_dpp v255, v255, v255, src_scc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
// GFX11: v_alignbit_b32_e64_dpp v255, v255, v255, src_scc row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x00,0x16,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x05,0x30]
+v_alignbit_b32_e64_dpp v5, v1, v2, v255.l row_mirror
+// GFX11: v_alignbit_b32_e64_dpp v5, v1, v2, v255.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x40,0x01,0xff]
+
+v_alignbit_b32_e64_dpp v5, v1, v2, s3 row_half_mirror
+// GFX11: v_alignbit_b32_e64_dpp v5, v1, v2, s3 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0x0e,0x00,0x01,0x41,0x01,0xff]
+
+v_alignbit_b32_e64_dpp v5, v1, v2, s105 row_shl:1
+// GFX11: v_alignbit_b32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
+
+v_alignbit_b32_e64_dpp v5, v1, v2, ttmp15 row_shl:15
+// GFX11: v_alignbit_b32_e64_dpp v5, v1, v2, ttmp15 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0xee,0x01,0x01,0x0f,0x01,0xff]
+
+v_alignbit_b32_e64_dpp v5, v1, v2, m0 row_ror:1
+// GFX11: v_alignbit_b32_e64_dpp v5, v1, v2, m0 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0xf6,0x01,0x01,0x21,0x01,0xff]
+
+v_alignbit_b32_e64_dpp v5, v1, v2, v255.h row_mirror
+// GFX11: [0x05,0x20,0x16,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x40,0x01,0xff]
+
v_alignbyte_b32_e64_dpp v5, v1, v2, v3.l quad_perm:[3,2,1,0]
// GFX11: v_alignbyte_b32_e64_dpp v5, v1, v2, v3.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
diff --git a/llvm/test/MC/AMDGPU/gfx11_asm_vop3_dpp8.s b/llvm/test/MC/AMDGPU/gfx11_asm_vop3_dpp8.s
index 300e5ef22f5ae..1368bec3c5889 100644
--- a/llvm/test/MC/AMDGPU/gfx11_asm_vop3_dpp8.s
+++ b/llvm/test/MC/AMDGPU/gfx11_asm_vop3_dpp8.s
@@ -154,11 +154,11 @@ v_add_nc_u16_e64_dpp v5.l, v1.l, v2.h dpp8:[7,6,5,4,3,2,1,0] fi:1
v_add_nc_u16_e64_dpp v255.h, v255.l, v255.l clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
// GFX11: v_add_nc_u16_e64_dpp v255.h, v255.l, v255.l op_sel:[0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0xc0,0x03,0xd7,0xe9,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
-v_alignbit_b32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_alignbit_b32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x16,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
+v_alignbit_b32_e64_dpp v5, v1, v2, v3.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_alignbit_b32_e64_dpp v5, v1, v2, v3.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x16,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
-v_alignbit_b32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0]
-// GFX11: v_alignbit_b32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x16,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+v_alignbit_b32_e64_dpp v5, v1, v2, v255.l dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_alignbit_b32_e64_dpp v5, v1, v2, v255.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x16,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
v_alignbit_b32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0]
// GFX11: v_alignbit_b32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x16,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
@@ -187,6 +187,15 @@ v_alignbit_b32_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] fi:1
v_alignbit_b32_e64_dpp v255, v255, v255, src_scc dpp8:[0,0,0,0,0,0,0,0] fi:0
// GFX11: v_alignbit_b32_e64_dpp v255, v255, v255, src_scc dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x00,0x16,0xd6,0xe9,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
+v_alignbit_b32_e64_dpp v5, v1, v2, s3 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_alignbit_b32_e64_dpp v5, v1, v2, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x16,0xd6,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05]
+
+v_alignbit_b32_e64_dpp v5, v1, v2, m0 dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: v_alignbit_b32_e64_dpp v5, v1, v2, m0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x16,0xd6,0xe9,0x04,0xf6,0x01,0x01,0x77,0x39,0x05]
+
+v_alignbit_b32_e64_dpp v5, v1, v2, v255.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX11: [0x05,0x20,0x16,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+
v_alignbyte_b32_e64_dpp v5, v1, v2, v3.l dpp8:[7,6,5,4,3,2,1,0]
// GFX11: v_alignbyte_b32_e64_dpp v5, v1, v2, v3.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x17,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vop3.s b/llvm/test/MC/AMDGPU/gfx12_asm_vop3.s
index 72d201e060df7..afd4af92b4f27 100644
--- a/llvm/test/MC/AMDGPU/gfx12_asm_vop3.s
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_vop3.s
@@ -407,6 +407,9 @@ v_alignbit_b32 v5, src_scc, vcc_lo, -1
v_alignbit_b32 v255, 0xaf123456, vcc_hi, null
// GFX12: v_alignbit_b32 v255, 0xaf123456, vcc_hi, null ; encoding: [0xff,0x00,0x16,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
+v_alignbit_b32 v5, vcc_hi, 0xaf123456, v255.h
+// GFX12: v_alignbit_b32 v5, vcc_hi, 0xaf123456, v255.h op_sel:[0,0,1,0] ; encoding: [0x05,0x20,0x16,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
+
v_alignbyte_b32 v5, v1, v2, s3
// GFX12: v_alignbyte_b32 v5, v1, v2, s3 ; encoding: [0x05,0x00,0x17,0xd6,0x01,0x05,0x0e,0x00]
diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vop3_dpp16.s b/llvm/test/MC/AMDGPU/gfx12_asm_vop3_dpp16.s
index 08be9225890c2..cfd01ee02aa7e 100644
--- a/llvm/test/MC/AMDGPU/gfx12_asm_vop3_dpp16.s
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_vop3_dpp16.s
@@ -437,6 +437,9 @@ v_alignbit_b32_e64_dpp v5, v1, v2, -1 row_xmask:0 row_mask:0x1 bank_mask:0x3 bou
v_alignbit_b32_e64_dpp v255, v255, v255, src_scc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
// GFX12: v_alignbit_b32_e64_dpp v255, v255, v255, src_scc row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x00,0x16,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x05,0x30]
+v_alignbit_b32_e64_dpp v5, v1, v2, v255.h row_mirror
+// GFX12: v_alignbit_b32_e64_dpp v5, v1, v2, v255.h op_sel:[0,0,1,0] row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x20,0x16,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x40,0x01,0xff]
+
v_alignbyte_b32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0]
// GFX12: v_alignbyte_b32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vop3_dpp8.s b/llvm/test/MC/AMDGPU/gfx12_asm_vop3_dpp8.s
index 0523b0fe8c9a4..b6d395c8ede89 100644
--- a/llvm/test/MC/AMDGPU/gfx12_asm_vop3_dpp8.s
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_vop3_dpp8.s
@@ -249,6 +249,9 @@ v_alignbit_b32_e64_dpp v5, v1, v2, -1 dpp8:[7,6,5,4,3,2,1,0] fi:1
v_alignbit_b32_e64_dpp v255, v255, v255, src_scc dpp8:[0,0,0,0,0,0,0,0] fi:0
// GFX12: v_alignbit_b32_e64_dpp v255, v255, v255, src_scc dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x00,0x16,0xd6,0xe9,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
+v_alignbit_b32_e64_dpp v5, v1, v2, v255.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: v_alignbit_b32_e64_dpp v5, v1, v2, v255.h op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x20,0x16,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+
v_alignbyte_b32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0]
// GFX12: v_alignbyte_b32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x17,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vop3.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vop3.txt
index b74128b21f563..1b29ddcb20a91 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vop3.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vop3.txt
@@ -463,10 +463,16 @@
# GFX11: v_alignbit_b32 v5, s105, s105, s105 ; encoding: [0x05,0x00,0x16,0xd6,0x69,0xd2,0xa4,0x01]
0x05,0x00,0x16,0xd6,0x6a,0xf6,0x0c,0x04
-# GFX11: v_alignbit_b32 v5, vcc_lo, ttmp15, v3 ; encoding: [0x05,0x00,0x16,0xd6,0x6a,0xf6,0x0c,0x04]
+# W32-REAL16: v_alignbit_b32 v5, vcc_lo, ttmp15, v3.l ; encoding: [0x05,0x00,0x16,0xd6,0x6a,0xf6,0x0c,0x04]
+# W32-FAKE16: v_alignbit_b32 v5, vcc_lo, ttmp15, v3 ; encoding: [0x05,0x00,0x16,0xd6,0x6a,0xf6,0x0c,0x04]
+# W64-REAL16: v_alignbit_b32 v5, vcc_lo, ttmp15, v3.l ; encoding: [0x05,0x00,0x16,0xd6,0x6a,0xf6,0x0c,0x04]
+# W64-FAKE16: v_alignbit_b32 v5, vcc_lo, ttmp15, v3 ; encoding: [0x05,0x00,0x16,0xd6,0x6a,0xf6,0x0c,0x04]
0x05,0x00,0x16,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf
-# GFX11: v_alignbit_b32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x16,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
+# W32-REAL16: v_alignbit_b32 v5, vcc_hi, 0xaf123456, v255.l ; encoding: [0x05,0x00,0x16,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
+# W32-FAKE16: v_alignbit_b32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x16,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
+# W64-REAL16: v_alignbit_b32 v5, vcc_hi, 0xaf123456, v255.l ; encoding: [0x05,0x00,0x16,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
+# W64-FAKE16: v_alignbit_b32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x16,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
0x05,0x00,0x16,0xd6,0x7b,0xfa,0xed,0x01
# GFX11: v_alignbit_b32 v5, ttmp15, src_scc, ttmp15 ; encoding: [0x05,0x00,0x16,0xd6,0x7b,0xfa,0xed,0x01]
@@ -495,6 +501,12 @@
0xff,0x00,0x16,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf
# GFX11: v_alignbit_b32 v255, 0xaf123456, vcc_hi, null ; encoding: [0xff,0x00,0x16,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
+0x05,0x20,0x16,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf
+# W32-REAL16: v_alignbit_b32 v5, vcc_hi, 0xaf123456, v255.h op_sel:[0,0,1,0] ; encoding: [0x05,0x20,0x16,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
+# W32-FAKE16: v_alignbit_b32 v5, vcc_hi, 0xaf123456, v255 op_sel:[0,0,1,0] ; encoding: [0x05,0x20,0x16,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
+# W64-REAL16: v_alignbit_b32 v5, vcc_hi, 0xaf123456, v255.h op_sel:[0,0,1,0] ; encoding: [0x05,0x20,0x16,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
+# W64-FAKE16: v_alignbit_b32 v5, vcc_hi, 0xaf123456, v255 op_sel:[0,0,1,0] ; encoding: [0x05,0x20,0x16,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
+
0x05,0x00,0x17,0xd6,0x01,0x05,0x0e,0x00
# GFX11: v_alignbyte_b32 v5, v1, v2, s3 ; encoding: [0x05,0x00,0x17,0xd6,0x01,0x05,0x0e,0x00]
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vop3_dpp16.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vop3_dpp16.txt
index 8e7122b902326..0f47d33761475 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vop3_dpp16.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vop3_dpp16.txt
@@ -186,19 +186,34 @@
# GFX11: v_add_nc_i32_e64_dpp v255, v255, v255 clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x80,0x26,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
0x05,0x00,0x16,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
-# GFX11: v_alignbit_b32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
+# W32-REAL16: v_alignbit_b32_e64_dpp v5, v1, v2, v3.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
+# W32-FAKE16: v_alignbit_b32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
+# W64-REAL16: v_alignbit_b32_e64_dpp v5, v1, v2, v3.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
+# W64-FAKE16: v_alignbit_b32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
0x05,0x00,0x16,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
-# GFX11: v_alignbit_b32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
+# W32-REAL16: v_alignbit_b32_e64_dpp v5, v1, v2, v3.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
+# W32-FAKE16: v_alignbit_b32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
+# W64-REAL16: v_alignbit_b32_e64_dpp v5, v1, v2, v3.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
+# W64-FAKE16: v_alignbit_b32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
0x05,0x00,0x16,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff
-# GFX11: v_alignbit_b32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
+# W32-REAL16: v_alignbit_b32_e64_dpp v5, v1, v2, v3.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
+# W32-FAKE16: v_alignbit_b32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
+# W64-REAL16: v_alignbit_b32_e64_dpp v5, v1, v2, v3.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
+# W64-FAKE16: v_alignbit_b32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
0x05,0x00,0x16,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x41,0x01,0xff
-# GFX11: v_alignbit_b32_e64_dpp v5, v1, v2, v3 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x41,0x01,0xff]
+# W32-REAL16: v_alignbit_b32_e64_dpp v5, v1, v2, v3.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x41,0x01,0xff]
+# W32-FAKE16: v_alignbit_b32_e64_dpp v5, v1, v2, v3 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x41,0x01,0xff]
+# W64-REAL16: v_alignbit_b32_e64_dpp v5, v1, v2, v3.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x41,0x01,0xff]
+# W64-FAKE16: v_alignbit_b32_e64_dpp v5, v1, v2, v3 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x41,0x01,0xff]
0x05,0x00,0x16,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x01,0x01,0xff
-# GFX11: v_alignbit_b32_e64_dpp v5, v1, v2, v255 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x01,0x01,0xff]
+# W32-REAL16: v_alignbit_b32_e64_dpp v5, v1, v2, v255.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x01,0x01,0xff]
+# W32-FAKE16: v_alignbit_b32_e64_dpp v5, v1, v2, v255 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x01,0x01,0xff]
+# W64-REAL16: v_alignbit_b32_e64_dpp v5, v1, v2, v255.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x01,0x01,0xff]
+# W64-FAKE16: v_alignbit_b32_e64_dpp v5, v1, v2, v255 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x01,0x01,0xff]
0x05,0x00,0x16,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x0f,0x01,0xff
# GFX11: v_alignbit_b32_e64_dpp v5, v1, v2, s105 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x0f,0x01,0xff]
@@ -227,6 +242,12 @@
0xff,0x00,0x16,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30
# GFX11: v_alignbit_b32_e64_dpp v255, v255, v255, src_scc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x16,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
+0x05,0x20,0x16,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x40,0x01,0xff
+# W32-REAL16: v_alignbit_b32_e64_dpp v5, v1, v2, v255.h op_sel:[0,0,1,0] row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x20,0x16,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x40,0x01,0xff]
+# W32-FAKE16: v_alignbit_b32_e64_dpp v5, v1, v2, v255 op_sel:[0,0,1,0] row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x20,0x16,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x40,0x01,0xff]
+# W64-REAL16: v_alignbit_b32_e64_dpp v5, v1, v2, v255.h op_sel:[0,0,1,0] row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x20,0x16,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x40,0x01,0xff]
+# W64-FAKE16: v_alignbit_b32_e64_dpp v5, v1, v2, v255 op_sel:[0,0,1,0] row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x20,0x16,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x40,0x01,0xff]
+
0x05,0x00,0x17,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
# W32-REAL16: v_alignbyte_b32_e64_dpp v5, v1, v2, v3.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
# W32-FAKE16: v_alignbyte_b32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vop3_dpp8.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vop3_dpp8.txt
index f67eb32385407..67bafd71d01e5 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vop3_dpp8.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vop3_dpp8.txt
@@ -102,10 +102,16 @@
# GFX11: v_add_nc_i32_e64_dpp v255, v255, v255 clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x80,0x26,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
0x05,0x00,0x16,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
-# GFX11: v_alignbit_b32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x16,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_alignbit_b32_e64_dpp v5, v1, v2, v3.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x16,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_alignbit_b32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x16,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_alignbit_b32_e64_dpp v5, v1, v2, v3.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x16,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_alignbit_b32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x16,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
0x05,0x00,0x16,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
-# GFX11: v_alignbit_b32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x16,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_alignbit_b32_e64_dpp v5, v1, v2, v255.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x16,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_alignbit_b32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x16,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_alignbit_b32_e64_dpp v5, v1, v2, v255.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x16,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_alignbit_b32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x16,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
0x05,0x00,0x16,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
# GFX11: v_alignbit_b32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x16,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
@@ -134,6 +140,12 @@
0xff,0x00,0x16,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00
# GFX11: v_alignbit_b32_e64_dpp v255, v255, v255, src_scc dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x16,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
+0x05,0x20,0x16,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# W32-REAL16: v_alignbit_b32_e64_dpp v5, v1, v2, v255.h op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x20,0x16,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_alignbit_b32_e64_dpp v5, v1, v2, v255 op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x20,0x16,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_alignbit_b32_e64_dpp v5, v1, v2, v255.h op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x20,0x16,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_alignbit_b32_e64_dpp v5, v1, v2, v255 op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x20,0x16,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+
0x05,0x00,0x17,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
# W32-REAL16: v_alignbyte_b32_e64_dpp v5, v1, v2, v3.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x17,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
# W32-FAKE16: v_alignbyte_b32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x17,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3.txt
index 3d5e78c86bc22..e7d92af7511f5 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3.txt
@@ -427,10 +427,16 @@
# GFX12: v_alignbit_b32 v5, s105, s105, s105 ; encoding: [0x05,0x00,0x16,0xd6,0x69,0xd2,0xa4,0x01]
0x05,0x00,0x16,0xd6,0x6a,0xf6,0x0c,0x04
-# GFX12: v_alignbit_b32 v5, vcc_lo, ttmp15, v3 ; encoding: [0x05,0x00,0x16,0xd6,0x6a,0xf6,0x0c,0x04]
+# W32-REAL16: v_alignbit_b32 v5, vcc_lo, ttmp15, v3.l ; encoding: [0x05,0x00,0x16,0xd6,0x6a,0xf6,0x0c,0x04]
+# W32-FAKE16: v_alignbit_b32 v5, vcc_lo, ttmp15, v3 ; encoding: [0x05,0x00,0x16,0xd6,0x6a,0xf6,0x0c,0x04]
+# W64-REAL16: v_alignbit_b32 v5, vcc_lo, ttmp15, v3.l ; encoding: [0x05,0x00,0x16,0xd6,0x6a,0xf6,0x0c,0x04]
+# W64-FAKE16: v_alignbit_b32 v5, vcc_lo, ttmp15, v3 ; encoding: [0x05,0x00,0x16,0xd6,0x6a,0xf6,0x0c,0x04]
0x05,0x00,0x16,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf
-# GFX12: v_alignbit_b32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x16,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
+# W32-REAL16: v_alignbit_b32 v5, vcc_hi, 0xaf123456, v255.l ; encoding: [0x05,0x00,0x16,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
+# W32-FAKE16: v_alignbit_b32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x16,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
+# W64-REAL16: v_alignbit_b32 v5, vcc_hi, 0xaf123456, v255.l ; encoding: [0x05,0x00,0x16,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
+# W64-FAKE16: v_alignbit_b32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x16,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
0x05,0x00,0x16,0xd6,0x7b,0xfa,0xed,0x01
# GFX12: v_alignbit_b32 v5, ttmp15, src_scc, ttmp15 ; encoding: [0x05,0x00,0x16,0xd6,0x7b,0xfa,0xed,0x01]
@@ -459,6 +465,12 @@
0xff,0x00,0x16,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf
# GFX12: v_alignbit_b32 v255, 0xaf123456, vcc_hi, null ; encoding: [0xff,0x00,0x16,0xd6,0xff,0xd6,0xf0,0x01,0x56,0x34,0x12,0xaf]
+0x05,0x20,0x16,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf
+# W32-REAL16: v_alignbit_b32 v5, vcc_hi, 0xaf123456, v255.h op_sel:[0,0,1,0] ; encoding: [0x05,0x20,0x16,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
+# W32-FAKE16: v_alignbit_b32 v5, vcc_hi, 0xaf123456, v255 op_sel:[0,0,1,0] ; encoding: [0x05,0x20,0x16,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
+# W64-REAL16: v_alignbit_b32 v5, vcc_hi, 0xaf123456, v255.h op_sel:[0,0,1,0] ; encoding: [0x05,0x20,0x16,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
+# W64-FAKE16: v_alignbit_b32 v5, vcc_hi, 0xaf123456, v255 op_sel:[0,0,1,0] ; encoding: [0x05,0x20,0x16,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
+
0x05,0x00,0x17,0xd6,0x01,0x05,0x0e,0x00
# GFX12: v_alignbyte_b32 v5, v1, v2, s3 ; encoding: [0x05,0x00,0x17,0xd6,0x01,0x05,0x0e,0x00]
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3_dpp16.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3_dpp16.txt
index c63c09b6b24c5..a84e0e1734ff5 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3_dpp16.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3_dpp16.txt
@@ -195,22 +195,40 @@
# GFX12: v_add_nc_i32_e64_dpp v255, v255, v255 clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x80,0x26,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
0x05,0x00,0x16,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
-# GFX12: v_alignbit_b32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
+# W32-REAL16: v_alignbit_b32_e64_dpp v5, v1, v2, v3.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
+# W32-FAKE16: v_alignbit_b32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
+# W64-REAL16: v_alignbit_b32_e64_dpp v5, v1, v2, v3.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
+# W64-FAKE16: v_alignbit_b32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
0x05,0x00,0x16,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff
-# GFX12: v_alignbit_b32_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
+# W32-REAL16: v_alignbit_b32_e64_dpp v5, v1, s3, v3.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
+# W32-FAKE16: v_alignbit_b32_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
+# W64-REAL16: v_alignbit_b32_e64_dpp v5, v1, s3, v3.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
+# W64-FAKE16: v_alignbit_b32_e64_dpp v5, v1, s3, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x06,0x0c,0x04,0x01,0x1b,0x00,0xff]
0x05,0x00,0x16,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
-# GFX12: v_alignbit_b32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
+# W32-REAL16: v_alignbit_b32_e64_dpp v5, v1, v2, v3.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
+# W32-FAKE16: v_alignbit_b32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
+# W64-REAL16: v_alignbit_b32_e64_dpp v5, v1, v2, v3.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
+# W64-FAKE16: v_alignbit_b32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
0x05,0x00,0x16,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff
-# GFX12: v_alignbit_b32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
+# W32-REAL16: v_alignbit_b32_e64_dpp v5, v1, v2, v3.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
+# W32-FAKE16: v_alignbit_b32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
+# W64-REAL16: v_alignbit_b32_e64_dpp v5, v1, v2, v3.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
+# W64-FAKE16: v_alignbit_b32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
0x05,0x00,0x16,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x41,0x01,0xff
-# GFX12: v_alignbit_b32_e64_dpp v5, v1, v2, v3 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x41,0x01,0xff]
+# W32-REAL16: v_alignbit_b32_e64_dpp v5, v1, v2, v3.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x41,0x01,0xff]
+# W32-FAKE16: v_alignbit_b32_e64_dpp v5, v1, v2, v3 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x41,0x01,0xff]
+# W64-REAL16: v_alignbit_b32_e64_dpp v5, v1, v2, v3.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x41,0x01,0xff]
+# W64-FAKE16: v_alignbit_b32_e64_dpp v5, v1, v2, v3 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x41,0x01,0xff]
0x05,0x00,0x16,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x01,0x01,0xff
-# GFX12: v_alignbit_b32_e64_dpp v5, v1, v2, v255 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x01,0x01,0xff]
+# W32-REAL16: v_alignbit_b32_e64_dpp v5, v1, v2, v255.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x01,0x01,0xff]
+# W32-FAKE16: v_alignbit_b32_e64_dpp v5, v1, v2, v255 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x01,0x01,0xff]
+# W64-REAL16: v_alignbit_b32_e64_dpp v5, v1, v2, v255.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x01,0x01,0xff]
+# W64-FAKE16: v_alignbit_b32_e64_dpp v5, v1, v2, v255 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x01,0x01,0xff]
0x05,0x00,0x16,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x0f,0x01,0xff
# GFX12: v_alignbit_b32_e64_dpp v5, v1, v2, s105 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x16,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x0f,0x01,0xff]
@@ -239,6 +257,12 @@
0xff,0x00,0x16,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30
# GFX12: v_alignbit_b32_e64_dpp v255, v255, v255, src_scc row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x00,0x16,0xd6,0xfa,0xfe,0xf7,0x03,0xff,0x6f,0x0d,0x30]
+0x05,0x20,0x16,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x40,0x01,0xff
+# W32-REAL16: v_alignbit_b32_e64_dpp v5, v1, v2, v255.h op_sel:[0,0,1,0] row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x20,0x16,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x40,0x01,0xff]
+# W32-FAKE16: v_alignbit_b32_e64_dpp v5, v1, v2, v255 op_sel:[0,0,1,0] row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x20,0x16,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x40,0x01,0xff]
+# W64-REAL16: v_alignbit_b32_e64_dpp v5, v1, v2, v255.h op_sel:[0,0,1,0] row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x20,0x16,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x40,0x01,0xff]
+# W64-FAKE16: v_alignbit_b32_e64_dpp v5, v1, v2, v255 op_sel:[0,0,1,0] row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x20,0x16,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x40,0x01,0xff]
+
0x05,0x00,0x17,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
# W32-REAL16: v_alignbyte_b32_e64_dpp v5, v1, v2, v3.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
# W32-FAKE16: v_alignbyte_b32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x17,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3_dpp8.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3_dpp8.txt
index 4182cd93f813c..e4bef0a17385c 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3_dpp8.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3_dpp8.txt
@@ -111,13 +111,22 @@
# GFX12: v_add_nc_i32_e64_dpp v255, v255, v255 clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x80,0x26,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
0x05,0x00,0x16,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
-# GFX12: v_alignbit_b32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x16,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_alignbit_b32_e64_dpp v5, v1, v2, v3.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x16,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_alignbit_b32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x16,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_alignbit_b32_e64_dpp v5, v1, v2, v3.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x16,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_alignbit_b32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x16,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
0x05,0x00,0x16,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05
-# GFX12: v_alignbit_b32_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x16,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_alignbit_b32_e64_dpp v5, v1, s3, v3.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x16,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_alignbit_b32_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x16,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_alignbit_b32_e64_dpp v5, v1, s3, v3.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x16,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_alignbit_b32_e64_dpp v5, v1, s3, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x16,0xd6,0xe9,0x06,0x0c,0x04,0x01,0x77,0x39,0x05]
0x05,0x00,0x16,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
-# GFX12: v_alignbit_b32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x16,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+# W32-REAL16: v_alignbit_b32_e64_dpp v5, v1, v2, v255.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x16,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_alignbit_b32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x16,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_alignbit_b32_e64_dpp v5, v1, v2, v255.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x16,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_alignbit_b32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x16,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
0x05,0x00,0x16,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
# GFX12: v_alignbit_b32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x16,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
@@ -146,6 +155,12 @@
0xff,0x00,0x16,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00
# GFX12: v_alignbit_b32_e64_dpp v255, v255, v255, src_scc dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x00,0x16,0xd6,0xea,0xfe,0xf7,0x03,0xff,0x00,0x00,0x00]
+0x05,0x20,0x16,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+# W32-REAL16: v_alignbit_b32_e64_dpp v5, v1, v2, v255.h op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x20,0x16,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+# W32-FAKE16: v_alignbit_b32_e64_dpp v5, v1, v2, v255 op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x20,0x16,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+# W64-REAL16: v_alignbit_b32_e64_dpp v5, v1, v2, v255.h op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x20,0x16,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+# W64-FAKE16: v_alignbit_b32_e64_dpp v5, v1, v2, v255 op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x20,0x16,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+
0x05,0x00,0x17,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
# W32-REAL16: v_alignbyte_b32_e64_dpp v5, v1, v2, v3.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x17,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
# W32-FAKE16: v_alignbyte_b32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x17,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
>From 84a90b4cc7f3907e5166e0df54ecf4e481e555a9 Mon Sep 17 00:00:00 2001
From: guochen2 <guochen2 at amd.com>
Date: Tue, 10 Dec 2024 14:40:46 -0500
Subject: [PATCH 2/4] update codegen pattern for fake16 flow
---
llvm/lib/Target/AMDGPU/SIInstructions.td | 102 ++++
llvm/lib/Target/AMDGPU/VOP3Instructions.td | 9 +-
.../AMDGPU/GlobalISel/inst-select-fshr.mir | 11 +-
llvm/test/CodeGen/AMDGPU/bf16.ll | 502 +++++++++---------
4 files changed, 371 insertions(+), 253 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIInstructions.td b/llvm/lib/Target/AMDGPU/SIInstructions.td
index 3faf0795157dc..e011ef7c22fa4 100644
--- a/llvm/lib/Target/AMDGPU/SIInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SIInstructions.td
@@ -2497,6 +2497,7 @@ def : AMDGPUPat <
$src1), sub1)
>;
+let True16Predicate = NotHasTrue16BitInsts in {
def : ROTRPattern <V_ALIGNBIT_B32_e64>;
def : GCNPat<(i32 (trunc (srl i64:$src0, (and i32:$src1, (i32 31))))),
@@ -2506,6 +2507,42 @@ def : GCNPat<(i32 (trunc (srl i64:$src0, (and i32:$src1, (i32 31))))),
def : GCNPat<(i32 (trunc (srl i64:$src0, (i32 ShiftAmt32Imm:$src1)))),
(V_ALIGNBIT_B32_e64 (i32 (EXTRACT_SUBREG (i64 $src0), sub1)),
(i32 (EXTRACT_SUBREG (i64 $src0), sub0)), $src1)>;
+} //end True16Predicate = NotHasTrue16BitInsts
+
+let True16Predicate = UseFakeTrue16Insts in {
+def ROTRPattern_fake16 : GCNPat <
+ (rotr i32:$src0, i32:$src1),
+ (V_ALIGNBIT_B32_fake16_e64 /* src0_modifiers */ 0, $src0,
+ /* src1_modifiers */ 0, $src0,
+ /* src2_modifiers */ 0,
+ $src1, /* clamp */ 0, /* op_sel */ 0)
+>;
+
+def : GCNPat<(i32 (trunc (srl i64:$src0, (and i32:$src1, (i32 31))))),
+ (V_ALIGNBIT_B32_fake16_e64 0, /* src0_modifiers */
+ (i32 (EXTRACT_SUBREG (i64 $src0), sub1)),
+ 0, /* src1_modifiers */
+ (i32 (EXTRACT_SUBREG (i64 $src0), sub0)),
+ 0, /* src2_modifiers */
+ $src1, /* clamp */ 0, /* op_sel */ 0)
+>;
+
+def : GCNPat<(i32 (trunc (srl i64:$src0, (i32 ShiftAmt32Imm:$src1)))),
+ (V_ALIGNBIT_B32_fake16_e64 0, /* src0_modifiers */
+ (i32 (EXTRACT_SUBREG (i64 $src0), sub1)),
+ 0, /* src1_modifiers */
+ (i32 (EXTRACT_SUBREG (i64 $src0), sub0)),
+ 0, /* src2_modifiers */
+ $src1, /* clamp */ 0, /* op_sel */ 0)
+>;
+
+def : GCNPat<(fshr i32:$src0, i32:$src1, i32:$src2),
+ (V_ALIGNBIT_B32_fake16_e64 /* src0_modifiers */ 0, $src0,
+ /* src1_modifiers */ 0, $src1,
+ /* src2_modifiers */ 0,
+ $src2, /* clamp */ 0, /* op_sel */ 0)
+>;
+} // end True16Predicate = UseFakeTrue16Insts
/********** ====================== **********/
/********** Indirect addressing **********/
@@ -3013,6 +3050,7 @@ def : GCNPat <
(i32 (EXTRACT_SUBREG $a, sub0))), (i32 1))
>;
+let True16Predicate = NotHasTrue16BitInsts in
def : GCNPat <
(i32 (bswap i32:$a)),
(V_BFI_B32_e64 (S_MOV_B32 (i32 0x00ff00ff)),
@@ -3020,8 +3058,27 @@ def : GCNPat <
(V_ALIGNBIT_B32_e64 VSrc_b32:$a, VSrc_b32:$a, (i32 8)))
>;
+let True16Predicate = UseFakeTrue16Insts in
+def : GCNPat <
+ (i32 (bswap i32:$a)),
+ (V_BFI_B32_e64 (S_MOV_B32 (i32 0x00ff00ff)),
+ (V_ALIGNBIT_B32_fake16_e64 0, /* src0_modifiers */
+ VSrc_b32:$a,
+ 0, /* src1_modifiers */
+ VSrc_b32:$a,
+ 0, /* src2_modifiers */
+ (i32 24), /* clamp */ 0, /* op_sel */ 0),
+ (V_ALIGNBIT_B32_fake16_e64 0, /* src0_modifiers */
+ VSrc_b32:$a,
+ 0, /* src1_modifiers */
+ VSrc_b32:$a,
+ 0, /* src2_modifiers */
+ (i32 8), /* clamp */ 0, /* op_sel */ 0))
+>;
+
// FIXME: This should have been narrowed to i32 during legalization.
// This pattern should also be skipped for GlobalISel
+let True16Predicate = NotHasTrue16BitInsts in
def : GCNPat <
(i64 (bswap i64:$a)),
(REG_SEQUENCE VReg_64,
@@ -3043,6 +3100,40 @@ def : GCNPat <
sub1)
>;
+let True16Predicate = UseFakeTrue16Insts in
+def : GCNPat <
+ (i64 (bswap i64:$a)),
+ (REG_SEQUENCE VReg_64,
+ (V_BFI_B32_e64 (S_MOV_B32 (i32 0x00ff00ff)),
+ (V_ALIGNBIT_B32_fake16_e64 0, /* src0_modifiers */
+ (i32 (EXTRACT_SUBREG VReg_64:$a, sub1)),
+ 0, /* src1_modifiers */
+ (i32 (EXTRACT_SUBREG VReg_64:$a, sub1)),
+ 0, /* src2_modifiers */
+ (i32 24), /* clamp */ 0, /* op_sel */ 0),
+ (V_ALIGNBIT_B32_fake16_e64 0, /* src0_modifiers */
+ (i32 (EXTRACT_SUBREG VReg_64:$a, sub0)),
+ 0, /* src1_modifiers */
+ (i32 (EXTRACT_SUBREG VReg_64:$a, sub0)),
+ 0, /* src2_modifiers */
+ (i32 8), /* clamp */ 0, /* op_sel */ 0)),
+ sub0,
+ (V_BFI_B32_e64 (S_MOV_B32 (i32 0x00ff00ff)),
+ (V_ALIGNBIT_B32_fake16_e64 0, /* src0_modifiers */
+ (i32 (EXTRACT_SUBREG VReg_64:$a, sub1)),
+ 0, /* src1_modifiers */
+ (i32 (EXTRACT_SUBREG VReg_64:$a, sub1)),
+ 0, /* src2_modifiers */
+ (i32 24), /* clamp */ 0, /* op_sel */ 0),
+ (V_ALIGNBIT_B32_fake16_e64 0, /* src0_modifiers */
+ (i32 (EXTRACT_SUBREG VReg_64:$a, sub0)),
+ 0, /* src1_modifiers */
+ (i32 (EXTRACT_SUBREG VReg_64:$a, sub0)),
+ 0, /* src2_modifiers */
+ (i32 8), /* clamp */ 0, /* op_sel */ 0)),
+ sub1)
+>;
+
// FIXME: The AddedComplexity should not be needed, but in GlobalISel
// the BFI pattern ends up taking precedence without it.
let SubtargetPredicate = isGFX8Plus, AddedComplexity = 1 in {
@@ -3437,6 +3528,7 @@ def : GCNPat <
// Take the upper 16 bits from V[0] and the lower 16 bits from V[1]
// Special case, can use V_ALIGNBIT (always uses encoded literal)
+let True16Predicate = NotHasTrue16BitInsts in
def : GCNPat <
(vecTy (DivergentBinFrag<build_vector>
(Ty !if(!eq(Ty, i16),
@@ -3446,6 +3538,16 @@ def : GCNPat <
(V_ALIGNBIT_B32_e64 VGPR_32:$b, VGPR_32:$a, (i32 16))
>;
+let True16Predicate = UseFakeTrue16Insts in
+def : GCNPat <
+ (vecTy (DivergentBinFrag<build_vector>
+ (Ty !if(!eq(Ty, i16),
+ (Ty (trunc (srl VGPR_32:$a, (i32 16)))),
+ (Ty (bitconvert (i16 (trunc (srl VGPR_32:$a, (i32 16)))))))),
+ (Ty VGPR_32:$b))),
+ (V_ALIGNBIT_B32_fake16_e64 0, VGPR_32:$b, 0, VGPR_32:$a, 0, (i16 16), 0, 0)
+>;
+
// Take the upper 16 bits from each VGPR_32 and concat them
def : GCNPat <
(vecTy (DivergentBinFrag<build_vector>
diff --git a/llvm/lib/Target/AMDGPU/VOP3Instructions.td b/llvm/lib/Target/AMDGPU/VOP3Instructions.td
index aae18e492e9bf..4b429cb94bd53 100644
--- a/llvm/lib/Target/AMDGPU/VOP3Instructions.td
+++ b/llvm/lib/Target/AMDGPU/VOP3Instructions.td
@@ -211,10 +211,11 @@ defm V_CUBEMA_F32 : VOP3Inst <"v_cubema_f32", VOP3_Profile<VOP_F32_F32_F32_F32>,
defm V_BFE_U32 : VOP3Inst <"v_bfe_u32", VOP3_Profile<VOP_I32_I32_I32_I32>, AMDGPUbfe_u32>;
defm V_BFE_I32 : VOP3Inst <"v_bfe_i32", VOP3_Profile<VOP_I32_I32_I32_I32>, AMDGPUbfe_i32>;
defm V_BFI_B32 : VOP3Inst <"v_bfi_b32", VOP3_Profile<VOP_I32_I32_I32_I32>, AMDGPUbfi>;
-defm V_ALIGNBIT_B32 : VOP3Inst_t16_with_profiles <"v_alignbit_b32", VOP3_Profile<VOP_I32_I32_I32_I32>,
- VOP3_Profile_True16<VOP_I32_I32_I32_I16, VOP3_OPSEL>,
- VOP3_Profile_Fake16<VOP_I32_I32_I32_I16, VOP3_OPSEL>,
- fshr, null_frag>;
+defm V_ALIGNBIT_B32 : VOP3Inst_t16_with_profiles <"v_alignbit_b32",
+ VOP3_Profile<VOP_I32_I32_I32_I32>,
+ VOP3_Profile_True16<VOP_I32_I32_I32_I16, VOP3_OPSEL>,
+ VOP3_Profile_Fake16<VOP_I32_I32_I32_I16, VOP3_OPSEL>,
+ fshr, null_frag>;
defm V_ALIGNBYTE_B32 : VOP3Inst <"v_alignbyte_b32", VOP3_Profile<VOP_I32_I32_I32_I32>, int_amdgcn_alignbyte>;
let True16Predicate = UseRealTrue16Insts in
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-fshr.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-fshr.mir
index 4291744d85732..1fb67fe17cb0a 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-fshr.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-fshr.mir
@@ -3,7 +3,7 @@
# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=instruction-select -verify-machineinstrs -o - %s | FileCheck -check-prefix=GCN %s
# RUN: llc -mtriple=amdgcn -mcpu=gfx900 -run-pass=instruction-select -verify-machineinstrs -o - %s | FileCheck -check-prefix=GCN %s
# RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -run-pass=instruction-select -verify-machineinstrs -o - %s | FileCheck -check-prefix=GCN %s
-# RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -run-pass=instruction-select -verify-machineinstrs -o - %s | FileCheck -check-prefix=GCN %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -run-pass=instruction-select -verify-machineinstrs -o - %s | FileCheck -check-prefix=GFX11 %s
---
@@ -23,6 +23,15 @@ body: |
; GCN-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
; GCN-NEXT: [[V_ALIGNBIT_B32_e64_:%[0-9]+]]:vgpr_32 = V_ALIGNBIT_B32_e64 [[COPY]], [[COPY1]], [[COPY2]], implicit $exec
; GCN-NEXT: S_ENDPGM 0, implicit [[V_ALIGNBIT_B32_e64_]]
+ ;
+ ; GFX11-LABEL: name: fshr_s32
+ ; GFX11: liveins: $vgpr0, $vgpr1, $vgpr2
+ ; GFX11-NEXT: {{ $}}
+ ; GFX11-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX11-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX11-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX11-NEXT: [[V_ALIGNBIT_B32_fake16_e64_:%[0-9]+]]:vgpr_32 = V_ALIGNBIT_B32_fake16_e64 0, [[COPY]], 0, [[COPY1]], 0, [[COPY2]], 0, 0, implicit $exec
+ ; GFX11-NEXT: S_ENDPGM 0, implicit [[V_ALIGNBIT_B32_fake16_e64_]]
%0:vgpr(s32) = COPY $vgpr0
%1:vgpr(s32) = COPY $vgpr1
%2:vgpr(s32) = COPY $vgpr2
diff --git a/llvm/test/CodeGen/AMDGPU/bf16.ll b/llvm/test/CodeGen/AMDGPU/bf16.ll
index 0382cc72a36ae..8467f07b333a3 100644
--- a/llvm/test/CodeGen/AMDGPU/bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/bf16.ll
@@ -9312,34 +9312,35 @@ define <3 x bfloat> @v_fadd_v3bf16(<3 x bfloat> %a, <3 x bfloat> %b) {
; GFX11TRUE16-LABEL: v_fadd_v3bf16:
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX11TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX11TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_dual_add_f32 v4, v5, v4 :: v_dual_lshlrev_b32 v1, 16, v1
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11TRUE16-NEXT: v_dual_add_f32 v0, v0, v2 :: v_dual_add_f32 v1, v1, v3
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_bfe_u32 v2, v4, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v4
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11TRUE16-NEXT: v_add_f32_e32 v3, v5, v4
; GFX11TRUE16-NEXT: v_bfe_u32 v5, v0, 16, 1
-; GFX11TRUE16-NEXT: v_bfe_u32 v3, v1, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v2, v2, v4, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_bfe_u32 v2, v1, 16, 1
+; GFX11TRUE16-NEXT: v_bfe_u32 v4, v3, 16, 1
; GFX11TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v1
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
+; GFX11TRUE16-NEXT: v_add3_u32 v2, v2, v1, 0x7fff
+; GFX11TRUE16-NEXT: v_add3_u32 v4, v4, v3, 0x7fff
; GFX11TRUE16-NEXT: v_add3_u32 v5, v5, v0, 0x7fff
-; GFX11TRUE16-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v7, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v2, v6, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v7, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v5, v8, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11TRUE16-NEXT: v_perm_b32 v0, v0, v2, 0x7060302
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v6, vcc_lo
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_alignbit_b32 v1, v0, v1, 16
+; GFX11TRUE16-NEXT: v_perm_b32 v0, v0, v2, 0x7060302
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_fadd_v3bf16:
@@ -13080,34 +13081,35 @@ define <3 x bfloat> @v_fsub_v3bf16(<3 x bfloat> %a, <3 x bfloat> %b) {
; GFX11TRUE16-LABEL: v_fsub_v3bf16:
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX11TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX11TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_dual_sub_f32 v4, v5, v4 :: v_dual_lshlrev_b32 v1, 16, v1
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11TRUE16-NEXT: v_dual_sub_f32 v0, v0, v2 :: v_dual_sub_f32 v1, v1, v3
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_bfe_u32 v2, v4, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v4
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11TRUE16-NEXT: v_sub_f32_e32 v3, v5, v4
; GFX11TRUE16-NEXT: v_bfe_u32 v5, v0, 16, 1
-; GFX11TRUE16-NEXT: v_bfe_u32 v3, v1, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v2, v2, v4, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_bfe_u32 v2, v1, 16, 1
+; GFX11TRUE16-NEXT: v_bfe_u32 v4, v3, 16, 1
; GFX11TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v1
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
+; GFX11TRUE16-NEXT: v_add3_u32 v2, v2, v1, 0x7fff
+; GFX11TRUE16-NEXT: v_add3_u32 v4, v4, v3, 0x7fff
; GFX11TRUE16-NEXT: v_add3_u32 v5, v5, v0, 0x7fff
-; GFX11TRUE16-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v7, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v2, v6, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v7, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v5, v8, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11TRUE16-NEXT: v_perm_b32 v0, v0, v2, 0x7060302
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v6, vcc_lo
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_alignbit_b32 v1, v0, v1, 16
+; GFX11TRUE16-NEXT: v_perm_b32 v0, v0, v2, 0x7060302
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_fsub_v3bf16:
@@ -13749,34 +13751,35 @@ define <3 x bfloat> @v_fmul_v3bf16(<3 x bfloat> %a, <3 x bfloat> %b) {
; GFX11TRUE16-LABEL: v_fmul_v3bf16:
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX11TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX11TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_dual_mul_f32 v4, v5, v4 :: v_dual_lshlrev_b32 v1, 16, v1
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11TRUE16-NEXT: v_dual_mul_f32 v0, v0, v2 :: v_dual_mul_f32 v1, v1, v3
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_bfe_u32 v2, v4, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v4
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11TRUE16-NEXT: v_mul_f32_e32 v3, v5, v4
; GFX11TRUE16-NEXT: v_bfe_u32 v5, v0, 16, 1
-; GFX11TRUE16-NEXT: v_bfe_u32 v3, v1, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v2, v2, v4, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_bfe_u32 v2, v1, 16, 1
+; GFX11TRUE16-NEXT: v_bfe_u32 v4, v3, 16, 1
; GFX11TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v1
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
+; GFX11TRUE16-NEXT: v_add3_u32 v2, v2, v1, 0x7fff
+; GFX11TRUE16-NEXT: v_add3_u32 v4, v4, v3, 0x7fff
; GFX11TRUE16-NEXT: v_add3_u32 v5, v5, v0, 0x7fff
-; GFX11TRUE16-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v7, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v2, v6, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v7, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v5, v8, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11TRUE16-NEXT: v_perm_b32 v0, v0, v2, 0x7060302
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v6, vcc_lo
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_alignbit_b32 v1, v0, v1, 16
+; GFX11TRUE16-NEXT: v_perm_b32 v0, v0, v2, 0x7060302
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_fmul_v3bf16:
@@ -17827,34 +17830,35 @@ define <3 x bfloat> @v_minnum_v3bf16(<3 x bfloat> %a, <3 x bfloat> %b) {
; GFX11TRUE16-LABEL: v_minnum_v3bf16:
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX11TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX11TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_dual_min_f32 v4, v5, v4 :: v_dual_lshlrev_b32 v1, 16, v1
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11TRUE16-NEXT: v_dual_min_f32 v0, v0, v2 :: v_dual_min_f32 v1, v1, v3
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_bfe_u32 v2, v4, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v4
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11TRUE16-NEXT: v_min_f32_e32 v3, v5, v4
; GFX11TRUE16-NEXT: v_bfe_u32 v5, v0, 16, 1
-; GFX11TRUE16-NEXT: v_bfe_u32 v3, v1, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v2, v2, v4, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_bfe_u32 v2, v1, 16, 1
+; GFX11TRUE16-NEXT: v_bfe_u32 v4, v3, 16, 1
; GFX11TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v1
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
+; GFX11TRUE16-NEXT: v_add3_u32 v2, v2, v1, 0x7fff
+; GFX11TRUE16-NEXT: v_add3_u32 v4, v4, v3, 0x7fff
; GFX11TRUE16-NEXT: v_add3_u32 v5, v5, v0, 0x7fff
-; GFX11TRUE16-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v7, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v2, v6, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v7, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v5, v8, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11TRUE16-NEXT: v_perm_b32 v0, v0, v2, 0x7060302
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v6, vcc_lo
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_alignbit_b32 v1, v0, v1, 16
+; GFX11TRUE16-NEXT: v_perm_b32 v0, v0, v2, 0x7060302
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_minnum_v3bf16:
@@ -21446,34 +21450,35 @@ define <3 x bfloat> @v_maxnum_v3bf16(<3 x bfloat> %a, <3 x bfloat> %b) {
; GFX11TRUE16-LABEL: v_maxnum_v3bf16:
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX11TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX11TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_dual_max_f32 v4, v5, v4 :: v_dual_lshlrev_b32 v1, 16, v1
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11TRUE16-NEXT: v_dual_max_f32 v0, v0, v2 :: v_dual_max_f32 v1, v1, v3
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_bfe_u32 v2, v4, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v4
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11TRUE16-NEXT: v_max_f32_e32 v3, v5, v4
; GFX11TRUE16-NEXT: v_bfe_u32 v5, v0, 16, 1
-; GFX11TRUE16-NEXT: v_bfe_u32 v3, v1, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v2, v2, v4, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_bfe_u32 v2, v1, 16, 1
+; GFX11TRUE16-NEXT: v_bfe_u32 v4, v3, 16, 1
; GFX11TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v1
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
+; GFX11TRUE16-NEXT: v_add3_u32 v2, v2, v1, 0x7fff
+; GFX11TRUE16-NEXT: v_add3_u32 v4, v4, v3, 0x7fff
; GFX11TRUE16-NEXT: v_add3_u32 v5, v5, v0, 0x7fff
-; GFX11TRUE16-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v7, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v2, v6, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v7, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v5, v8, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11TRUE16-NEXT: v_perm_b32 v0, v0, v2, 0x7060302
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v6, vcc_lo
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_alignbit_b32 v1, v0, v1, 16
+; GFX11TRUE16-NEXT: v_perm_b32 v0, v0, v2, 0x7060302
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_maxnum_v3bf16:
@@ -30460,34 +30465,33 @@ define <3 x bfloat> @v_sitofp_v3i16_to_v3bf16(<3 x i16> %x) {
; GFX11TRUE16-LABEL: v_sitofp_v3i16_to_v3bf16:
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_bfe_i32 v2, v0, 0, 16
; GFX11TRUE16-NEXT: v_bfe_i32 v1, v1, 0, 16
+; GFX11TRUE16-NEXT: v_bfe_i32 v2, v0, 0, 16
; GFX11TRUE16-NEXT: v_ashrrev_i32_e32 v0, 16, v0
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_cvt_f32_i32_e32 v2, v2
; GFX11TRUE16-NEXT: v_cvt_f32_i32_e32 v1, v1
+; GFX11TRUE16-NEXT: v_cvt_f32_i32_e32 v2, v2
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11TRUE16-NEXT: v_cvt_f32_i32_e32 v0, v0
-; GFX11TRUE16-NEXT: v_bfe_u32 v3, v2, 16, 1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_bfe_u32 v4, v1, 16, 1
+; GFX11TRUE16-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v1
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX11TRUE16-NEXT: v_bfe_u32 v5, v0, 16, 1
+; GFX11TRUE16-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
+; GFX11TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v2
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11TRUE16-NEXT: v_add3_u32 v3, v3, v2, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v1
; GFX11TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
; GFX11TRUE16-NEXT: v_add3_u32 v5, v5, v0, 0x7fff
-; GFX11TRUE16-NEXT: v_add3_u32 v4, v4, v1, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v3, v7, vcc_lo
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v6, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v7, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v5, v8, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v4, v6, vcc_lo
; GFX11TRUE16-NEXT: v_perm_b32 v0, v0, v2, 0x7060302
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_alignbit_b32 v1, v0, v1, 16
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_sitofp_v3i16_to_v3bf16:
@@ -30986,28 +30990,30 @@ define <3 x bfloat> @v_sitofp_v3i32_to_v3bf16(<3 x i32> %x) {
; GFX11TRUE16-LABEL: v_sitofp_v3i32_to_v3bf16:
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11TRUE16-NEXT: v_cvt_f32_i32_e32 v2, v2
; GFX11TRUE16-NEXT: v_cvt_f32_i32_e32 v0, v0
; GFX11TRUE16-NEXT: v_cvt_f32_i32_e32 v1, v1
-; GFX11TRUE16-NEXT: v_cvt_f32_i32_e32 v2, v2
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX11TRUE16-NEXT: v_bfe_u32 v3, v2, 16, 1
+; GFX11TRUE16-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX11TRUE16-NEXT: v_bfe_u32 v5, v1, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11TRUE16-NEXT: v_add3_u32 v3, v3, v2, 0x7fff
+; GFX11TRUE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v0
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v1
; GFX11TRUE16-NEXT: v_add3_u32 v5, v5, v1, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2
-; GFX11TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v7, vcc_lo
+; GFX11TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v1
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v3, v6, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v4, v7, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v5, v8, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11TRUE16-NEXT: v_perm_b32 v0, v1, v0, 0x7060302
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v6, vcc_lo
-; GFX11TRUE16-NEXT: v_alignbit_b32 v1, v0, v2, 16
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_sitofp_v3i32_to_v3bf16:
@@ -31864,70 +31870,68 @@ define <3 x bfloat> @v_sitofp_v3i64_to_v3bf16(<3 x i64> %x) {
; GFX11TRUE16-LABEL: v_sitofp_v3i64_to_v3bf16:
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_xor_b32_e32 v8, v0, v1
; GFX11TRUE16-NEXT: v_xor_b32_e32 v7, v4, v5
+; GFX11TRUE16-NEXT: v_cls_i32_e32 v6, v5
+; GFX11TRUE16-NEXT: v_xor_b32_e32 v8, v0, v1
; GFX11TRUE16-NEXT: v_xor_b32_e32 v9, v2, v3
; GFX11TRUE16-NEXT: v_cls_i32_e32 v10, v1
-; GFX11TRUE16-NEXT: v_cls_i32_e32 v6, v5
-; GFX11TRUE16-NEXT: v_ashrrev_i32_e32 v8, 31, v8
; GFX11TRUE16-NEXT: v_ashrrev_i32_e32 v7, 31, v7
+; GFX11TRUE16-NEXT: v_add_nc_u32_e32 v6, -1, v6
+; GFX11TRUE16-NEXT: v_ashrrev_i32_e32 v8, 31, v8
; GFX11TRUE16-NEXT: v_cls_i32_e32 v11, v3
; GFX11TRUE16-NEXT: v_ashrrev_i32_e32 v9, 31, v9
+; GFX11TRUE16-NEXT: v_add_nc_u32_e32 v7, 32, v7
; GFX11TRUE16-NEXT: v_add_nc_u32_e32 v10, -1, v10
; GFX11TRUE16-NEXT: v_add_nc_u32_e32 v8, 32, v8
-; GFX11TRUE16-NEXT: v_add_nc_u32_e32 v6, -1, v6
-; GFX11TRUE16-NEXT: v_add_nc_u32_e32 v7, 32, v7
; GFX11TRUE16-NEXT: v_add_nc_u32_e32 v11, -1, v11
; GFX11TRUE16-NEXT: v_add_nc_u32_e32 v9, 32, v9
-; GFX11TRUE16-NEXT: v_min_u32_e32 v8, v10, v8
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11TRUE16-NEXT: v_min_u32_e32 v6, v6, v7
-; GFX11TRUE16-NEXT: v_min_u32_e32 v7, v11, v9
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_min_u32_e32 v7, v10, v8
+; GFX11TRUE16-NEXT: v_min_u32_e32 v8, v11, v9
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_lshlrev_b64 v[0:1], v8, v[0:1]
; GFX11TRUE16-NEXT: v_lshlrev_b64 v[4:5], v6, v[4:5]
-; GFX11TRUE16-NEXT: v_sub_nc_u32_e32 v6, 32, v6
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_lshlrev_b64 v[2:3], v7, v[2:3]
-; GFX11TRUE16-NEXT: v_min_u32_e32 v0, 1, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_lshlrev_b64 v[0:1], v7, v[0:1]
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_lshlrev_b64 v[2:3], v8, v[2:3]
; GFX11TRUE16-NEXT: v_min_u32_e32 v4, 1, v4
-; GFX11TRUE16-NEXT: v_min_u32_e32 v2, 1, v2
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_min_u32_e32 v0, 1, v0
+; GFX11TRUE16-NEXT: v_min_u32_e32 v2, 1, v2
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_or_b32_e32 v4, v5, v4
+; GFX11TRUE16-NEXT: v_sub_nc_u32_e32 v5, 32, v8
; GFX11TRUE16-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX11TRUE16-NEXT: v_or_b32_e32 v1, v5, v4
-; GFX11TRUE16-NEXT: v_sub_nc_u32_e32 v4, 32, v7
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_sub_nc_u32_e32 v1, 32, v6
; GFX11TRUE16-NEXT: v_or_b32_e32 v2, v3, v2
-; GFX11TRUE16-NEXT: v_sub_nc_u32_e32 v3, 32, v8
+; GFX11TRUE16-NEXT: v_cvt_f32_i32_e32 v4, v4
+; GFX11TRUE16-NEXT: v_sub_nc_u32_e32 v3, 32, v7
; GFX11TRUE16-NEXT: v_cvt_f32_i32_e32 v0, v0
-; GFX11TRUE16-NEXT: v_cvt_f32_i32_e32 v1, v1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11TRUE16-NEXT: v_cvt_f32_i32_e32 v2, v2
+; GFX11TRUE16-NEXT: v_ldexp_f32 v1, v4, v1
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11TRUE16-NEXT: v_ldexp_f32 v0, v0, v3
-; GFX11TRUE16-NEXT: v_ldexp_f32 v1, v1, v6
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_ldexp_f32 v2, v2, v4
-; GFX11TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v0
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_ldexp_f32 v2, v2, v5
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v1
+; GFX11TRUE16-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX11TRUE16-NEXT: v_bfe_u32 v5, v2, 16, 1
-; GFX11TRUE16-NEXT: v_bfe_u32 v4, v1, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
+; GFX11TRUE16-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v0
+; GFX11TRUE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
; GFX11TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
-; GFX11TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v1
; GFX11TRUE16-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
-; GFX11TRUE16-NEXT: v_add3_u32 v4, v4, v1, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v7, vcc_lo
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v6, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v4, v7, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v8, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX11TRUE16-NEXT: v_perm_b32 v0, v2, v0, 0x7060302
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v4, v6, vcc_lo
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_alignbit_b32 v1, v0, v1, 16
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_sitofp_v3i64_to_v3bf16:
@@ -32733,35 +32737,35 @@ define <3 x bfloat> @v_uitofp_v3i16_to_v3bf16(<3 x i16> %x) {
; GFX11TRUE16-LABEL: v_uitofp_v3i16_to_v3bf16:
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_cvt_f32_u32_e32 v1, v1
-; GFX11TRUE16-NEXT: v_bfe_u32 v4, v1, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_add3_u32 v4, v4, v1, 0x7fff
; GFX11TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v0
; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11TRUE16-NEXT: v_cvt_f32_u32_e32 v2, v2
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_cvt_f32_u32_e32 v2, v2
; GFX11TRUE16-NEXT: v_cvt_f32_u32_e32 v0, v0
-; GFX11TRUE16-NEXT: v_bfe_u32 v3, v2, 16, 1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_bfe_u32 v5, v0, 16, 1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1
; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v2
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11TRUE16-NEXT: v_bfe_u32 v5, v0, 16, 1
; GFX11TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
-; GFX11TRUE16-NEXT: v_add3_u32 v3, v3, v2, 0x7fff
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
+; GFX11TRUE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX11TRUE16-NEXT: v_add3_u32 v5, v5, v0, 0x7fff
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v3, v7, vcc_lo
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11TRUE16-NEXT: v_cvt_f32_u32_e32 v1, v1
+; GFX11TRUE16-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v1
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11TRUE16-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v6, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v7, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v5, v8, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11TRUE16-NEXT: v_perm_b32 v0, v0, v2, 0x7060302
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v4, v6, vcc_lo
-; GFX11TRUE16-NEXT: v_alignbit_b32 v1, v0, v1, 16
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_uitofp_v3i16_to_v3bf16:
@@ -33263,28 +33267,30 @@ define <3 x bfloat> @v_uitofp_v3i32_to_v3bf16(<3 x i32> %x) {
; GFX11TRUE16-LABEL: v_uitofp_v3i32_to_v3bf16:
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11TRUE16-NEXT: v_cvt_f32_u32_e32 v2, v2
; GFX11TRUE16-NEXT: v_cvt_f32_u32_e32 v0, v0
; GFX11TRUE16-NEXT: v_cvt_f32_u32_e32 v1, v1
-; GFX11TRUE16-NEXT: v_cvt_f32_u32_e32 v2, v2
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX11TRUE16-NEXT: v_bfe_u32 v3, v2, 16, 1
+; GFX11TRUE16-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX11TRUE16-NEXT: v_bfe_u32 v5, v1, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11TRUE16-NEXT: v_add3_u32 v3, v3, v2, 0x7fff
+; GFX11TRUE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v0
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v1
; GFX11TRUE16-NEXT: v_add3_u32 v5, v5, v1, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2
-; GFX11TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v7, vcc_lo
+; GFX11TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v1
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v3, v6, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v4, v7, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v5, v8, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11TRUE16-NEXT: v_perm_b32 v0, v1, v0, 0x7060302
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v6, vcc_lo
-; GFX11TRUE16-NEXT: v_alignbit_b32 v1, v0, v2, 16
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_uitofp_v3i32_to_v3bf16:
@@ -34003,58 +34009,57 @@ define <3 x bfloat> @v_uitofp_v3i64_to_v3bf16(<3 x i64> %x) {
; GFX11TRUE16-LABEL: v_uitofp_v3i64_to_v3bf16:
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_clz_i32_u32_e32 v6, v1
-; GFX11TRUE16-NEXT: v_clz_i32_u32_e32 v7, v5
+; GFX11TRUE16-NEXT: v_clz_i32_u32_e32 v6, v5
+; GFX11TRUE16-NEXT: v_clz_i32_u32_e32 v7, v1
; GFX11TRUE16-NEXT: v_clz_i32_u32_e32 v8, v3
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11TRUE16-NEXT: v_min_u32_e32 v6, 32, v6
; GFX11TRUE16-NEXT: v_min_u32_e32 v7, 32, v7
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11TRUE16-NEXT: v_min_u32_e32 v8, 32, v8
-; GFX11TRUE16-NEXT: v_lshlrev_b64 v[0:1], v6, v[0:1]
+; GFX11TRUE16-NEXT: v_lshlrev_b64 v[4:5], v6, v[4:5]
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_lshlrev_b64 v[4:5], v7, v[4:5]
+; GFX11TRUE16-NEXT: v_lshlrev_b64 v[0:1], v7, v[0:1]
; GFX11TRUE16-NEXT: v_lshlrev_b64 v[2:3], v8, v[2:3]
-; GFX11TRUE16-NEXT: v_sub_nc_u32_e32 v7, 32, v7
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_min_u32_e32 v0, 1, v0
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11TRUE16-NEXT: v_min_u32_e32 v4, 1, v4
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_min_u32_e32 v0, 1, v0
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11TRUE16-NEXT: v_min_u32_e32 v2, 1, v2
+; GFX11TRUE16-NEXT: v_or_b32_e32 v4, v5, v4
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
; GFX11TRUE16-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_or_b32_e32 v1, v5, v4
+; GFX11TRUE16-NEXT: v_sub_nc_u32_e32 v1, 32, v6
; GFX11TRUE16-NEXT: v_or_b32_e32 v2, v3, v2
-; GFX11TRUE16-NEXT: v_sub_nc_u32_e32 v3, 32, v6
-; GFX11TRUE16-NEXT: v_sub_nc_u32_e32 v4, 32, v8
+; GFX11TRUE16-NEXT: v_sub_nc_u32_e32 v3, 32, v7
+; GFX11TRUE16-NEXT: v_cvt_f32_u32_e32 v4, v4
; GFX11TRUE16-NEXT: v_cvt_f32_u32_e32 v0, v0
-; GFX11TRUE16-NEXT: v_cvt_f32_u32_e32 v1, v1
+; GFX11TRUE16-NEXT: v_sub_nc_u32_e32 v5, 32, v8
; GFX11TRUE16-NEXT: v_cvt_f32_u32_e32 v2, v2
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_ldexp_f32 v1, v4, v1
; GFX11TRUE16-NEXT: v_ldexp_f32 v0, v0, v3
-; GFX11TRUE16-NEXT: v_ldexp_f32 v1, v1, v7
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_ldexp_f32 v2, v2, v4
-; GFX11TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_bfe_u32 v4, v1, 16, 1
+; GFX11TRUE16-NEXT: v_ldexp_f32 v2, v2, v5
+; GFX11TRUE16-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v1
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX11TRUE16-NEXT: v_bfe_u32 v5, v2, 16, 1
+; GFX11TRUE16-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
+; GFX11TRUE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v0
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v1
; GFX11TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
; GFX11TRUE16-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
-; GFX11TRUE16-NEXT: v_add3_u32 v4, v4, v1, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v7, vcc_lo
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v6, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v4, v7, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v8, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v4, v6, vcc_lo
; GFX11TRUE16-NEXT: v_perm_b32 v0, v2, v0, 0x7060302
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_alignbit_b32 v1, v0, v1, 16
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_uitofp_v3i64_to_v3bf16:
@@ -40495,40 +40500,40 @@ define <3 x bfloat> @v_fma_v3bf16(<3 x bfloat> %a, <3 x bfloat> %b, <3 x bfloat>
; GFX11TRUE16-LABEL: v_fma_v3bf16:
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v4
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v2
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v0
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; GFX11TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
; GFX11TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX11TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11TRUE16-NEXT: v_dual_fmac_f32 v6, v8, v7 :: v_dual_lshlrev_b32 v5, 16, v5
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_fmac_f32_e32 v4, v0, v2
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_bfe_u32 v2, v4, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v4
-; GFX11TRUE16-NEXT: v_add3_u32 v2, v2, v4, 0x7fff
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_fmac_f32_e32 v6, v8, v7
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_dual_fmac_f32 v4, v0, v2 :: v_dual_lshlrev_b32 v3, 16, v3
+; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v6
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_4)
; GFX11TRUE16-NEXT: v_fmac_f32_e32 v5, v1, v3
; GFX11TRUE16-NEXT: v_bfe_u32 v1, v6, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v6
+; GFX11TRUE16-NEXT: v_bfe_u32 v2, v4, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v4
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX11TRUE16-NEXT: v_bfe_u32 v0, v5, 16, 1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v5
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX11TRUE16-NEXT: v_add3_u32 v1, v1, v6, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11TRUE16-NEXT: v_add3_u32 v2, v2, v4, 0x7fff
; GFX11TRUE16-NEXT: v_add3_u32 v0, v0, v5, 0x7fff
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc_lo
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v7, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v7, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v3, v0, v8, vcc_lo
-; GFX11TRUE16-NEXT: v_perm_b32 v0, v2, v1, 0x7060302
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v8, vcc_lo
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_alignbit_b32 v1, v0, v3, 16
+; GFX11TRUE16-NEXT: v_perm_b32 v0, v2, v1, 0x7060302
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v3.l
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_fma_v3bf16:
@@ -41445,8 +41450,9 @@ define <3 x bfloat> @v_fmuladd_v3bf16(<3 x bfloat> %a, <3 x bfloat> %b, <3 x bfl
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v0
; GFX11TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX11TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_dual_mul_f32 v0, v0, v2 :: v_dual_lshlrev_b32 v3, 16, v3
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11TRUE16-NEXT: v_mul_f32_e32 v0, v0, v2
; GFX11TRUE16-NEXT: v_bfe_u32 v8, v0, 16, 1
; GFX11TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v0
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
@@ -41478,27 +41484,27 @@ define <3 x bfloat> @v_fmuladd_v3bf16(<3 x bfloat> %a, <3 x bfloat> %b, <3 x bfl
; GFX11TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX11TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v2
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11TRUE16-NEXT: v_dual_add_f32 v0, v0, v4 :: v_dual_add_f32 v1, v1, v3
-; GFX11TRUE16-NEXT: v_bfe_u32 v3, v2, 16, 1
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1
; GFX11TRUE16-NEXT: v_bfe_u32 v5, v0, 16, 1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_bfe_u32 v4, v1, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v3, v3, v2, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_bfe_u32 v3, v1, 16, 1
; GFX11TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v1
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
; GFX11TRUE16-NEXT: v_add3_u32 v5, v5, v0, 0x7fff
-; GFX11TRUE16-NEXT: v_add3_u32 v4, v4, v1, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v3, v7, vcc_lo
+; GFX11TRUE16-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v6, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v7, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v5, v8, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11TRUE16-NEXT: v_perm_b32 v0, v0, v2, 0x7060302
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v4, v6, vcc_lo
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_alignbit_b32 v1, v0, v1, 16
+; GFX11TRUE16-NEXT: v_perm_b32 v0, v0, v2, 0x7060302
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_fmuladd_v3bf16:
>From 0c781971efc81926ce6055ad21837cb2c4094636 Mon Sep 17 00:00:00 2001
From: guochen2 <guochen2 at amd.com>
Date: Tue, 21 Jan 2025 13:10:06 -0500
Subject: [PATCH 3/4] simplify and remove duplication
---
llvm/lib/Target/AMDGPU/SIInstructions.td | 83 +++++++++---------------
llvm/test/CodeGen/AMDGPU/bswap.ll | 28 +++++---
2 files changed, 50 insertions(+), 61 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIInstructions.td b/llvm/lib/Target/AMDGPU/SIInstructions.td
index e011ef7c22fa4..50a78a56c5b65 100644
--- a/llvm/lib/Target/AMDGPU/SIInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SIInstructions.td
@@ -3078,61 +3078,40 @@ def : GCNPat <
// FIXME: This should have been narrowed to i32 during legalization.
// This pattern should also be skipped for GlobalISel
+class bswapi64ExtPat<Instruction inst, bit hasTrue16> {
+ dag pattern = (i64 (bswap i64:$a));
+ dag operand1 = (i32 (EXTRACT_SUBREG VReg_64:$a, sub1));
+ dag operand2 = (i32 (EXTRACT_SUBREG VReg_64:$a, sub0));
+ dag ALIGNBIT32_INST1 = !if(hasTrue16,
+ (inst 0, operand1, 0, operand1, 0, (i32 24), 0, 0),
+ (inst operand1, operand1, (i32 24)));
+ dag ALIGNBIT32_INST2 = !if(hasTrue16,
+ (inst 0, operand1, 0, operand1, 0, (i32 8), 0, 0),
+ (inst operand1, operand1, (i32 8)));
+ dag ALIGNBIT32_INST3 = !if(hasTrue16,
+ (inst 0, operand2, 0, operand2, 0, (i32 24), 0, 0),
+ (inst operand2, operand2, (i32 24)));
+ dag ALIGNBIT32_INST4 = !if(hasTrue16,
+ (inst 0, operand2, 0, operand2, 0, (i32 8), 0, 0),
+ (inst operand2, operand2, (i32 8)));
+ dag result = (REG_SEQUENCE VReg_64,
+ (V_BFI_B32_e64 (S_MOV_B32 (i32 0x00ff00ff)),
+ ALIGNBIT32_INST1,
+ ALIGNBIT32_INST2),
+ sub0,
+ (V_BFI_B32_e64 (S_MOV_B32 (i32 0x00ff00ff)),
+ ALIGNBIT32_INST3,
+ ALIGNBIT32_INST4),
+ sub1);
+}
+
let True16Predicate = NotHasTrue16BitInsts in
-def : GCNPat <
- (i64 (bswap i64:$a)),
- (REG_SEQUENCE VReg_64,
- (V_BFI_B32_e64 (S_MOV_B32 (i32 0x00ff00ff)),
- (V_ALIGNBIT_B32_e64 (i32 (EXTRACT_SUBREG VReg_64:$a, sub1)),
- (i32 (EXTRACT_SUBREG VReg_64:$a, sub1)),
- (i32 24)),
- (V_ALIGNBIT_B32_e64 (i32 (EXTRACT_SUBREG VReg_64:$a, sub1)),
- (i32 (EXTRACT_SUBREG VReg_64:$a, sub1)),
- (i32 8))),
- sub0,
- (V_BFI_B32_e64 (S_MOV_B32 (i32 0x00ff00ff)),
- (V_ALIGNBIT_B32_e64 (i32 (EXTRACT_SUBREG VReg_64:$a, sub0)),
- (i32 (EXTRACT_SUBREG VReg_64:$a, sub0)),
- (i32 24)),
- (V_ALIGNBIT_B32_e64 (i32 (EXTRACT_SUBREG VReg_64:$a, sub0)),
- (i32 (EXTRACT_SUBREG VReg_64:$a, sub0)),
- (i32 8))),
- sub1)
->;
+def : GCNPat <bswapi64ExtPat<V_ALIGNBIT_B32_e64, 0>.pattern,
+ bswapi64ExtPat<V_ALIGNBIT_B32_e64, 0>.result>;
let True16Predicate = UseFakeTrue16Insts in
-def : GCNPat <
- (i64 (bswap i64:$a)),
- (REG_SEQUENCE VReg_64,
- (V_BFI_B32_e64 (S_MOV_B32 (i32 0x00ff00ff)),
- (V_ALIGNBIT_B32_fake16_e64 0, /* src0_modifiers */
- (i32 (EXTRACT_SUBREG VReg_64:$a, sub1)),
- 0, /* src1_modifiers */
- (i32 (EXTRACT_SUBREG VReg_64:$a, sub1)),
- 0, /* src2_modifiers */
- (i32 24), /* clamp */ 0, /* op_sel */ 0),
- (V_ALIGNBIT_B32_fake16_e64 0, /* src0_modifiers */
- (i32 (EXTRACT_SUBREG VReg_64:$a, sub0)),
- 0, /* src1_modifiers */
- (i32 (EXTRACT_SUBREG VReg_64:$a, sub0)),
- 0, /* src2_modifiers */
- (i32 8), /* clamp */ 0, /* op_sel */ 0)),
- sub0,
- (V_BFI_B32_e64 (S_MOV_B32 (i32 0x00ff00ff)),
- (V_ALIGNBIT_B32_fake16_e64 0, /* src0_modifiers */
- (i32 (EXTRACT_SUBREG VReg_64:$a, sub1)),
- 0, /* src1_modifiers */
- (i32 (EXTRACT_SUBREG VReg_64:$a, sub1)),
- 0, /* src2_modifiers */
- (i32 24), /* clamp */ 0, /* op_sel */ 0),
- (V_ALIGNBIT_B32_fake16_e64 0, /* src0_modifiers */
- (i32 (EXTRACT_SUBREG VReg_64:$a, sub0)),
- 0, /* src1_modifiers */
- (i32 (EXTRACT_SUBREG VReg_64:$a, sub0)),
- 0, /* src2_modifiers */
- (i32 8), /* clamp */ 0, /* op_sel */ 0)),
- sub1)
->;
+def : GCNPat <bswapi64ExtPat<V_ALIGNBIT_B32_fake16_e64, 1>.pattern,
+ bswapi64ExtPat<V_ALIGNBIT_B32_fake16_e64, 1>.result>;
// FIXME: The AddedComplexity should not be needed, but in GlobalISel
// the BFI pattern ends up taking precedence without it.
diff --git a/llvm/test/CodeGen/AMDGPU/bswap.ll b/llvm/test/CodeGen/AMDGPU/bswap.ll
index a95a1aba0c914..b98ca1ff55f59 100644
--- a/llvm/test/CodeGen/AMDGPU/bswap.ll
+++ b/llvm/test/CodeGen/AMDGPU/bswap.ll
@@ -737,15 +737,25 @@ define i64 @v_bswap_i48(i64 %src) {
; VI-NEXT: v_lshrrev_b32_e32 v1, 16, v2
; VI-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: v_bswap_i48:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_perm_b32 v2, 0, v0, 0x10203
-; GFX11-NEXT: v_perm_b32 v0, 0, v1, 0x10203
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_lshrrev_b32_e32 v1, 16, v2
-; GFX11-NEXT: v_alignbit_b32 v0, v2, v0, 16
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-REAL16-LABEL: v_bswap_i48:
+; GFX11-REAL16: ; %bb.0:
+; GFX11-REAL16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-REAL16-NEXT: v_perm_b32 v2, 0, v0, 0x10203
+; GFX11-REAL16-NEXT: v_perm_b32 v1, 0, v1, 0x10203
+; GFX11-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-REAL16-NEXT: v_lshrrev_b64 v[0:1], 16, v[1:2]
+; GFX11-REAL16-NEXT: v_lshrrev_b32_e32 v1, 16, v2
+; GFX11-REAL16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: v_bswap_i48:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_perm_b32 v2, 0, v0, 0x10203
+; GFX11-FAKE16-NEXT: v_perm_b32 v0, 0, v1, 0x10203
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v2
+; GFX11-FAKE16-NEXT: v_alignbit_b32 v0, v2, v0, 16
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
%trunc = trunc i64 %src to i48
%bswap = call i48 @llvm.bswap.i48(i48 %trunc)
%zext = zext i48 %bswap to i64
>From e53fc9c7581c3ad8017ad848b4145e3416c61039 Mon Sep 17 00:00:00 2001
From: guochen2 <guochen2 at amd.com>
Date: Wed, 19 Feb 2025 14:30:57 -0500
Subject: [PATCH 4/4] patch 2
---
llvm/lib/Target/AMDGPU/SIInstructions.td | 63 ++++++++++++------------
1 file changed, 32 insertions(+), 31 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIInstructions.td b/llvm/lib/Target/AMDGPU/SIInstructions.td
index 50a78a56c5b65..adad683abca9f 100644
--- a/llvm/lib/Target/AMDGPU/SIInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SIInstructions.td
@@ -3076,42 +3076,43 @@ def : GCNPat <
(i32 8), /* clamp */ 0, /* op_sel */ 0))
>;
-// FIXME: This should have been narrowed to i32 during legalization.
-// This pattern should also be skipped for GlobalISel
-class bswapi64ExtPat<Instruction inst, bit hasTrue16> {
- dag pattern = (i64 (bswap i64:$a));
- dag operand1 = (i32 (EXTRACT_SUBREG VReg_64:$a, sub1));
- dag operand2 = (i32 (EXTRACT_SUBREG VReg_64:$a, sub0));
- dag ALIGNBIT32_INST1 = !if(hasTrue16,
- (inst 0, operand1, 0, operand1, 0, (i32 24), 0, 0),
- (inst operand1, operand1, (i32 24)));
- dag ALIGNBIT32_INST2 = !if(hasTrue16,
- (inst 0, operand1, 0, operand1, 0, (i32 8), 0, 0),
- (inst operand1, operand1, (i32 8)));
- dag ALIGNBIT32_INST3 = !if(hasTrue16,
- (inst 0, operand2, 0, operand2, 0, (i32 24), 0, 0),
- (inst operand2, operand2, (i32 24)));
- dag ALIGNBIT32_INST4 = !if(hasTrue16,
- (inst 0, operand2, 0, operand2, 0, (i32 8), 0, 0),
- (inst operand2, operand2, (i32 8)));
- dag result = (REG_SEQUENCE VReg_64,
- (V_BFI_B32_e64 (S_MOV_B32 (i32 0x00ff00ff)),
- ALIGNBIT32_INST1,
- ALIGNBIT32_INST2),
- sub0,
- (V_BFI_B32_e64 (S_MOV_B32 (i32 0x00ff00ff)),
- ALIGNBIT32_INST3,
- ALIGNBIT32_INST4),
- sub1);
+class AlignBit32Inst<dag op1, dag op2, dag op3, bit hasTrue16> {
+ Instruction inst = !if(hasTrue16, V_ALIGNBIT_B32_fake16_e64, V_ALIGNBIT_B32_e64);
+ dag NoMods = !if(hasTrue16, (inst 0), (inst));
+ dag expr = !con(NoMods, (inst op1), NoMods, (inst op2),
+ NoMods, (inst op3), NoMods, NoMods);
+}
+
+multiclass bswapi64ExtPat<bit hasTrue16> {
+def : GCNPat <
+ (i64 (bswap i64:$a)),
+ (REG_SEQUENCE VReg_64,
+ (V_BFI_B32_e64 (S_MOV_B32 (i32 0x00ff00ff)),
+ AlignBit32Inst<(i32 (EXTRACT_SUBREG VReg_64:$a, sub1)),
+ (i32 (EXTRACT_SUBREG VReg_64:$a, sub1)),
+ (i32 24), hasTrue16>.expr,
+ AlignBit32Inst<(i32 (EXTRACT_SUBREG VReg_64:$a, sub1)),
+ (i32 (EXTRACT_SUBREG VReg_64:$a, sub1)),
+ (i32 8), hasTrue16>.expr),
+ sub0,
+ (V_BFI_B32_e64 (S_MOV_B32 (i32 0x00ff00ff)),
+ AlignBit32Inst<(i32 (EXTRACT_SUBREG VReg_64:$a, sub0)),
+ (i32 (EXTRACT_SUBREG VReg_64:$a, sub0)),
+ (i32 24), hasTrue16>.expr,
+ AlignBit32Inst<(i32 (EXTRACT_SUBREG VReg_64:$a, sub0)),
+ (i32 (EXTRACT_SUBREG VReg_64:$a, sub0)),
+ (i32 8), hasTrue16>.expr),
+ sub1)
+>;
}
+// FIXME: This should have been narrowed to i32 during legalization.
+// This pattern should also be skipped for GlobalISel
let True16Predicate = NotHasTrue16BitInsts in
-def : GCNPat <bswapi64ExtPat<V_ALIGNBIT_B32_e64, 0>.pattern,
- bswapi64ExtPat<V_ALIGNBIT_B32_e64, 0>.result>;
+defm : bswapi64ExtPat</*hasTrue16*/0>;
let True16Predicate = UseFakeTrue16Insts in
-def : GCNPat <bswapi64ExtPat<V_ALIGNBIT_B32_fake16_e64, 1>.pattern,
- bswapi64ExtPat<V_ALIGNBIT_B32_fake16_e64, 1>.result>;
+defm : bswapi64ExtPat</*hasTrue16*/1>;
// FIXME: The AddedComplexity should not be needed, but in GlobalISel
// the BFI pattern ends up taking precedence without it.
More information about the llvm-commits
mailing list