[llvm] Main merge true16 fix bfe (PR #144024)
Brox Chen via llvm-commits
llvm-commits at lists.llvm.org
Thu Jun 12 23:52:19 PDT 2025
https://github.com/broxigarchen created https://github.com/llvm/llvm-project/pull/144024
None
>From 89dfa75d04c6fcd04263b8dd542196dd17e87454 Mon Sep 17 00:00:00 2001
From: guochen2 <guochen2 at amd.com>
Date: Fri, 13 Jun 2025 02:08:01 -0400
Subject: [PATCH 1/2] fix bfe for true16 mode
---
llvm/lib/Target/AMDGPU/SIInstructions.td | 19 +++++++++++++++++
llvm/lib/Target/AMDGPU/VOP3Instructions.td | 24 ++++++++++++++++++++++
2 files changed, 43 insertions(+)
diff --git a/llvm/lib/Target/AMDGPU/SIInstructions.td b/llvm/lib/Target/AMDGPU/SIInstructions.td
index 360fd05cb3d96..ea8bbcea63584 100644
--- a/llvm/lib/Target/AMDGPU/SIInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SIInstructions.td
@@ -2560,6 +2560,8 @@ def : GCNPat<
(i32 (DivergentSextInreg<i1> i32:$src)),
(V_BFE_I32_e64 i32:$src, (i32 0), (i32 1))>;
+foreach p = [NotHasTrue16BitInsts, UseFakeTrue16Insts] in
+let True16Predicate = p in {
def : GCNPat <
(i16 (DivergentSextInreg<i1> i16:$src)),
(V_BFE_I32_e64 $src, (i32 0), (i32 1))
@@ -2569,6 +2571,23 @@ def : GCNPat <
(i16 (DivergentSextInreg<i8> i16:$src)),
(V_BFE_I32_e64 $src, (i32 0), (i32 8))
>;
+}
+
+let True16Predicate = UseRealTrue16Insts in {
+def : GCNPat <
+ (i16 (DivergentSextInreg<i1> i16:$src)),
+ (V_BFE_I32_e64
+ (REG_SEQUENCE VGPR_32, VGPR_16:$src, lo16, (i16 (IMPLICIT_DEF)), hi16),
+ (i32 0), (i32 1))
+>;
+
+def : GCNPat <
+ (i16 (DivergentSextInreg<i8> i16:$src)),
+ (V_BFE_I32_e64
+ (REG_SEQUENCE VGPR_32, VGPR_16:$src, lo16, (i16 (IMPLICIT_DEF)), hi16),
+ (i32 0), (i32 8))
+>;
+}
def : GCNPat<
(i32 (DivergentSextInreg<i8> i32:$src)),
diff --git a/llvm/lib/Target/AMDGPU/VOP3Instructions.td b/llvm/lib/Target/AMDGPU/VOP3Instructions.td
index a005e0245b8ff..c213b69a423ae 100644
--- a/llvm/lib/Target/AMDGPU/VOP3Instructions.td
+++ b/llvm/lib/Target/AMDGPU/VOP3Instructions.td
@@ -319,11 +319,21 @@ let SchedRW = [Write64Bit] in {
} // End SchedRW = [Write64Bit]
} // End isReMaterializable = 1
+foreach p = [NotHasTrue16BitInsts, UseFakeTrue16Insts] in
+let True16Predicate = p in
def : GCNPat<
(i32 (DivergentUnaryFrag<sext> i16:$src)),
(i32 (V_BFE_I32_e64 i16:$src, (i32 0), (i32 0x10)))
>;
+let True16Predicate = UseRealTrue16Insts in
+def : GCNPat<
+ (i32 (DivergentUnaryFrag<sext> i16:$src)),
+ (i32 (V_BFE_I32_e64
+ (REG_SEQUENCE VGPR_32, VGPR_16:$src, lo16, (i16 (IMPLICIT_DEF)), hi16),
+ (i32 0), (i32 0x10)))
+>;
+
let isReMaterializable = 1 in {
let SubtargetPredicate = isGFX6GFX7GFX10Plus in {
defm V_MULLIT_F32 : VOP3Inst <"v_mullit_f32", VOP3_Profile<VOP_F32_F32_F32_F32>>;
@@ -423,6 +433,8 @@ def V_INTERP_P1LV_F16 : VOP3Interp <"v_interp_p1lv_f16", VOP3_INTERP16<[f32, f32
} // End SubtargetPredicate = Has16BitInsts, isCommutable = 1
+foreach p = [NotHasTrue16BitInsts, UseFakeTrue16Insts] in
+let True16Predicate = p in
def : GCNPat<
(i64 (DivergentUnaryFrag<sext> i16:$src)),
(REG_SEQUENCE VReg_64,
@@ -432,6 +444,18 @@ def : GCNPat<
), VGPR_32)), sub1)
>;
+let True16Predicate = UseRealTrue16Insts in
+def : GCNPat<
+ (i64 (DivergentUnaryFrag<sext> i16:$src)),
+ (REG_SEQUENCE VReg_64,
+ (i32 (V_BFE_I32_e64
+ (REG_SEQUENCE VGPR_32, VGPR_16:$src, lo16, (i16 (IMPLICIT_DEF)), hi16),
+ (S_MOV_B32 (i32 0)), (S_MOV_B32 (i32 0x10)))), sub0,
+ (i32 (COPY_TO_REGCLASS
+ (V_ASHRREV_I32_e32 (S_MOV_B32 (i32 0x1f)), (i32 (V_BFE_I32_e64 $src, (S_MOV_B32 (i32 0)), (S_MOV_B32 (i32 0x10))))
+ ), VGPR_32)), sub1)
+>;
+
let SubtargetPredicate = isGFX8Plus, Uses = [MODE, M0, EXEC], OtherPredicates = [isNotGFX90APlus] in {
def V_INTERP_P1_F32_e64 : VOP3Interp <"v_interp_p1_f32", VOP3_INTERP>;
def V_INTERP_P2_F32_e64 : VOP3Interp <"v_interp_p2_f32", VOP3_INTERP>;
>From 8dcdb7ba278c7dcc9127c7a2a96d08d10db1dcf4 Mon Sep 17 00:00:00 2001
From: guochen2 <guochen2 at amd.com>
Date: Fri, 13 Jun 2025 02:43:15 -0400
Subject: [PATCH 2/2] update test
---
llvm/test/CodeGen/AMDGPU/sext-in-reg.ll | 2522 ++++++++++++++++++++---
1 file changed, 2249 insertions(+), 273 deletions(-)
diff --git a/llvm/test/CodeGen/AMDGPU/sext-in-reg.ll b/llvm/test/CodeGen/AMDGPU/sext-in-reg.ll
index cc07ee4ee4780..0ad807eef19bb 100644
--- a/llvm/test/CodeGen/AMDGPU/sext-in-reg.ll
+++ b/llvm/test/CodeGen/AMDGPU/sext-in-reg.ll
@@ -1,38 +1,98 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgcn-- -verify-machineinstrs < %s | FileCheck -enable-var-scope --check-prefixes=GCN,SI,FUNC %s
; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgcn-- -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -enable-var-scope --check-prefixes=GCN,GFX89,FUNC %s
; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgcn-- -mcpu=gfx900 -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -enable-var-scope --check-prefixes=GCN,GFX9,GFX89,FUNC %s
; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=r600-- -mcpu=cypress < %s | FileCheck -enable-var-scope --check-prefixes=EG,FUNC %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgcn-- -mcpu=gfx1100 -mattr=-flat-for-global,+real-true16 -verify-machineinstrs < %s | FileCheck -enable-var-scope --check-prefixes=GCN,GFX9,GFX89,FUNC,GFX11,GFX11-TRUE16 %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgcn-- -mcpu=gfx1100 -mattr=-flat-for-global,-real-true16 -verify-machineinstrs < %s | FileCheck -enable-var-scope --check-prefixes=GCN,GFX9,GFX89,FUNC,GFX11,GFX11-FAKE16 %s
; FIXME: i16 promotion pass ruins the scalar cases when legal.
; FIXME: r600 fails verifier
-; FUNC-LABEL: {{^}}sext_in_reg_i1_i32:
-; GCN: s_load_dword [[ARG:s[0-9]+]],
-; GCN: s_bfe_i32 [[SEXTRACT:s[0-9]+]], [[ARG]], 0x10000
-; GCN: v_mov_b32_e32 [[EXTRACT:v[0-9]+]], [[SEXTRACT]]
-; GCN: buffer_store_dword [[EXTRACT]],
-; EG: MEM_{{.*}} STORE_{{.*}} [[RES:T[0-9]+\.[XYZW]]], [[ADDR:T[0-9]+.[XYZW]]]
-; EG: LSHR * [[ADDR]]
-; EG: BFE_INT * [[RES]], {{.*}}, 0.0, 1
define amdgpu_kernel void @sext_in_reg_i1_i32(ptr addrspace(1) %out, i32 %in) #0 {
+; SI-LABEL: sext_in_reg_i1_i32:
+; SI: ; %bb.0:
+; SI-NEXT: s_load_dword s2, s[4:5], 0xb
+; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_bfe_i32 s4, s2, 0x10000
+; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: v_mov_b32_e32 v0, s4
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_endpgm
+;
+; EG-LABEL: sext_in_reg_i1_i32:
+; EG: ; %bb.0:
+; EG-NEXT: ALU 2, @4, KC0[CB0:0-32], KC1[]
+; EG-NEXT: MEM_RAT_CACHELESS STORE_RAW T1.X, T0.X, 1
+; EG-NEXT: CF_END
+; EG-NEXT: PAD
+; EG-NEXT: ALU clause starting at 4:
+; EG-NEXT: LSHR * T0.X, KC0[2].Y, literal.x,
+; EG-NEXT: 2(2.802597e-45), 0(0.000000e+00)
+; EG-NEXT: BFE_INT * T1.X, KC0[2].Z, 0.0, 1,
+;
+; GFX11-LABEL: sext_in_reg_i1_i32:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_clause 0x1
+; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT: s_mov_b32 s3, 0x31016000
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: s_bfe_i32 s2, s2, 0x10000
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: v_mov_b32_e32 v0, s2
+; GFX11-NEXT: s_mov_b32 s2, -1
+; GFX11-NEXT: buffer_store_b32 v0, off, s[0:3], 0
+; GFX11-NEXT: s_endpgm
%shl = shl i32 %in, 31
%sext = ashr i32 %shl, 31
store i32 %sext, ptr addrspace(1) %out
ret void
}
-; FUNC-LABEL: {{^}}sext_in_reg_i8_to_i32:
-; GCN: s_add_i32 [[VAL:s[0-9]+]],
-; GCN: s_sext_i32_i8 [[EXTRACT:s[0-9]+]], [[VAL]]
-; GCN: v_mov_b32_e32 [[VEXTRACT:v[0-9]+]], [[EXTRACT]]
-; GCN: buffer_store_dword [[VEXTRACT]],
-; EG: MEM_{{.*}} STORE_{{.*}} [[RES:T[0-9]+\.[XYZW]]], [[ADDR:T[0-9]+.[XYZW]]]
-; EG: ADD_INT
-; EG-NEXT: BFE_INT [[RES]], {{.*}}, 0.0, literal
-; EG-NEXT: LSHR * [[ADDR]]
define amdgpu_kernel void @sext_in_reg_i8_to_i32(ptr addrspace(1) %out, i32 %a, i32 %b) #0 {
+; SI-LABEL: sext_in_reg_i8_to_i32:
+; SI: ; %bb.0:
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_add_i32 s2, s2, s3
+; SI-NEXT: s_sext_i32_i8 s2, s2
+; SI-NEXT: s_mov_b32 s6, -1
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_mov_b32_e32 v0, s2
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
+; SI-NEXT: s_endpgm
+;
+; EG-LABEL: sext_in_reg_i8_to_i32:
+; EG: ; %bb.0:
+; EG-NEXT: ALU 3, @4, KC0[CB0:0-32], KC1[]
+; EG-NEXT: MEM_RAT_CACHELESS STORE_RAW T0.X, T1.X, 1
+; EG-NEXT: CF_END
+; EG-NEXT: PAD
+; EG-NEXT: ALU clause starting at 4:
+; EG-NEXT: ADD_INT * T0.W, KC0[2].Z, KC0[2].W,
+; EG-NEXT: BFE_INT T0.X, PV.W, 0.0, literal.x,
+; EG-NEXT: LSHR * T1.X, KC0[2].Y, literal.y,
+; EG-NEXT: 8(1.121039e-44), 2(2.802597e-45)
+;
+; GFX11-LABEL: sext_in_reg_i8_to_i32:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: s_add_i32 s2, s2, s3
+; GFX11-NEXT: s_mov_b32 s3, 0x31016000
+; GFX11-NEXT: s_sext_i32_i8 s2, s2
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: v_mov_b32_e32 v0, s2
+; GFX11-NEXT: s_mov_b32 s2, -1
+; GFX11-NEXT: buffer_store_b32 v0, off, s[0:3], 0
+; GFX11-NEXT: s_endpgm
%c = add i32 %a, %b ; add to prevent folding into extload
%shl = shl i32 %c, 24
%ashr = ashr i32 %shl, 24
@@ -40,17 +100,46 @@ define amdgpu_kernel void @sext_in_reg_i8_to_i32(ptr addrspace(1) %out, i32 %a,
ret void
}
-; FUNC-LABEL: {{^}}sext_in_reg_i16_to_i32:
-; GCN: s_add_i32 [[VAL:s[0-9]+]],
-; GCN: s_sext_i32_i16 [[EXTRACT:s[0-9]+]], [[VAL]]
-; GCN: v_mov_b32_e32 [[VEXTRACT:v[0-9]+]], [[EXTRACT]]
-; GCN: buffer_store_dword [[VEXTRACT]],
-; EG: MEM_{{.*}} STORE_{{.*}} [[RES:T[0-9]+\.[XYZW]]], [[ADDR:T[0-9]+.[XYZW]]]
-; EG: ADD_INT
-; EG-NEXT: BFE_INT [[RES]], {{.*}}, 0.0, literal
-; EG-NEXT: LSHR * [[ADDR]]
define amdgpu_kernel void @sext_in_reg_i16_to_i32(ptr addrspace(1) %out, i32 %a, i32 %b) #0 {
+; SI-LABEL: sext_in_reg_i16_to_i32:
+; SI: ; %bb.0:
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_add_i32 s2, s2, s3
+; SI-NEXT: s_sext_i32_i16 s2, s2
+; SI-NEXT: s_mov_b32 s6, -1
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_mov_b32_e32 v0, s2
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
+; SI-NEXT: s_endpgm
+;
+; EG-LABEL: sext_in_reg_i16_to_i32:
+; EG: ; %bb.0:
+; EG-NEXT: ALU 3, @4, KC0[CB0:0-32], KC1[]
+; EG-NEXT: MEM_RAT_CACHELESS STORE_RAW T0.X, T1.X, 1
+; EG-NEXT: CF_END
+; EG-NEXT: PAD
+; EG-NEXT: ALU clause starting at 4:
+; EG-NEXT: ADD_INT * T0.W, KC0[2].Z, KC0[2].W,
+; EG-NEXT: BFE_INT T0.X, PV.W, 0.0, literal.x,
+; EG-NEXT: LSHR * T1.X, KC0[2].Y, literal.y,
+; EG-NEXT: 16(2.242078e-44), 2(2.802597e-45)
+;
+; GFX11-LABEL: sext_in_reg_i16_to_i32:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: s_add_i32 s2, s2, s3
+; GFX11-NEXT: s_mov_b32 s3, 0x31016000
+; GFX11-NEXT: s_sext_i32_i16 s2, s2
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: v_mov_b32_e32 v0, s2
+; GFX11-NEXT: s_mov_b32 s2, -1
+; GFX11-NEXT: buffer_store_b32 v0, off, s[0:3], 0
+; GFX11-NEXT: s_endpgm
%c = add i32 %a, %b ; add to prevent folding into extload
%shl = shl i32 %c, 16
%ashr = ashr i32 %shl, 16
@@ -58,17 +147,46 @@ define amdgpu_kernel void @sext_in_reg_i16_to_i32(ptr addrspace(1) %out, i32 %a,
ret void
}
-; FUNC-LABEL: {{^}}sext_in_reg_i8_to_v1i32:
-; GCN: s_add_i32 [[VAL:s[0-9]+]],
-; GCN: s_sext_i32_i8 [[EXTRACT:s[0-9]+]], [[VAL]]
-; GCN: v_mov_b32_e32 [[VEXTRACT:v[0-9]+]], [[EXTRACT]]
-; GCN: buffer_store_dword [[VEXTRACT]],
-; EG: MEM_{{.*}} STORE_{{.*}} [[RES:T[0-9]+\.[XYZW]]], [[ADDR:T[0-9]+.[XYZW]]]
-; EG: ADD_INT
-; EG-NEXT: BFE_INT [[RES]], {{.*}}, 0.0, literal
-; EG-NEXT: LSHR * [[ADDR]]
define amdgpu_kernel void @sext_in_reg_i8_to_v1i32(ptr addrspace(1) %out, <1 x i32> %a, <1 x i32> %b) #0 {
+; SI-LABEL: sext_in_reg_i8_to_v1i32:
+; SI: ; %bb.0:
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_add_i32 s2, s2, s3
+; SI-NEXT: s_sext_i32_i8 s2, s2
+; SI-NEXT: s_mov_b32 s6, -1
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_mov_b32_e32 v0, s2
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
+; SI-NEXT: s_endpgm
+;
+; EG-LABEL: sext_in_reg_i8_to_v1i32:
+; EG: ; %bb.0:
+; EG-NEXT: ALU 3, @4, KC0[CB0:0-32], KC1[]
+; EG-NEXT: MEM_RAT_CACHELESS STORE_RAW T0.X, T1.X, 1
+; EG-NEXT: CF_END
+; EG-NEXT: PAD
+; EG-NEXT: ALU clause starting at 4:
+; EG-NEXT: ADD_INT * T0.W, KC0[2].Z, KC0[2].W,
+; EG-NEXT: BFE_INT T0.X, PV.W, 0.0, literal.x,
+; EG-NEXT: LSHR * T1.X, KC0[2].Y, literal.y,
+; EG-NEXT: 8(1.121039e-44), 2(2.802597e-45)
+;
+; GFX11-LABEL: sext_in_reg_i8_to_v1i32:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: s_add_i32 s2, s2, s3
+; GFX11-NEXT: s_mov_b32 s3, 0x31016000
+; GFX11-NEXT: s_sext_i32_i8 s2, s2
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: v_mov_b32_e32 v0, s2
+; GFX11-NEXT: s_mov_b32 s2, -1
+; GFX11-NEXT: buffer_store_b32 v0, off, s[0:3], 0
+; GFX11-NEXT: s_endpgm
%c = add <1 x i32> %a, %b ; add to prevent folding into extload
%shl = shl <1 x i32> %c, <i32 24>
%ashr = ashr <1 x i32> %shl, <i32 24>
@@ -76,13 +194,53 @@ define amdgpu_kernel void @sext_in_reg_i8_to_v1i32(ptr addrspace(1) %out, <1 x i
ret void
}
-; FUNC-LABEL: {{^}}sext_in_reg_i1_to_i64:
-; GCN: s_lshl_b64 [[VAL:s\[[0-9]+:[0-9]+\]]]
-; GCN-DAG: s_bfe_i64 s[[[SLO:[0-9]+]]:[[SHI:[0-9]+]]], [[VAL]], 0x10000
-; GCN-DAG: v_mov_b32_e32 v[[VLO:[0-9]+]], s[[SLO]]
-; GCN-DAG: v_mov_b32_e32 v[[VHI:[0-9]+]], s[[SHI]]
-; GCN: buffer_store_dwordx2 v[[[VLO]]:[[VHI]]]
define amdgpu_kernel void @sext_in_reg_i1_to_i64(ptr addrspace(1) %out, i64 %a, i64 %b) #0 {
+; SI-LABEL: sext_in_reg_i1_to_i64:
+; SI: ; %bb.0:
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_load_dword s4, s[4:5], 0xd
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_lshl_b64 s[4:5], s[2:3], s4
+; SI-NEXT: s_bfe_i64 s[4:5], s[4:5], 0x10000
+; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: v_mov_b32_e32 v0, s4
+; SI-NEXT: v_mov_b32_e32 v1, s5
+; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; SI-NEXT: s_endpgm
+;
+; EG-LABEL: sext_in_reg_i1_to_i64:
+; EG: ; %bb.0:
+; EG-NEXT: ALU 9, @4, KC0[CB0:0-32], KC1[]
+; EG-NEXT: MEM_RAT_CACHELESS STORE_RAW T0.XY, T1.X, 1
+; EG-NEXT: CF_END
+; EG-NEXT: PAD
+; EG-NEXT: ALU clause starting at 4:
+; EG-NEXT: AND_INT * T0.W, KC0[3].Y, literal.x,
+; EG-NEXT: 31(4.344025e-44), 0(0.000000e+00)
+; EG-NEXT: LSHL T0.W, KC0[2].W, PV.W,
+; EG-NEXT: AND_INT * T1.W, KC0[3].Y, literal.x,
+; EG-NEXT: 32(4.484155e-44), 0(0.000000e+00)
+; EG-NEXT: CNDE_INT * T0.W, PS, PV.W, 0.0,
+; EG-NEXT: BFE_INT T0.X, PV.W, 0.0, 1,
+; EG-NEXT: LSHR * T1.X, KC0[2].Y, literal.x,
+; EG-NEXT: 2(2.802597e-45), 0(0.000000e+00)
+; EG-NEXT: MOV * T0.Y, PV.X,
+;
+; GFX11-LABEL: sext_in_reg_i1_to_i64:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: s_load_b32 s3, s[4:5], 0x34
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: s_lshl_b64 s[2:3], s[2:3], s3
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_bfe_i64 s[4:5], s[2:3], 0x10000
+; GFX11-NEXT: s_mov_b32 s3, 0x31016000
+; GFX11-NEXT: v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v1, s5
+; GFX11-NEXT: s_mov_b32 s2, -1
+; GFX11-NEXT: buffer_store_b64 v[0:1], off, s[0:3], 0
+; GFX11-NEXT: s_endpgm
%c = shl i64 %a, %b
%shl = shl i64 %c, 63
%ashr = ashr i64 %shl, 63
@@ -90,13 +248,54 @@ define amdgpu_kernel void @sext_in_reg_i1_to_i64(ptr addrspace(1) %out, i64 %a,
ret void
}
-; FUNC-LABEL: {{^}}sext_in_reg_i8_to_i64:
-; GCN: s_lshl_b64 [[VAL:s\[[0-9]+:[0-9]+\]]]
-; GCN-DAG: s_bfe_i64 s[[[SLO:[0-9]+]]:[[SHI:[0-9]+]]], [[VAL]], 0x80000
-; GCN-DAG: v_mov_b32_e32 v[[VLO:[0-9]+]], s[[SLO]]
-; GCN-DAG: v_mov_b32_e32 v[[VHI:[0-9]+]], s[[SHI]]
-; GCN: buffer_store_dwordx2 v[[[VLO]]:[[VHI]]]
define amdgpu_kernel void @sext_in_reg_i8_to_i64(ptr addrspace(1) %out, i64 %a, i64 %b) #0 {
+; SI-LABEL: sext_in_reg_i8_to_i64:
+; SI: ; %bb.0:
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_load_dword s4, s[4:5], 0xd
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_lshl_b64 s[4:5], s[2:3], s4
+; SI-NEXT: s_bfe_i64 s[4:5], s[4:5], 0x80000
+; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: v_mov_b32_e32 v0, s4
+; SI-NEXT: v_mov_b32_e32 v1, s5
+; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; SI-NEXT: s_endpgm
+;
+; EG-LABEL: sext_in_reg_i8_to_i64:
+; EG: ; %bb.0:
+; EG-NEXT: ALU 10, @4, KC0[CB0:0-32], KC1[]
+; EG-NEXT: MEM_RAT_CACHELESS STORE_RAW T0.XY, T1.X, 1
+; EG-NEXT: CF_END
+; EG-NEXT: PAD
+; EG-NEXT: ALU clause starting at 4:
+; EG-NEXT: AND_INT * T0.W, KC0[3].Y, literal.x,
+; EG-NEXT: 31(4.344025e-44), 0(0.000000e+00)
+; EG-NEXT: LSHL T0.W, KC0[2].W, PV.W,
+; EG-NEXT: AND_INT * T1.W, KC0[3].Y, literal.x,
+; EG-NEXT: 32(4.484155e-44), 0(0.000000e+00)
+; EG-NEXT: CNDE_INT * T0.W, PS, PV.W, 0.0,
+; EG-NEXT: BFE_INT T0.X, PV.W, 0.0, literal.x,
+; EG-NEXT: LSHR * T1.X, KC0[2].Y, literal.y,
+; EG-NEXT: 8(1.121039e-44), 2(2.802597e-45)
+; EG-NEXT: ASHR * T0.Y, PV.X, literal.x,
+; EG-NEXT: 31(4.344025e-44), 0(0.000000e+00)
+;
+; GFX11-LABEL: sext_in_reg_i8_to_i64:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: s_load_b32 s3, s[4:5], 0x34
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: s_lshl_b64 s[2:3], s[2:3], s3
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_bfe_i64 s[4:5], s[2:3], 0x80000
+; GFX11-NEXT: s_mov_b32 s3, 0x31016000
+; GFX11-NEXT: v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v1, s5
+; GFX11-NEXT: s_mov_b32 s2, -1
+; GFX11-NEXT: buffer_store_b64 v[0:1], off, s[0:3], 0
+; GFX11-NEXT: s_endpgm
%c = shl i64 %a, %b
%shl = shl i64 %c, 56
%ashr = ashr i64 %shl, 56
@@ -104,14 +303,55 @@ define amdgpu_kernel void @sext_in_reg_i8_to_i64(ptr addrspace(1) %out, i64 %a,
ret void
}
-; FUNC-LABEL: {{^}}sext_in_reg_i16_to_i64:
-; GCN: s_lshl_b64 [[VAL:s\[[0-9]+:[0-9]+\]]]
-; GCN-DAG: s_bfe_i64 s[[[SLO:[0-9]+]]:[[SHI:[0-9]+]]], [[VAL]], 0x100000
-; GCN-DAG: v_mov_b32_e32 v[[VLO:[0-9]+]], s[[SLO]]
-; GCN-DAG: v_mov_b32_e32 v[[VHI:[0-9]+]], s[[SHI]]
-; GCN: buffer_store_dwordx2 v[[[VLO]]:[[VHI]]]
define amdgpu_kernel void @sext_in_reg_i16_to_i64(ptr addrspace(1) %out, i64 %a, i64 %b) #0 {
+; SI-LABEL: sext_in_reg_i16_to_i64:
+; SI: ; %bb.0:
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_load_dword s4, s[4:5], 0xd
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_lshl_b64 s[4:5], s[2:3], s4
+; SI-NEXT: s_bfe_i64 s[4:5], s[4:5], 0x100000
+; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: v_mov_b32_e32 v0, s4
+; SI-NEXT: v_mov_b32_e32 v1, s5
+; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; SI-NEXT: s_endpgm
+;
+; EG-LABEL: sext_in_reg_i16_to_i64:
+; EG: ; %bb.0:
+; EG-NEXT: ALU 10, @4, KC0[CB0:0-32], KC1[]
+; EG-NEXT: MEM_RAT_CACHELESS STORE_RAW T0.XY, T1.X, 1
+; EG-NEXT: CF_END
+; EG-NEXT: PAD
+; EG-NEXT: ALU clause starting at 4:
+; EG-NEXT: AND_INT * T0.W, KC0[3].Y, literal.x,
+; EG-NEXT: 31(4.344025e-44), 0(0.000000e+00)
+; EG-NEXT: LSHL T0.W, KC0[2].W, PV.W,
+; EG-NEXT: AND_INT * T1.W, KC0[3].Y, literal.x,
+; EG-NEXT: 32(4.484155e-44), 0(0.000000e+00)
+; EG-NEXT: CNDE_INT * T0.W, PS, PV.W, 0.0,
+; EG-NEXT: BFE_INT T0.X, PV.W, 0.0, literal.x,
+; EG-NEXT: LSHR * T1.X, KC0[2].Y, literal.y,
+; EG-NEXT: 16(2.242078e-44), 2(2.802597e-45)
+; EG-NEXT: ASHR * T0.Y, PV.X, literal.x,
+; EG-NEXT: 31(4.344025e-44), 0(0.000000e+00)
+;
+; GFX11-LABEL: sext_in_reg_i16_to_i64:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: s_load_b32 s3, s[4:5], 0x34
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: s_lshl_b64 s[2:3], s[2:3], s3
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_bfe_i64 s[4:5], s[2:3], 0x100000
+; GFX11-NEXT: s_mov_b32 s3, 0x31016000
+; GFX11-NEXT: v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v1, s5
+; GFX11-NEXT: s_mov_b32 s2, -1
+; GFX11-NEXT: buffer_store_b64 v[0:1], off, s[0:3], 0
+; GFX11-NEXT: s_endpgm
%c = shl i64 %a, %b
%shl = shl i64 %c, 48
%ashr = ashr i64 %shl, 48
@@ -119,13 +359,53 @@ define amdgpu_kernel void @sext_in_reg_i16_to_i64(ptr addrspace(1) %out, i64 %a,
ret void
}
-; FUNC-LABEL: {{^}}sext_in_reg_i32_to_i64:
-; GCN: s_lshl_b64 [[VAL:s\[[0-9]+:[0-9]+\]]]
-; GCN-DAG: s_bfe_i64 s[[[SLO:[0-9]+]]:[[SHI:[0-9]+]]], [[VAL]], 0x200000
-; GCN-DAG: v_mov_b32_e32 v[[VLO:[0-9]+]], s[[SLO]]
-; GCN-DAG: v_mov_b32_e32 v[[VHI:[0-9]+]], s[[SHI]]
-; GCN: buffer_store_dwordx2 v[[[VLO]]:[[VHI]]]
define amdgpu_kernel void @sext_in_reg_i32_to_i64(ptr addrspace(1) %out, i64 %a, i64 %b) #0 {
+; SI-LABEL: sext_in_reg_i32_to_i64:
+; SI: ; %bb.0:
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_load_dword s4, s[4:5], 0xd
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_lshl_b64 s[4:5], s[2:3], s4
+; SI-NEXT: s_bfe_i64 s[4:5], s[4:5], 0x200000
+; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: v_mov_b32_e32 v0, s4
+; SI-NEXT: v_mov_b32_e32 v1, s5
+; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; SI-NEXT: s_endpgm
+;
+; EG-LABEL: sext_in_reg_i32_to_i64:
+; EG: ; %bb.0:
+; EG-NEXT: ALU 9, @4, KC0[CB0:0-32], KC1[]
+; EG-NEXT: MEM_RAT_CACHELESS STORE_RAW T0.XY, T1.X, 1
+; EG-NEXT: CF_END
+; EG-NEXT: PAD
+; EG-NEXT: ALU clause starting at 4:
+; EG-NEXT: AND_INT * T0.W, KC0[3].Y, literal.x,
+; EG-NEXT: 31(4.344025e-44), 0(0.000000e+00)
+; EG-NEXT: LSHL T0.W, KC0[2].W, PV.W,
+; EG-NEXT: AND_INT * T1.W, KC0[3].Y, literal.x,
+; EG-NEXT: 32(4.484155e-44), 0(0.000000e+00)
+; EG-NEXT: CNDE_INT T0.X, PS, PV.W, 0.0,
+; EG-NEXT: LSHR * T1.X, KC0[2].Y, literal.x,
+; EG-NEXT: 2(2.802597e-45), 0(0.000000e+00)
+; EG-NEXT: ASHR * T0.Y, PV.X, literal.x,
+; EG-NEXT: 31(4.344025e-44), 0(0.000000e+00)
+;
+; GFX11-LABEL: sext_in_reg_i32_to_i64:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: s_load_b32 s3, s[4:5], 0x34
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: s_lshl_b64 s[2:3], s[2:3], s3
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_bfe_i64 s[4:5], s[2:3], 0x200000
+; GFX11-NEXT: s_mov_b32 s3, 0x31016000
+; GFX11-NEXT: v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v1, s5
+; GFX11-NEXT: s_mov_b32 s2, -1
+; GFX11-NEXT: buffer_store_b64 v[0:1], off, s[0:3], 0
+; GFX11-NEXT: s_endpgm
%c = shl i64 %a, %b
%shl = shl i64 %c, 32
%ashr = ashr i64 %shl, 32
@@ -148,19 +428,70 @@ define amdgpu_kernel void @sext_in_reg_i32_to_i64(ptr addrspace(1) %out, i64 %a,
; ret void
; }
-; FUNC-LABEL: {{^}}v_sext_in_reg_i1_to_i64:
-; SI: buffer_load_dwordx2
-; SI: v_lshl_b64 v[[[VAL_LO:[0-9]+]]:[[VAL_HI:[0-9]+]]]
-; GFX89: {{flat|global}}_load_dwordx2
-; GFX89: v_lshlrev_b64 v[[[VAL_LO:[0-9]+]]:[[VAL_HI:[0-9]+]]]
-; GCN: v_bfe_i32 v[[LO:[0-9]+]], v[[VAL_LO]], 0, 1
-; GCN: v_ashrrev_i32_e32 v[[HI:[0-9]+]], 31, v[[LO]]
-; SI: buffer_store_dwordx2 v[[[LO]]:[[HI]]]
-; GFX89: {{flat|global}}_store_dwordx2 v{{.+}}, v[[[LO]]:[[HI]]]
define amdgpu_kernel void @v_sext_in_reg_i1_to_i64(ptr addrspace(1) %out, ptr addrspace(1) %aptr, ptr addrspace(1) %bptr) #0 {
+; SI-LABEL: v_sext_in_reg_i1_to_i64:
+; SI: ; %bb.0:
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, 0
+; SI-NEXT: v_lshlrev_b32_e32 v0, 3, v0
+; SI-NEXT: v_mov_b32_e32 v1, 0
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b64 s[4:5], s[2:3]
+; SI-NEXT: buffer_load_dwordx2 v[2:3], v[0:1], s[4:7], 0 addr64
+; SI-NEXT: s_mov_b64 s[2:3], s[6:7]
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshl_b64 v[2:3], v[2:3], v2
+; SI-NEXT: v_bfe_i32 v2, v2, 0, 1
+; SI-NEXT: v_ashrrev_i32_e32 v3, 31, v2
+; SI-NEXT: buffer_store_dwordx2 v[2:3], v[0:1], s[0:3], 0 addr64
+; SI-NEXT: s_endpgm
+;
+; EG-LABEL: v_sext_in_reg_i1_to_i64:
+; EG: ; %bb.0:
+; EG-NEXT: ALU 2, @8, KC0[CB0:0-32], KC1[]
+; EG-NEXT: TEX 0 @6
+; EG-NEXT: ALU 10, @11, KC0[CB0:0-32], KC1[]
+; EG-NEXT: MEM_RAT_CACHELESS STORE_RAW T0.XY, T1.X, 1
+; EG-NEXT: CF_END
+; EG-NEXT: PAD
+; EG-NEXT: Fetch clause starting at 6:
+; EG-NEXT: VTX_READ_32 T0.X, T0.X, 0, #1
+; EG-NEXT: ALU clause starting at 8:
+; EG-NEXT: LSHL * T0.W, T0.X, literal.x,
+; EG-NEXT: 3(4.203895e-45), 0(0.000000e+00)
+; EG-NEXT: ADD_INT * T0.X, KC0[2].Z, PV.W,
+; EG-NEXT: ALU clause starting at 11:
+; EG-NEXT: AND_INT * T1.W, T0.X, literal.x,
+; EG-NEXT: 31(4.344025e-44), 0(0.000000e+00)
+; EG-NEXT: LSHL T1.W, T0.X, PV.W,
+; EG-NEXT: AND_INT * T2.W, T0.X, literal.x,
+; EG-NEXT: 32(4.484155e-44), 0(0.000000e+00)
+; EG-NEXT: CNDE_INT * T1.W, PS, PV.W, 0.0,
+; EG-NEXT: BFE_INT T0.X, PV.W, 0.0, 1,
+; EG-NEXT: ADD_INT * T0.W, KC0[2].Y, T0.W,
+; EG-NEXT: LSHR T1.X, PV.W, literal.x,
+; EG-NEXT: MOV * T0.Y, PV.X,
+; EG-NEXT: 2(2.802597e-45), 0(0.000000e+00)
+;
+; GFX11-LABEL: v_sext_in_reg_i1_to_i64:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshlrev_b32_e32 v2, 3, v0
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: global_load_b64 v[0:1], v2, s[2:3]
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_lshlrev_b64 v[0:1], v0, v[0:1]
+; GFX11-NEXT: v_bfe_i32 v0, v0, 0, 1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_ashrrev_i32_e32 v1, 31, v0
+; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX11-NEXT: s_endpgm
%tid = call i32 @llvm.amdgcn.workitem.id.x()
%a.gep = getelementptr i64, ptr addrspace(1) %aptr, i32 %tid
%b.gep = getelementptr i64, ptr addrspace(1) %aptr, i32 %tid
@@ -175,19 +506,71 @@ define amdgpu_kernel void @v_sext_in_reg_i1_to_i64(ptr addrspace(1) %out, ptr ad
ret void
}
-; FUNC-LABEL: {{^}}v_sext_in_reg_i8_to_i64:
-; SI: buffer_load_dwordx2
-; SI: v_lshl_b64 v[[[VAL_LO:[0-9]+]]:[[VAL_HI:[0-9]+]]]
-; GFX89: {{flat|global}}_load_dwordx2
-; GFX89: v_lshlrev_b64 v[[[VAL_LO:[0-9]+]]:[[VAL_HI:[0-9]+]]]
-; GCN: v_bfe_i32 v[[LO:[0-9]+]], v[[VAL_LO]], 0, 8
-; GCN: v_ashrrev_i32_e32 v[[HI:[0-9]+]], 31, v[[LO]]
-; SI: buffer_store_dwordx2 v[[[LO]]:[[HI]]]
-; GFX89: {{flat|global}}_store_dwordx2 v{{.+}}, v[[[LO]]:[[HI]]]
define amdgpu_kernel void @v_sext_in_reg_i8_to_i64(ptr addrspace(1) %out, ptr addrspace(1) %aptr, ptr addrspace(1) %bptr) #0 {
+; SI-LABEL: v_sext_in_reg_i8_to_i64:
+; SI: ; %bb.0:
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, 0
+; SI-NEXT: v_lshlrev_b32_e32 v0, 3, v0
+; SI-NEXT: v_mov_b32_e32 v1, 0
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b64 s[4:5], s[2:3]
+; SI-NEXT: buffer_load_dwordx2 v[2:3], v[0:1], s[4:7], 0 addr64
+; SI-NEXT: s_mov_b64 s[2:3], s[6:7]
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshl_b64 v[2:3], v[2:3], v2
+; SI-NEXT: v_bfe_i32 v2, v2, 0, 8
+; SI-NEXT: v_ashrrev_i32_e32 v3, 31, v2
+; SI-NEXT: buffer_store_dwordx2 v[2:3], v[0:1], s[0:3], 0 addr64
+; SI-NEXT: s_endpgm
+;
+; EG-LABEL: v_sext_in_reg_i8_to_i64:
+; EG: ; %bb.0:
+; EG-NEXT: ALU 2, @8, KC0[CB0:0-32], KC1[]
+; EG-NEXT: TEX 0 @6
+; EG-NEXT: ALU 11, @11, KC0[CB0:0-32], KC1[]
+; EG-NEXT: MEM_RAT_CACHELESS STORE_RAW T0.XY, T1.X, 1
+; EG-NEXT: CF_END
+; EG-NEXT: PAD
+; EG-NEXT: Fetch clause starting at 6:
+; EG-NEXT: VTX_READ_32 T0.X, T0.X, 0, #1
+; EG-NEXT: ALU clause starting at 8:
+; EG-NEXT: LSHL * T0.W, T0.X, literal.x,
+; EG-NEXT: 3(4.203895e-45), 0(0.000000e+00)
+; EG-NEXT: ADD_INT * T0.X, KC0[2].Z, PV.W,
+; EG-NEXT: ALU clause starting at 11:
+; EG-NEXT: AND_INT * T1.W, T0.X, literal.x,
+; EG-NEXT: 31(4.344025e-44), 0(0.000000e+00)
+; EG-NEXT: LSHL T1.W, T0.X, PV.W,
+; EG-NEXT: AND_INT * T2.W, T0.X, literal.x,
+; EG-NEXT: 32(4.484155e-44), 0(0.000000e+00)
+; EG-NEXT: CNDE_INT * T1.W, PS, PV.W, 0.0,
+; EG-NEXT: BFE_INT T0.X, PV.W, 0.0, literal.x,
+; EG-NEXT: ADD_INT * T0.W, KC0[2].Y, T0.W,
+; EG-NEXT: 8(1.121039e-44), 0(0.000000e+00)
+; EG-NEXT: LSHR T1.X, PV.W, literal.x,
+; EG-NEXT: ASHR * T0.Y, PV.X, literal.y,
+; EG-NEXT: 2(2.802597e-45), 31(4.344025e-44)
+;
+; GFX11-LABEL: v_sext_in_reg_i8_to_i64:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshlrev_b32_e32 v2, 3, v0
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: global_load_b64 v[0:1], v2, s[2:3]
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_lshlrev_b64 v[0:1], v0, v[0:1]
+; GFX11-NEXT: v_bfe_i32 v0, v0, 0, 8
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_ashrrev_i32_e32 v1, 31, v0
+; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX11-NEXT: s_endpgm
%tid = call i32 @llvm.amdgcn.workitem.id.x()
%a.gep = getelementptr i64, ptr addrspace(1) %aptr, i32 %tid
%b.gep = getelementptr i64, ptr addrspace(1) %aptr, i32 %tid
@@ -202,19 +585,71 @@ define amdgpu_kernel void @v_sext_in_reg_i8_to_i64(ptr addrspace(1) %out, ptr ad
ret void
}
-; FUNC-LABEL: {{^}}v_sext_in_reg_i16_to_i64:
-; SI: buffer_load_dwordx2
-; SI: v_lshl_b64 v[[[VAL_LO:[0-9]+]]:[[VAL_HI:[0-9]+]]]
-; GFX89: {{flat|global}}_load_dwordx2
-; GFX89: v_lshlrev_b64 v[[[VAL_LO:[0-9]+]]:[[VAL_HI:[0-9]+]]]
-; GCN: v_bfe_i32 v[[LO:[0-9]+]], v[[VAL_LO]], 0, 16
-; GCN: v_ashrrev_i32_e32 v[[HI:[0-9]+]], 31, v[[LO]]
-; SI: buffer_store_dwordx2 v[[[LO]]:[[HI]]]
-; GFX89: {{flat|global}}_store_dwordx2 v{{.+}}, v[[[LO]]:[[HI]]]
define amdgpu_kernel void @v_sext_in_reg_i16_to_i64(ptr addrspace(1) %out, ptr addrspace(1) %aptr, ptr addrspace(1) %bptr) #0 {
+; SI-LABEL: v_sext_in_reg_i16_to_i64:
+; SI: ; %bb.0:
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, 0
+; SI-NEXT: v_lshlrev_b32_e32 v0, 3, v0
+; SI-NEXT: v_mov_b32_e32 v1, 0
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b64 s[4:5], s[2:3]
+; SI-NEXT: buffer_load_dwordx2 v[2:3], v[0:1], s[4:7], 0 addr64
+; SI-NEXT: s_mov_b64 s[2:3], s[6:7]
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshl_b64 v[2:3], v[2:3], v2
+; SI-NEXT: v_bfe_i32 v2, v2, 0, 16
+; SI-NEXT: v_ashrrev_i32_e32 v3, 31, v2
+; SI-NEXT: buffer_store_dwordx2 v[2:3], v[0:1], s[0:3], 0 addr64
+; SI-NEXT: s_endpgm
+;
+; EG-LABEL: v_sext_in_reg_i16_to_i64:
+; EG: ; %bb.0:
+; EG-NEXT: ALU 2, @8, KC0[CB0:0-32], KC1[]
+; EG-NEXT: TEX 0 @6
+; EG-NEXT: ALU 11, @11, KC0[CB0:0-32], KC1[]
+; EG-NEXT: MEM_RAT_CACHELESS STORE_RAW T0.XY, T1.X, 1
+; EG-NEXT: CF_END
+; EG-NEXT: PAD
+; EG-NEXT: Fetch clause starting at 6:
+; EG-NEXT: VTX_READ_32 T0.X, T0.X, 0, #1
+; EG-NEXT: ALU clause starting at 8:
+; EG-NEXT: LSHL * T0.W, T0.X, literal.x,
+; EG-NEXT: 3(4.203895e-45), 0(0.000000e+00)
+; EG-NEXT: ADD_INT * T0.X, KC0[2].Z, PV.W,
+; EG-NEXT: ALU clause starting at 11:
+; EG-NEXT: AND_INT * T1.W, T0.X, literal.x,
+; EG-NEXT: 31(4.344025e-44), 0(0.000000e+00)
+; EG-NEXT: LSHL T1.W, T0.X, PV.W,
+; EG-NEXT: AND_INT * T2.W, T0.X, literal.x,
+; EG-NEXT: 32(4.484155e-44), 0(0.000000e+00)
+; EG-NEXT: CNDE_INT * T1.W, PS, PV.W, 0.0,
+; EG-NEXT: BFE_INT T0.X, PV.W, 0.0, literal.x,
+; EG-NEXT: ADD_INT * T0.W, KC0[2].Y, T0.W,
+; EG-NEXT: 16(2.242078e-44), 0(0.000000e+00)
+; EG-NEXT: LSHR T1.X, PV.W, literal.x,
+; EG-NEXT: ASHR * T0.Y, PV.X, literal.y,
+; EG-NEXT: 2(2.802597e-45), 31(4.344025e-44)
+;
+; GFX11-LABEL: v_sext_in_reg_i16_to_i64:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshlrev_b32_e32 v2, 3, v0
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: global_load_b64 v[0:1], v2, s[2:3]
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_lshlrev_b64 v[0:1], v0, v[0:1]
+; GFX11-NEXT: v_bfe_i32 v0, v0, 0, 16
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_ashrrev_i32_e32 v1, 31, v0
+; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX11-NEXT: s_endpgm
%tid = call i32 @llvm.amdgcn.workitem.id.x()
%a.gep = getelementptr i64, ptr addrspace(1) %aptr, i32 %tid
%b.gep = getelementptr i64, ptr addrspace(1) %aptr, i32 %tid
@@ -229,16 +664,65 @@ define amdgpu_kernel void @v_sext_in_reg_i16_to_i64(ptr addrspace(1) %out, ptr a
ret void
}
-; FUNC-LABEL: {{^}}v_sext_in_reg_i32_to_i64:
-; SI: buffer_load_dwordx2
-; SI: v_lshl_b64 v[[[LO:[0-9]+]]:[[HI:[0-9]+]]],
-; GFX89: {{flat|global}}_load_dwordx2
-; GFX89: v_lshlrev_b64 v[[[LO:[0-9]+]]:[[HI:[0-9]+]]],
-; GCN: v_ashrrev_i32_e32 v[[SHR:[0-9]+]], 31, v[[LO]]
-; GFX89: {{flat|global}}_store_dwordx2 v{{.+}}, v[[[LO]]:[[SHR]]]
define amdgpu_kernel void @v_sext_in_reg_i32_to_i64(ptr addrspace(1) %out, ptr addrspace(1) %aptr, ptr addrspace(1) %bptr) #0 {
+; SI-LABEL: v_sext_in_reg_i32_to_i64:
+; SI: ; %bb.0:
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, 0
+; SI-NEXT: v_lshlrev_b32_e32 v0, 3, v0
+; SI-NEXT: v_mov_b32_e32 v1, 0
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b64 s[4:5], s[2:3]
+; SI-NEXT: buffer_load_dwordx2 v[2:3], v[0:1], s[4:7], 0 addr64
+; SI-NEXT: s_mov_b64 s[2:3], s[6:7]
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshl_b64 v[2:3], v[2:3], v2
+; SI-NEXT: v_ashrrev_i32_e32 v3, 31, v2
+; SI-NEXT: buffer_store_dwordx2 v[2:3], v[0:1], s[0:3], 0 addr64
+; SI-NEXT: s_endpgm
+;
+; EG-LABEL: v_sext_in_reg_i32_to_i64:
+; EG: ; %bb.0:
+; EG-NEXT: ALU 2, @8, KC0[CB0:0-32], KC1[]
+; EG-NEXT: TEX 0 @6
+; EG-NEXT: ALU 9, @11, KC0[CB0:0-32], KC1[]
+; EG-NEXT: MEM_RAT_CACHELESS STORE_RAW T0.XY, T1.X, 1
+; EG-NEXT: CF_END
+; EG-NEXT: PAD
+; EG-NEXT: Fetch clause starting at 6:
+; EG-NEXT: VTX_READ_32 T0.X, T0.X, 0, #1
+; EG-NEXT: ALU clause starting at 8:
+; EG-NEXT: LSHL * T0.W, T0.X, literal.x,
+; EG-NEXT: 3(4.203895e-45), 0(0.000000e+00)
+; EG-NEXT: ADD_INT * T0.X, KC0[2].Z, PV.W,
+; EG-NEXT: ALU clause starting at 11:
+; EG-NEXT: AND_INT * T1.W, T0.X, literal.x,
+; EG-NEXT: 31(4.344025e-44), 0(0.000000e+00)
+; EG-NEXT: LSHL T1.W, T0.X, PV.W,
+; EG-NEXT: AND_INT * T2.W, T0.X, literal.x,
+; EG-NEXT: 32(4.484155e-44), 0(0.000000e+00)
+; EG-NEXT: CNDE_INT T0.X, PS, PV.W, 0.0,
+; EG-NEXT: ADD_INT * T0.W, KC0[2].Y, T0.W,
+; EG-NEXT: LSHR T1.X, PV.W, literal.x,
+; EG-NEXT: ASHR * T0.Y, PV.X, literal.y,
+; EG-NEXT: 2(2.802597e-45), 31(4.344025e-44)
+;
+; GFX11-LABEL: v_sext_in_reg_i32_to_i64:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshlrev_b32_e32 v2, 3, v0
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: global_load_b64 v[0:1], v2, s[2:3]
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_lshlrev_b64 v[0:1], v0, v[0:1]
+; GFX11-NEXT: v_ashrrev_i32_e32 v1, 31, v0
+; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX11-NEXT: s_endpgm
%tid = call i32 @llvm.amdgcn.workitem.id.x()
%a.gep = getelementptr i64, ptr addrspace(1) %aptr, i32 %tid
%b.gep = getelementptr i64, ptr addrspace(1) %aptr, i32 %tid
@@ -253,18 +737,48 @@ define amdgpu_kernel void @v_sext_in_reg_i32_to_i64(ptr addrspace(1) %out, ptr a
ret void
}
-; FUNC-LABEL: {{^}}sext_in_reg_i1_in_i32_other_amount:
-; GCN-NOT: s_lshl
-; GCN-NOT: s_ashr
-; GCN: s_bfe_i32 {{s[0-9]+}}, {{s[0-9]+}}, 0x190001
-; EG: MEM_{{.*}} STORE_{{.*}} [[RES:T[0-9]+\.[XYZW]]], [[ADDR:T[0-9]+.[XYZW]]]
-; EG-NOT: BFE
-; EG: ADD_INT
-; EG: LSHL
-; EG: ASHR [[RES]]
-; EG: LSHR {{\*?}} [[ADDR]]
define amdgpu_kernel void @sext_in_reg_i1_in_i32_other_amount(ptr addrspace(1) %out, i32 %a, i32 %b) #0 {
+; SI-LABEL: sext_in_reg_i1_in_i32_other_amount:
+; SI: ; %bb.0:
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_add_i32 s2, s2, s3
+; SI-NEXT: s_bfe_i32 s2, s2, 0x190001
+; SI-NEXT: s_mov_b32 s6, -1
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_mov_b32_e32 v0, s2
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
+; SI-NEXT: s_endpgm
+;
+; EG-LABEL: sext_in_reg_i1_in_i32_other_amount:
+; EG: ; %bb.0:
+; EG-NEXT: ALU 5, @4, KC0[CB0:0-32], KC1[]
+; EG-NEXT: MEM_RAT_CACHELESS STORE_RAW T0.X, T1.X, 1
+; EG-NEXT: CF_END
+; EG-NEXT: PAD
+; EG-NEXT: ALU clause starting at 4:
+; EG-NEXT: ADD_INT * T0.W, KC0[2].Z, KC0[2].W,
+; EG-NEXT: LSHL * T0.W, PV.W, literal.x,
+; EG-NEXT: 6(8.407791e-45), 0(0.000000e+00)
+; EG-NEXT: ASHR T0.X, PV.W, literal.x,
+; EG-NEXT: LSHR * T1.X, KC0[2].Y, literal.y,
+; EG-NEXT: 7(9.809089e-45), 2(2.802597e-45)
+;
+; GFX11-LABEL: sext_in_reg_i1_in_i32_other_amount:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: s_add_i32 s2, s2, s3
+; GFX11-NEXT: s_mov_b32 s3, 0x31016000
+; GFX11-NEXT: s_bfe_i32 s2, s2, 0x190001
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: v_mov_b32_e32 v0, s2
+; GFX11-NEXT: s_mov_b32 s2, -1
+; GFX11-NEXT: buffer_store_b32 v0, off, s[0:3], 0
+; GFX11-NEXT: s_endpgm
%c = add i32 %a, %b
%x = shl i32 %c, 6
%y = ashr i32 %x, 7
@@ -272,22 +786,58 @@ define amdgpu_kernel void @sext_in_reg_i1_in_i32_other_amount(ptr addrspace(1) %
ret void
}
-; FUNC-LABEL: {{^}}sext_in_reg_v2i1_in_v2i32_other_amount:
-; GCN-NOT: s_lshl
-; GCN-NOT: s_ashr
-; GCN-DAG: s_bfe_i32 {{s[0-9]+}}, {{s[0-9]+}}, 0x190001
-; GCN-DAG: s_bfe_i32 {{s[0-9]+}}, {{s[0-9]+}}, 0x190001
-; GCN: s_endpgm
-; EG: MEM_{{.*}} STORE_{{.*}} [[RES:T[0-9]+]]{{\.[XYZW][XYZW]}}, [[ADDR:T[0-9]+.[XYZW]]]
-; EG-NOT: BFE
-; EG: ADD_INT
-; EG: LSHL
-; EG: ASHR [[RES]]
-; EG: LSHL
-; EG: ASHR [[RES]]
-; EG: LSHR {{\*?}} [[ADDR]]
define amdgpu_kernel void @sext_in_reg_v2i1_in_v2i32_other_amount(ptr addrspace(1) %out, <2 x i32> %a, <2 x i32> %b) #0 {
+; SI-LABEL: sext_in_reg_v2i1_in_v2i32_other_amount:
+; SI: ; %bb.0:
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0xb
+; SI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_add_i32 s0, s0, s2
+; SI-NEXT: s_add_i32 s1, s1, s3
+; SI-NEXT: s_bfe_i32 s1, s1, 0x190001
+; SI-NEXT: s_bfe_i32 s0, s0, 0x190001
+; SI-NEXT: s_mov_b32 s6, -1
+; SI-NEXT: v_mov_b32_e32 v0, s0
+; SI-NEXT: v_mov_b32_e32 v1, s1
+; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; SI-NEXT: s_endpgm
+;
+; EG-LABEL: sext_in_reg_v2i1_in_v2i32_other_amount:
+; EG: ; %bb.0:
+; EG-NEXT: ALU 9, @4, KC0[CB0:0-32], KC1[]
+; EG-NEXT: MEM_RAT_CACHELESS STORE_RAW T0.XY, T1.X, 1
+; EG-NEXT: CF_END
+; EG-NEXT: PAD
+; EG-NEXT: ALU clause starting at 4:
+; EG-NEXT: ADD_INT * T0.W, KC0[3].X, KC0[3].Z,
+; EG-NEXT: ADD_INT T1.W, KC0[2].W, KC0[3].Y,
+; EG-NEXT: LSHL * T0.W, PV.W, literal.x,
+; EG-NEXT: 6(8.407791e-45), 0(0.000000e+00)
+; EG-NEXT: ASHR T0.Y, PS, literal.x,
+; EG-NEXT: LSHL * T0.W, PV.W, literal.y,
+; EG-NEXT: 7(9.809089e-45), 6(8.407791e-45)
+; EG-NEXT: ASHR T0.X, PV.W, literal.x,
+; EG-NEXT: LSHR * T1.X, KC0[2].Y, literal.y,
+; EG-NEXT: 7(9.809089e-45), 2(2.802597e-45)
+;
+; GFX11-LABEL: sext_in_reg_v2i1_in_v2i32_other_amount:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_clause 0x1
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x2c
+; GFX11-NEXT: s_load_b64 s[4:5], s[4:5], 0x24
+; GFX11-NEXT: s_mov_b32 s7, 0x31016000
+; GFX11-NEXT: s_mov_b32 s6, -1
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: s_add_i32 s0, s0, s2
+; GFX11-NEXT: s_add_i32 s1, s1, s3
+; GFX11-NEXT: s_bfe_i32 s0, s0, 0x190001
+; GFX11-NEXT: s_bfe_i32 s1, s1, 0x190001
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT: buffer_store_b64 v[0:1], off, s[4:7], 0
+; GFX11-NEXT: s_endpgm
%c = add <2 x i32> %a, %b
%x = shl <2 x i32> %c, <i32 6, i32 6>
%y = ashr <2 x i32> %x, <i32 7, i32 7>
@@ -296,16 +846,54 @@ define amdgpu_kernel void @sext_in_reg_v2i1_in_v2i32_other_amount(ptr addrspace(
}
-; FUNC-LABEL: {{^}}sext_in_reg_v2i1_to_v2i32:
-; GCN: s_bfe_i32 {{s[0-9]+}}, {{s[0-9]+}}, 0x10000
-; GCN: s_bfe_i32 {{s[0-9]+}}, {{s[0-9]+}}, 0x10000
-; GCN: buffer_store_dwordx2
-; EG: MEM_{{.*}} STORE_{{.*}} [[RES:T[0-9]+]]{{\.[XYZW][XYZW]}}, [[ADDR:T[0-9]+.[XYZW]]]
-; EG: BFE_INT [[RES]]
-; EG: BFE_INT [[RES]]
-; EG: LSHR {{\*?}} [[ADDR]]
define amdgpu_kernel void @sext_in_reg_v2i1_to_v2i32(ptr addrspace(1) %out, <2 x i32> %a, <2 x i32> %b) #0 {
+; SI-LABEL: sext_in_reg_v2i1_to_v2i32:
+; SI: ; %bb.0:
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0xb
+; SI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_add_i32 s0, s0, s2
+; SI-NEXT: s_add_i32 s1, s1, s3
+; SI-NEXT: s_bfe_i32 s1, s1, 0x10000
+; SI-NEXT: s_bfe_i32 s0, s0, 0x10000
+; SI-NEXT: s_mov_b32 s6, -1
+; SI-NEXT: v_mov_b32_e32 v0, s0
+; SI-NEXT: v_mov_b32_e32 v1, s1
+; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; SI-NEXT: s_endpgm
+;
+; EG-LABEL: sext_in_reg_v2i1_to_v2i32:
+; EG: ; %bb.0:
+; EG-NEXT: ALU 5, @4, KC0[CB0:0-32], KC1[]
+; EG-NEXT: MEM_RAT_CACHELESS STORE_RAW T0.XY, T1.X, 1
+; EG-NEXT: CF_END
+; EG-NEXT: PAD
+; EG-NEXT: ALU clause starting at 4:
+; EG-NEXT: ADD_INT * T0.W, KC0[3].X, KC0[3].Z,
+; EG-NEXT: BFE_INT T0.Y, PV.W, 0.0, 1,
+; EG-NEXT: ADD_INT * T0.W, KC0[2].W, KC0[3].Y,
+; EG-NEXT: BFE_INT T0.X, PV.W, 0.0, 1,
+; EG-NEXT: LSHR * T1.X, KC0[2].Y, literal.x,
+; EG-NEXT: 2(2.802597e-45), 0(0.000000e+00)
+;
+; GFX11-LABEL: sext_in_reg_v2i1_to_v2i32:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_clause 0x1
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x2c
+; GFX11-NEXT: s_load_b64 s[4:5], s[4:5], 0x24
+; GFX11-NEXT: s_mov_b32 s7, 0x31016000
+; GFX11-NEXT: s_mov_b32 s6, -1
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: s_add_i32 s0, s0, s2
+; GFX11-NEXT: s_add_i32 s1, s1, s3
+; GFX11-NEXT: s_bfe_i32 s0, s0, 0x10000
+; GFX11-NEXT: s_bfe_i32 s1, s1, 0x10000
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT: buffer_store_b64 v[0:1], off, s[4:7], 0
+; GFX11-NEXT: s_endpgm
%c = add <2 x i32> %a, %b ; add to prevent folding into extload
%shl = shl <2 x i32> %c, <i32 31, i32 31>
%ashr = ashr <2 x i32> %shl, <i32 31, i32 31>
@@ -313,20 +901,69 @@ define amdgpu_kernel void @sext_in_reg_v2i1_to_v2i32(ptr addrspace(1) %out, <2 x
ret void
}
-; FUNC-LABEL: {{^}}sext_in_reg_v4i1_to_v4i32:
-; GCN: s_bfe_i32 {{s[0-9]+}}, {{s[0-9]+}}, 0x10000
-; GCN: s_bfe_i32 {{s[0-9]+}}, {{s[0-9]+}}, 0x10000
-; GCN: s_bfe_i32 {{s[0-9]+}}, {{s[0-9]+}}, 0x10000
-; GCN: s_bfe_i32 {{s[0-9]+}}, {{s[0-9]+}}, 0x10000
-; GCN: buffer_store_dwordx4
-; EG: MEM_{{.*}} STORE_{{.*}} [[RES:T[0-9]+]]{{\.[XYZW][XYZW][XYZW][XYZW]}}, [[ADDR:T[0-9]+.[XYZW]]]
-; EG: BFE_INT [[RES]]
-; EG: BFE_INT [[RES]]
-; EG: BFE_INT [[RES]]
-; EG: BFE_INT [[RES]]
-; EG: LSHR {{\*?}} [[ADDR]]
define amdgpu_kernel void @sext_in_reg_v4i1_to_v4i32(ptr addrspace(1) %out, <4 x i32> %a, <4 x i32> %b) #0 {
+; SI-LABEL: sext_in_reg_v4i1_to_v4i32:
+; SI: ; %bb.0:
+; SI-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0xd
+; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_add_i32 s2, s8, s12
+; SI-NEXT: s_add_i32 s4, s9, s13
+; SI-NEXT: s_add_i32 s5, s10, s14
+; SI-NEXT: s_add_i32 s6, s11, s15
+; SI-NEXT: s_bfe_i32 s6, s6, 0x10000
+; SI-NEXT: s_bfe_i32 s5, s5, 0x10000
+; SI-NEXT: s_bfe_i32 s4, s4, 0x10000
+; SI-NEXT: s_bfe_i32 s7, s2, 0x10000
+; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: v_mov_b32_e32 v0, s7
+; SI-NEXT: v_mov_b32_e32 v1, s4
+; SI-NEXT: v_mov_b32_e32 v2, s5
+; SI-NEXT: v_mov_b32_e32 v3, s6
+; SI-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0
+; SI-NEXT: s_endpgm
+;
+; EG-LABEL: sext_in_reg_v4i1_to_v4i32:
+; EG: ; %bb.0:
+; EG-NEXT: ALU 9, @4, KC0[CB0:0-32], KC1[]
+; EG-NEXT: MEM_RAT_CACHELESS STORE_RAW T0.XYZW, T1.X, 1
+; EG-NEXT: CF_END
+; EG-NEXT: PAD
+; EG-NEXT: ALU clause starting at 4:
+; EG-NEXT: ADD_INT * T0.W, KC0[4].X, KC0[5].X,
+; EG-NEXT: BFE_INT T0.W, PV.W, 0.0, 1,
+; EG-NEXT: ADD_INT * T1.W, KC0[3].W, KC0[4].W,
+; EG-NEXT: BFE_INT T0.Z, PS, 0.0, 1,
+; EG-NEXT: ADD_INT * T1.W, KC0[3].Z, KC0[4].Z,
+; EG-NEXT: BFE_INT T0.Y, PV.W, 0.0, 1,
+; EG-NEXT: ADD_INT * T1.W, KC0[3].Y, KC0[4].Y,
+; EG-NEXT: BFE_INT T0.X, PV.W, 0.0, 1,
+; EG-NEXT: LSHR * T1.X, KC0[2].Y, literal.x,
+; EG-NEXT: 2(2.802597e-45), 0(0.000000e+00)
+;
+; GFX11-LABEL: sext_in_reg_v4i1_to_v4i32:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_clause 0x1
+; GFX11-NEXT: s_load_b256 s[8:15], s[4:5], 0x34
+; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: s_add_i32 s2, s8, s12
+; GFX11-NEXT: s_add_i32 s3, s9, s13
+; GFX11-NEXT: s_add_i32 s4, s11, s15
+; GFX11-NEXT: s_add_i32 s5, s10, s14
+; GFX11-NEXT: s_bfe_i32 s4, s4, 0x10000
+; GFX11-NEXT: s_bfe_i32 s5, s5, 0x10000
+; GFX11-NEXT: s_bfe_i32 s2, s2, 0x10000
+; GFX11-NEXT: s_bfe_i32 s3, s3, 0x10000
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX11-NEXT: v_dual_mov_b32 v2, s5 :: v_dual_mov_b32 v3, s4
+; GFX11-NEXT: s_mov_b32 s3, 0x31016000
+; GFX11-NEXT: s_mov_b32 s2, -1
+; GFX11-NEXT: buffer_store_b128 v[0:3], off, s[0:3], 0
+; GFX11-NEXT: s_endpgm
%c = add <4 x i32> %a, %b ; add to prevent folding into extload
%shl = shl <4 x i32> %c, <i32 31, i32 31, i32 31, i32 31>
%ashr = ashr <4 x i32> %shl, <i32 31, i32 31, i32 31, i32 31>
@@ -334,16 +971,55 @@ define amdgpu_kernel void @sext_in_reg_v4i1_to_v4i32(ptr addrspace(1) %out, <4 x
ret void
}
-; FUNC-LABEL: {{^}}sext_in_reg_v2i8_to_v2i32:
-; GCN: s_sext_i32_i8 {{s[0-9]+}}, {{s[0-9]+}}
-; GCN: s_sext_i32_i8 {{s[0-9]+}}, {{s[0-9]+}}
-; GCN: buffer_store_dwordx2
-; EG: MEM_{{.*}} STORE_{{.*}} [[RES:T[0-9]+]]{{\.[XYZW][XYZW]}}, [[ADDR:T[0-9]+.[XYZW]]]
-; EG: BFE_INT [[RES]]
-; EG: BFE_INT [[RES]]
-; EG: LSHR {{\*?}} [[ADDR]]
define amdgpu_kernel void @sext_in_reg_v2i8_to_v2i32(ptr addrspace(1) %out, <2 x i32> %a, <2 x i32> %b) #0 {
+; SI-LABEL: sext_in_reg_v2i8_to_v2i32:
+; SI: ; %bb.0:
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0xb
+; SI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_add_i32 s0, s0, s2
+; SI-NEXT: s_add_i32 s1, s1, s3
+; SI-NEXT: s_sext_i32_i8 s1, s1
+; SI-NEXT: s_sext_i32_i8 s0, s0
+; SI-NEXT: s_mov_b32 s6, -1
+; SI-NEXT: v_mov_b32_e32 v0, s0
+; SI-NEXT: v_mov_b32_e32 v1, s1
+; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; SI-NEXT: s_endpgm
+;
+; EG-LABEL: sext_in_reg_v2i8_to_v2i32:
+; EG: ; %bb.0:
+; EG-NEXT: ALU 6, @4, KC0[CB0:0-32], KC1[]
+; EG-NEXT: MEM_RAT_CACHELESS STORE_RAW T0.XY, T1.X, 1
+; EG-NEXT: CF_END
+; EG-NEXT: PAD
+; EG-NEXT: ALU clause starting at 4:
+; EG-NEXT: ADD_INT * T0.W, KC0[3].X, KC0[3].Z,
+; EG-NEXT: BFE_INT T0.Y, PV.W, 0.0, literal.x,
+; EG-NEXT: ADD_INT * T0.W, KC0[2].W, KC0[3].Y,
+; EG-NEXT: 8(1.121039e-44), 0(0.000000e+00)
+; EG-NEXT: BFE_INT T0.X, PV.W, 0.0, literal.x,
+; EG-NEXT: LSHR * T1.X, KC0[2].Y, literal.y,
+; EG-NEXT: 8(1.121039e-44), 2(2.802597e-45)
+;
+; GFX11-LABEL: sext_in_reg_v2i8_to_v2i32:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_clause 0x1
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x2c
+; GFX11-NEXT: s_load_b64 s[4:5], s[4:5], 0x24
+; GFX11-NEXT: s_mov_b32 s7, 0x31016000
+; GFX11-NEXT: s_mov_b32 s6, -1
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: s_add_i32 s0, s0, s2
+; GFX11-NEXT: s_add_i32 s1, s1, s3
+; GFX11-NEXT: s_sext_i32_i8 s0, s0
+; GFX11-NEXT: s_sext_i32_i8 s1, s1
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT: buffer_store_b64 v[0:1], off, s[4:7], 0
+; GFX11-NEXT: s_endpgm
%c = add <2 x i32> %a, %b ; add to prevent folding into extload
%shl = shl <2 x i32> %c, <i32 24, i32 24>
%ashr = ashr <2 x i32> %shl, <i32 24, i32 24>
@@ -351,20 +1027,72 @@ define amdgpu_kernel void @sext_in_reg_v2i8_to_v2i32(ptr addrspace(1) %out, <2 x
ret void
}
-; FUNC-LABEL: {{^}}sext_in_reg_v4i8_to_v4i32:
-; GCN: s_sext_i32_i8 {{s[0-9]+}}, {{s[0-9]+}}
-; GCN: s_sext_i32_i8 {{s[0-9]+}}, {{s[0-9]+}}
-; GCN: s_sext_i32_i8 {{s[0-9]+}}, {{s[0-9]+}}
-; GCN: s_sext_i32_i8 {{s[0-9]+}}, {{s[0-9]+}}
-; GCN: buffer_store_dwordx4
-; EG: MEM_{{.*}} STORE_{{.*}} [[RES:T[0-9]+]]{{\.[XYZW][XYZW][XYZW][XYZW]}}, [[ADDR:T[0-9]+.[XYZW]]]
-; EG: BFE_INT [[RES]]
-; EG: BFE_INT [[RES]]
-; EG: BFE_INT [[RES]]
-; EG: BFE_INT [[RES]]
-; EG: LSHR {{\*?}} [[ADDR]]
define amdgpu_kernel void @sext_in_reg_v4i8_to_v4i32(ptr addrspace(1) %out, <4 x i32> %a, <4 x i32> %b) #0 {
+; SI-LABEL: sext_in_reg_v4i8_to_v4i32:
+; SI: ; %bb.0:
+; SI-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0xd
+; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_add_i32 s2, s8, s12
+; SI-NEXT: s_add_i32 s4, s9, s13
+; SI-NEXT: s_add_i32 s5, s10, s14
+; SI-NEXT: s_add_i32 s6, s11, s15
+; SI-NEXT: s_sext_i32_i8 s6, s6
+; SI-NEXT: s_sext_i32_i8 s5, s5
+; SI-NEXT: s_sext_i32_i8 s4, s4
+; SI-NEXT: s_sext_i32_i8 s7, s2
+; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: v_mov_b32_e32 v0, s7
+; SI-NEXT: v_mov_b32_e32 v1, s4
+; SI-NEXT: v_mov_b32_e32 v2, s5
+; SI-NEXT: v_mov_b32_e32 v3, s6
+; SI-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0
+; SI-NEXT: s_endpgm
+;
+; EG-LABEL: sext_in_reg_v4i8_to_v4i32:
+; EG: ; %bb.0:
+; EG-NEXT: ALU 12, @4, KC0[CB0:0-32], KC1[]
+; EG-NEXT: MEM_RAT_CACHELESS STORE_RAW T0.XYZW, T1.X, 1
+; EG-NEXT: CF_END
+; EG-NEXT: PAD
+; EG-NEXT: ALU clause starting at 4:
+; EG-NEXT: ADD_INT * T0.W, KC0[4].X, KC0[5].X,
+; EG-NEXT: BFE_INT T0.W, PV.W, 0.0, literal.x,
+; EG-NEXT: ADD_INT * T1.W, KC0[3].W, KC0[4].W,
+; EG-NEXT: 8(1.121039e-44), 0(0.000000e+00)
+; EG-NEXT: BFE_INT T0.Z, PS, 0.0, literal.x,
+; EG-NEXT: ADD_INT * T1.W, KC0[3].Z, KC0[4].Z,
+; EG-NEXT: 8(1.121039e-44), 0(0.000000e+00)
+; EG-NEXT: BFE_INT T0.Y, PV.W, 0.0, literal.x,
+; EG-NEXT: ADD_INT * T1.W, KC0[3].Y, KC0[4].Y,
+; EG-NEXT: 8(1.121039e-44), 0(0.000000e+00)
+; EG-NEXT: BFE_INT T0.X, PV.W, 0.0, literal.x,
+; EG-NEXT: LSHR * T1.X, KC0[2].Y, literal.y,
+; EG-NEXT: 8(1.121039e-44), 2(2.802597e-45)
+;
+; GFX11-LABEL: sext_in_reg_v4i8_to_v4i32:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_clause 0x1
+; GFX11-NEXT: s_load_b256 s[8:15], s[4:5], 0x34
+; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: s_add_i32 s2, s8, s12
+; GFX11-NEXT: s_add_i32 s3, s9, s13
+; GFX11-NEXT: s_add_i32 s4, s11, s15
+; GFX11-NEXT: s_add_i32 s5, s10, s14
+; GFX11-NEXT: s_sext_i32_i8 s4, s4
+; GFX11-NEXT: s_sext_i32_i8 s5, s5
+; GFX11-NEXT: s_sext_i32_i8 s2, s2
+; GFX11-NEXT: s_sext_i32_i8 s3, s3
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX11-NEXT: v_dual_mov_b32 v2, s5 :: v_dual_mov_b32 v3, s4
+; GFX11-NEXT: s_mov_b32 s3, 0x31016000
+; GFX11-NEXT: s_mov_b32 s2, -1
+; GFX11-NEXT: buffer_store_b128 v[0:3], off, s[0:3], 0
+; GFX11-NEXT: s_endpgm
%c = add <4 x i32> %a, %b ; add to prevent folding into extload
%shl = shl <4 x i32> %c, <i32 24, i32 24, i32 24, i32 24>
%ashr = ashr <4 x i32> %shl, <i32 24, i32 24, i32 24, i32 24>
@@ -372,16 +1100,55 @@ define amdgpu_kernel void @sext_in_reg_v4i8_to_v4i32(ptr addrspace(1) %out, <4 x
ret void
}
-; FUNC-LABEL: {{^}}sext_in_reg_v2i16_to_v2i32:
-; GCN: s_sext_i32_i16 {{s[0-9]+}}, {{s[0-9]+}}
-; GCN: s_sext_i32_i16 {{s[0-9]+}}, {{s[0-9]+}}
-; GCN: buffer_store_dwordx2
-; EG: MEM_{{.*}} STORE_{{.*}} [[RES:T[0-9]+]]{{\.[XYZW][XYZW]}}, [[ADDR:T[0-9]+.[XYZW]]]
-; EG: BFE_INT [[RES]]
-; EG: BFE_INT [[RES]]
-; EG: LSHR {{\*?}} [[ADDR]]
define amdgpu_kernel void @sext_in_reg_v2i16_to_v2i32(ptr addrspace(1) %out, <2 x i32> %a, <2 x i32> %b) #0 {
+; SI-LABEL: sext_in_reg_v2i16_to_v2i32:
+; SI: ; %bb.0:
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0xb
+; SI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_add_i32 s0, s0, s2
+; SI-NEXT: s_add_i32 s1, s1, s3
+; SI-NEXT: s_sext_i32_i16 s1, s1
+; SI-NEXT: s_sext_i32_i16 s0, s0
+; SI-NEXT: s_mov_b32 s6, -1
+; SI-NEXT: v_mov_b32_e32 v0, s0
+; SI-NEXT: v_mov_b32_e32 v1, s1
+; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; SI-NEXT: s_endpgm
+;
+; EG-LABEL: sext_in_reg_v2i16_to_v2i32:
+; EG: ; %bb.0:
+; EG-NEXT: ALU 6, @4, KC0[CB0:0-32], KC1[]
+; EG-NEXT: MEM_RAT_CACHELESS STORE_RAW T0.XY, T1.X, 1
+; EG-NEXT: CF_END
+; EG-NEXT: PAD
+; EG-NEXT: ALU clause starting at 4:
+; EG-NEXT: ADD_INT * T0.W, KC0[3].X, KC0[3].Z,
+; EG-NEXT: BFE_INT T0.Y, PV.W, 0.0, literal.x,
+; EG-NEXT: ADD_INT * T0.W, KC0[2].W, KC0[3].Y,
+; EG-NEXT: 16(2.242078e-44), 0(0.000000e+00)
+; EG-NEXT: BFE_INT T0.X, PV.W, 0.0, literal.x,
+; EG-NEXT: LSHR * T1.X, KC0[2].Y, literal.y,
+; EG-NEXT: 16(2.242078e-44), 2(2.802597e-45)
+;
+; GFX11-LABEL: sext_in_reg_v2i16_to_v2i32:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_clause 0x1
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x2c
+; GFX11-NEXT: s_load_b64 s[4:5], s[4:5], 0x24
+; GFX11-NEXT: s_mov_b32 s7, 0x31016000
+; GFX11-NEXT: s_mov_b32 s6, -1
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: s_add_i32 s0, s0, s2
+; GFX11-NEXT: s_add_i32 s1, s1, s3
+; GFX11-NEXT: s_sext_i32_i16 s0, s0
+; GFX11-NEXT: s_sext_i32_i16 s1, s1
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT: buffer_store_b64 v[0:1], off, s[4:7], 0
+; GFX11-NEXT: s_endpgm
%c = add <2 x i32> %a, %b ; add to prevent folding into extload
%shl = shl <2 x i32> %c, <i32 16, i32 16>
%ashr = ashr <2 x i32> %shl, <i32 16, i32 16>
@@ -389,8 +1156,91 @@ define amdgpu_kernel void @sext_in_reg_v2i16_to_v2i32(ptr addrspace(1) %out, <2
ret void
}
-; FUNC-LABEL: {{^}}testcase:
define amdgpu_kernel void @testcase(ptr addrspace(1) %out, i8 %a) #0 {
+; SI-LABEL: testcase:
+; SI: ; %bb.0:
+; SI-NEXT: s_load_dword s2, s[4:5], 0xb
+; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_sext_i32_i8 s4, s2
+; SI-NEXT: s_bfe_i32 s5, s2, 0x10000
+; SI-NEXT: s_max_i32 s4, s4, 0
+; SI-NEXT: s_and_b32 s2, s5, s2
+; SI-NEXT: s_xor_b32 s4, s4, s2
+; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: v_mov_b32_e32 v0, s4
+; SI-NEXT: buffer_store_byte v0, off, s[0:3], 0
+; SI-NEXT: s_endpgm
+;
+; EG-LABEL: testcase:
+; EG: ; %bb.0:
+; EG-NEXT: ALU 0, @8, KC0[], KC1[]
+; EG-NEXT: TEX 0 @6
+; EG-NEXT: ALU 17, @9, KC0[CB0:0-32], KC1[]
+; EG-NEXT: MEM_RAT MSKOR T0.XW, T1.X
+; EG-NEXT: CF_END
+; EG-NEXT: PAD
+; EG-NEXT: Fetch clause starting at 6:
+; EG-NEXT: VTX_READ_8 T0.X, T0.X, 40, #3
+; EG-NEXT: ALU clause starting at 8:
+; EG-NEXT: MOV * T0.X, 0.0,
+; EG-NEXT: ALU clause starting at 9:
+; EG-NEXT: BFE_INT T0.Z, T0.X, 0.0, literal.x,
+; EG-NEXT: BFE_INT * T0.W, T0.X, 0.0, 1,
+; EG-NEXT: 8(1.121039e-44), 0(0.000000e+00)
+; EG-NEXT: AND_INT T0.W, PV.W, T0.X,
+; EG-NEXT: MAX_INT * T1.W, PV.Z, 0.0,
+; EG-NEXT: AND_INT T2.W, KC0[2].Y, literal.x,
+; EG-NEXT: XOR_INT * T0.W, PS, PV.W,
+; EG-NEXT: 3(4.203895e-45), 0(0.000000e+00)
+; EG-NEXT: AND_INT T0.W, PS, literal.x,
+; EG-NEXT: LSHL * T1.W, PV.W, literal.y,
+; EG-NEXT: 255(3.573311e-43), 3(4.203895e-45)
+; EG-NEXT: LSHL T0.X, PV.W, PS,
+; EG-NEXT: LSHL * T0.W, literal.x, PS,
+; EG-NEXT: 255(3.573311e-43), 0(0.000000e+00)
+; EG-NEXT: MOV T0.Y, 0.0,
+; EG-NEXT: MOV * T0.Z, 0.0,
+; EG-NEXT: LSHR * T1.X, KC0[2].Y, literal.x,
+; EG-NEXT: 2(2.802597e-45), 0(0.000000e+00)
+;
+; GFX11-TRUE16-LABEL: testcase:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_clause 0x1
+; GFX11-TRUE16-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GFX11-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT: s_mov_b32 s3, s2
+; GFX11-TRUE16-NEXT: s_sext_i32_i8 s4, s2
+; GFX11-TRUE16-NEXT: s_bfe_i32 s3, s3, 0x10000
+; GFX11-TRUE16-NEXT: s_max_i32 s4, s4, 0
+; GFX11-TRUE16-NEXT: s_and_b32 s2, s3, s2
+; GFX11-TRUE16-NEXT: s_mov_b32 s3, 0x31016000
+; GFX11-TRUE16-NEXT: s_xor_b32 s2, s4, s2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, s2
+; GFX11-TRUE16-NEXT: s_mov_b32 s2, -1
+; GFX11-TRUE16-NEXT: buffer_store_b8 v0, off, s[0:3], 0
+; GFX11-TRUE16-NEXT: s_endpgm
+;
+; GFX11-FAKE16-LABEL: testcase:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_clause 0x1
+; GFX11-FAKE16-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GFX11-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT: s_sext_i32_i8 s3, s2
+; GFX11-FAKE16-NEXT: s_bfe_i32 s4, s2, 0x10000
+; GFX11-FAKE16-NEXT: s_max_i32 s3, s3, 0
+; GFX11-FAKE16-NEXT: s_and_b32 s2, s4, s2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_xor_b32 s2, s3, s2
+; GFX11-FAKE16-NEXT: s_mov_b32 s3, 0x31016000
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, s2
+; GFX11-FAKE16-NEXT: s_mov_b32 s2, -1
+; GFX11-FAKE16-NEXT: buffer_store_b8 v0, off, s[0:3], 0
+; GFX11-FAKE16-NEXT: s_endpgm
%and_a_1 = and i8 %a, 1
%cmp_eq = icmp eq i8 %and_a_1, 0
%cmp_slt = icmp slt i8 %a, 0
@@ -401,8 +1251,91 @@ define amdgpu_kernel void @testcase(ptr addrspace(1) %out, i8 %a) #0 {
ret void
}
-; FUNC-LABEL: {{^}}testcase_3:
define amdgpu_kernel void @testcase_3(ptr addrspace(1) %out, i8 %a) #0 {
+; SI-LABEL: testcase_3:
+; SI: ; %bb.0:
+; SI-NEXT: s_load_dword s2, s[4:5], 0xb
+; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_sext_i32_i8 s4, s2
+; SI-NEXT: s_bfe_i32 s5, s2, 0x10000
+; SI-NEXT: s_max_i32 s4, s4, 0
+; SI-NEXT: s_and_b32 s2, s5, s2
+; SI-NEXT: s_xor_b32 s4, s4, s2
+; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: v_mov_b32_e32 v0, s4
+; SI-NEXT: buffer_store_byte v0, off, s[0:3], 0
+; SI-NEXT: s_endpgm
+;
+; EG-LABEL: testcase_3:
+; EG: ; %bb.0:
+; EG-NEXT: ALU 0, @8, KC0[], KC1[]
+; EG-NEXT: TEX 0 @6
+; EG-NEXT: ALU 17, @9, KC0[CB0:0-32], KC1[]
+; EG-NEXT: MEM_RAT MSKOR T0.XW, T1.X
+; EG-NEXT: CF_END
+; EG-NEXT: PAD
+; EG-NEXT: Fetch clause starting at 6:
+; EG-NEXT: VTX_READ_8 T0.X, T0.X, 40, #3
+; EG-NEXT: ALU clause starting at 8:
+; EG-NEXT: MOV * T0.X, 0.0,
+; EG-NEXT: ALU clause starting at 9:
+; EG-NEXT: BFE_INT T0.Z, T0.X, 0.0, literal.x,
+; EG-NEXT: BFE_INT * T0.W, T0.X, 0.0, 1,
+; EG-NEXT: 8(1.121039e-44), 0(0.000000e+00)
+; EG-NEXT: AND_INT T0.W, PV.W, T0.X,
+; EG-NEXT: MAX_INT * T1.W, PV.Z, 0.0,
+; EG-NEXT: AND_INT T2.W, KC0[2].Y, literal.x,
+; EG-NEXT: XOR_INT * T0.W, PS, PV.W,
+; EG-NEXT: 3(4.203895e-45), 0(0.000000e+00)
+; EG-NEXT: AND_INT T0.W, PS, literal.x,
+; EG-NEXT: LSHL * T1.W, PV.W, literal.y,
+; EG-NEXT: 255(3.573311e-43), 3(4.203895e-45)
+; EG-NEXT: LSHL T0.X, PV.W, PS,
+; EG-NEXT: LSHL * T0.W, literal.x, PS,
+; EG-NEXT: 255(3.573311e-43), 0(0.000000e+00)
+; EG-NEXT: MOV T0.Y, 0.0,
+; EG-NEXT: MOV * T0.Z, 0.0,
+; EG-NEXT: LSHR * T1.X, KC0[2].Y, literal.x,
+; EG-NEXT: 2(2.802597e-45), 0(0.000000e+00)
+;
+; GFX11-TRUE16-LABEL: testcase_3:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_clause 0x1
+; GFX11-TRUE16-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GFX11-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT: s_mov_b32 s3, s2
+; GFX11-TRUE16-NEXT: s_sext_i32_i8 s4, s2
+; GFX11-TRUE16-NEXT: s_bfe_i32 s3, s3, 0x10000
+; GFX11-TRUE16-NEXT: s_max_i32 s4, s4, 0
+; GFX11-TRUE16-NEXT: s_and_b32 s2, s3, s2
+; GFX11-TRUE16-NEXT: s_mov_b32 s3, 0x31016000
+; GFX11-TRUE16-NEXT: s_xor_b32 s2, s4, s2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, s2
+; GFX11-TRUE16-NEXT: s_mov_b32 s2, -1
+; GFX11-TRUE16-NEXT: buffer_store_b8 v0, off, s[0:3], 0
+; GFX11-TRUE16-NEXT: s_endpgm
+;
+; GFX11-FAKE16-LABEL: testcase_3:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_clause 0x1
+; GFX11-FAKE16-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GFX11-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT: s_sext_i32_i8 s3, s2
+; GFX11-FAKE16-NEXT: s_bfe_i32 s4, s2, 0x10000
+; GFX11-FAKE16-NEXT: s_max_i32 s3, s3, 0
+; GFX11-FAKE16-NEXT: s_and_b32 s2, s4, s2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_xor_b32 s2, s3, s2
+; GFX11-FAKE16-NEXT: s_mov_b32 s3, 0x31016000
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, s2
+; GFX11-FAKE16-NEXT: s_mov_b32 s2, -1
+; GFX11-FAKE16-NEXT: buffer_store_b8 v0, off, s[0:3], 0
+; GFX11-FAKE16-NEXT: s_endpgm
%and_a_1 = and i8 %a, 1
%cmp_eq = icmp eq i8 %and_a_1, 0
%cmp_slt = icmp slt i8 %a, 0
@@ -413,12 +1346,96 @@ define amdgpu_kernel void @testcase_3(ptr addrspace(1) %out, i8 %a) #0 {
ret void
}
-; FUNC-LABEL: {{^}}vgpr_sext_in_reg_v4i8_to_v4i32:
-; GCN: v_bfe_i32 [[EXTRACT:v[0-9]+]], {{v[0-9]+}}, 0, 8
-; GCN: v_bfe_i32 [[EXTRACT:v[0-9]+]], {{v[0-9]+}}, 0, 8
-; GCN: v_bfe_i32 [[EXTRACT:v[0-9]+]], {{v[0-9]+}}, 0, 8
-; GCN: v_bfe_i32 [[EXTRACT:v[0-9]+]], {{v[0-9]+}}, 0, 8
define amdgpu_kernel void @vgpr_sext_in_reg_v4i8_to_v4i32(ptr addrspace(1) %out, ptr addrspace(1) %a, ptr addrspace(1) %b) #0 {
+; SI-LABEL: vgpr_sext_in_reg_v4i8_to_v4i32:
+; SI: ; %bb.0:
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd
+; SI-NEXT: s_mov_b32 s11, 0xf000
+; SI-NEXT: s_mov_b32 s10, -1
+; SI-NEXT: s_mov_b32 s14, s10
+; SI-NEXT: s_mov_b32 s15, s11
+; SI-NEXT: s_mov_b32 s6, s10
+; SI-NEXT: s_mov_b32 s7, s11
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s12, s2
+; SI-NEXT: s_mov_b32 s13, s3
+; SI-NEXT: buffer_load_dwordx4 v[0:3], off, s[12:15], 0
+; SI-NEXT: buffer_load_dwordx4 v[4:7], off, s[4:7], 0
+; SI-NEXT: s_mov_b32 s8, s0
+; SI-NEXT: s_mov_b32 s9, s1
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_add_i32_e32 v0, vcc, v0, v4
+; SI-NEXT: v_add_i32_e32 v1, vcc, v1, v5
+; SI-NEXT: v_add_i32_e32 v2, vcc, v2, v6
+; SI-NEXT: v_add_i32_e32 v3, vcc, v3, v7
+; SI-NEXT: v_bfe_i32 v3, v3, 0, 8
+; SI-NEXT: v_bfe_i32 v2, v2, 0, 8
+; SI-NEXT: v_bfe_i32 v1, v1, 0, 8
+; SI-NEXT: v_bfe_i32 v0, v0, 0, 8
+; SI-NEXT: buffer_store_dwordx4 v[0:3], off, s[8:11], 0
+; SI-NEXT: s_endpgm
+;
+; EG-LABEL: vgpr_sext_in_reg_v4i8_to_v4i32:
+; EG: ; %bb.0:
+; EG-NEXT: ALU 1, @10, KC0[CB0:0-32], KC1[]
+; EG-NEXT: TEX 1 @6
+; EG-NEXT: ALU 12, @12, KC0[CB0:0-32], KC1[]
+; EG-NEXT: MEM_RAT_CACHELESS STORE_RAW T0.XYZW, T1.X, 1
+; EG-NEXT: CF_END
+; EG-NEXT: PAD
+; EG-NEXT: Fetch clause starting at 6:
+; EG-NEXT: VTX_READ_128 T1.XYZW, T1.X, 0, #1
+; EG-NEXT: VTX_READ_128 T0.XYZW, T0.X, 0, #1
+; EG-NEXT: ALU clause starting at 10:
+; EG-NEXT: MOV T0.X, KC0[2].Z,
+; EG-NEXT: MOV * T1.X, KC0[2].W,
+; EG-NEXT: ALU clause starting at 12:
+; EG-NEXT: ADD_INT * T0.W, T0.W, T1.W,
+; EG-NEXT: BFE_INT T0.W, PV.W, 0.0, literal.x,
+; EG-NEXT: ADD_INT * T1.W, T0.Z, T1.Z,
+; EG-NEXT: 8(1.121039e-44), 0(0.000000e+00)
+; EG-NEXT: BFE_INT T0.Z, PS, 0.0, literal.x,
+; EG-NEXT: ADD_INT * T1.W, T0.Y, T1.Y,
+; EG-NEXT: 8(1.121039e-44), 0(0.000000e+00)
+; EG-NEXT: BFE_INT T0.Y, PV.W, 0.0, literal.x,
+; EG-NEXT: ADD_INT * T1.W, T0.X, T1.X,
+; EG-NEXT: 8(1.121039e-44), 0(0.000000e+00)
+; EG-NEXT: BFE_INT T0.X, PV.W, 0.0, literal.x,
+; EG-NEXT: LSHR * T1.X, KC0[2].Y, literal.y,
+; EG-NEXT: 8(1.121039e-44), 2(2.802597e-45)
+;
+; GFX11-LABEL: vgpr_sext_in_reg_v4i8_to_v4i32:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_clause 0x1
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT: s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-NEXT: s_mov_b32 s10, -1
+; GFX11-NEXT: s_mov_b32 s11, 0x31016000
+; GFX11-NEXT: s_mov_b32 s6, s10
+; GFX11-NEXT: s_mov_b32 s7, s11
+; GFX11-NEXT: s_mov_b32 s14, s10
+; GFX11-NEXT: s_mov_b32 s15, s11
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: s_mov_b32 s12, s2
+; GFX11-NEXT: s_mov_b32 s13, s3
+; GFX11-NEXT: buffer_load_b128 v[0:3], off, s[4:7], 0
+; GFX11-NEXT: buffer_load_b128 v[4:7], off, s[12:15], 0
+; GFX11-NEXT: s_mov_b32 s8, s0
+; GFX11-NEXT: s_mov_b32 s9, s1
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_add_nc_u32_e32 v3, v7, v3
+; GFX11-NEXT: v_add_nc_u32_e32 v2, v6, v2
+; GFX11-NEXT: v_add_nc_u32_e32 v1, v5, v1
+; GFX11-NEXT: v_add_nc_u32_e32 v0, v4, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_bfe_i32 v3, v3, 0, 8
+; GFX11-NEXT: v_bfe_i32 v2, v2, 0, 8
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_bfe_i32 v1, v1, 0, 8
+; GFX11-NEXT: v_bfe_i32 v0, v0, 0, 8
+; GFX11-NEXT: buffer_store_b128 v[0:3], off, s[8:11], 0
+; GFX11-NEXT: s_endpgm
%loada = load <4 x i32>, ptr addrspace(1) %a, align 16
%loadb = load <4 x i32>, ptr addrspace(1) %b, align 16
%c = add <4 x i32> %loada, %loadb ; add to prevent folding into extload
@@ -428,10 +1445,96 @@ define amdgpu_kernel void @vgpr_sext_in_reg_v4i8_to_v4i32(ptr addrspace(1) %out,
ret void
}
-; FUNC-LABEL: {{^}}vgpr_sext_in_reg_v4i16_to_v4i32:
-; GCN: v_bfe_i32 [[EXTRACT:v[0-9]+]], {{v[0-9]+}}, 0, 16
-; GCN: v_bfe_i32 [[EXTRACT:v[0-9]+]], {{v[0-9]+}}, 0, 16
define amdgpu_kernel void @vgpr_sext_in_reg_v4i16_to_v4i32(ptr addrspace(1) %out, ptr addrspace(1) %a, ptr addrspace(1) %b) #0 {
+; SI-LABEL: vgpr_sext_in_reg_v4i16_to_v4i32:
+; SI: ; %bb.0:
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd
+; SI-NEXT: s_mov_b32 s11, 0xf000
+; SI-NEXT: s_mov_b32 s10, -1
+; SI-NEXT: s_mov_b32 s14, s10
+; SI-NEXT: s_mov_b32 s15, s11
+; SI-NEXT: s_mov_b32 s6, s10
+; SI-NEXT: s_mov_b32 s7, s11
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s12, s2
+; SI-NEXT: s_mov_b32 s13, s3
+; SI-NEXT: buffer_load_dwordx4 v[0:3], off, s[12:15], 0
+; SI-NEXT: buffer_load_dwordx4 v[4:7], off, s[4:7], 0
+; SI-NEXT: s_mov_b32 s8, s0
+; SI-NEXT: s_mov_b32 s9, s1
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_add_i32_e32 v0, vcc, v0, v4
+; SI-NEXT: v_add_i32_e32 v1, vcc, v1, v5
+; SI-NEXT: v_add_i32_e32 v2, vcc, v2, v6
+; SI-NEXT: v_add_i32_e32 v3, vcc, v3, v7
+; SI-NEXT: v_bfe_i32 v3, v3, 0, 16
+; SI-NEXT: v_bfe_i32 v2, v2, 0, 16
+; SI-NEXT: v_bfe_i32 v1, v1, 0, 16
+; SI-NEXT: v_bfe_i32 v0, v0, 0, 16
+; SI-NEXT: buffer_store_dwordx4 v[0:3], off, s[8:11], 0
+; SI-NEXT: s_endpgm
+;
+; EG-LABEL: vgpr_sext_in_reg_v4i16_to_v4i32:
+; EG: ; %bb.0:
+; EG-NEXT: ALU 1, @10, KC0[CB0:0-32], KC1[]
+; EG-NEXT: TEX 1 @6
+; EG-NEXT: ALU 12, @12, KC0[CB0:0-32], KC1[]
+; EG-NEXT: MEM_RAT_CACHELESS STORE_RAW T0.XYZW, T1.X, 1
+; EG-NEXT: CF_END
+; EG-NEXT: PAD
+; EG-NEXT: Fetch clause starting at 6:
+; EG-NEXT: VTX_READ_128 T1.XYZW, T1.X, 0, #1
+; EG-NEXT: VTX_READ_128 T0.XYZW, T0.X, 0, #1
+; EG-NEXT: ALU clause starting at 10:
+; EG-NEXT: MOV T0.X, KC0[2].Z,
+; EG-NEXT: MOV * T1.X, KC0[2].W,
+; EG-NEXT: ALU clause starting at 12:
+; EG-NEXT: ADD_INT * T0.W, T0.W, T1.W,
+; EG-NEXT: BFE_INT T0.W, PV.W, 0.0, literal.x,
+; EG-NEXT: ADD_INT * T1.W, T0.Z, T1.Z,
+; EG-NEXT: 16(2.242078e-44), 0(0.000000e+00)
+; EG-NEXT: BFE_INT T0.Z, PS, 0.0, literal.x,
+; EG-NEXT: ADD_INT * T1.W, T0.Y, T1.Y,
+; EG-NEXT: 16(2.242078e-44), 0(0.000000e+00)
+; EG-NEXT: BFE_INT T0.Y, PV.W, 0.0, literal.x,
+; EG-NEXT: ADD_INT * T1.W, T0.X, T1.X,
+; EG-NEXT: 16(2.242078e-44), 0(0.000000e+00)
+; EG-NEXT: BFE_INT T0.X, PV.W, 0.0, literal.x,
+; EG-NEXT: LSHR * T1.X, KC0[2].Y, literal.y,
+; EG-NEXT: 16(2.242078e-44), 2(2.802597e-45)
+;
+; GFX11-LABEL: vgpr_sext_in_reg_v4i16_to_v4i32:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_clause 0x1
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT: s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-NEXT: s_mov_b32 s10, -1
+; GFX11-NEXT: s_mov_b32 s11, 0x31016000
+; GFX11-NEXT: s_mov_b32 s6, s10
+; GFX11-NEXT: s_mov_b32 s7, s11
+; GFX11-NEXT: s_mov_b32 s14, s10
+; GFX11-NEXT: s_mov_b32 s15, s11
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: s_mov_b32 s12, s2
+; GFX11-NEXT: s_mov_b32 s13, s3
+; GFX11-NEXT: buffer_load_b128 v[0:3], off, s[4:7], 0
+; GFX11-NEXT: buffer_load_b128 v[4:7], off, s[12:15], 0
+; GFX11-NEXT: s_mov_b32 s8, s0
+; GFX11-NEXT: s_mov_b32 s9, s1
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_add_nc_u32_e32 v3, v7, v3
+; GFX11-NEXT: v_add_nc_u32_e32 v2, v6, v2
+; GFX11-NEXT: v_add_nc_u32_e32 v1, v5, v1
+; GFX11-NEXT: v_add_nc_u32_e32 v0, v4, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_bfe_i32 v3, v3, 0, 16
+; GFX11-NEXT: v_bfe_i32 v2, v2, 0, 16
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_bfe_i32 v1, v1, 0, 16
+; GFX11-NEXT: v_bfe_i32 v0, v0, 0, 16
+; GFX11-NEXT: buffer_store_b128 v[0:3], off, s[8:11], 0
+; GFX11-NEXT: s_endpgm
%loada = load <4 x i32>, ptr addrspace(1) %a, align 16
%loadb = load <4 x i32>, ptr addrspace(1) %b, align 16
%c = add <4 x i32> %loada, %loadb ; add to prevent folding into extload
@@ -441,12 +1544,69 @@ define amdgpu_kernel void @vgpr_sext_in_reg_v4i16_to_v4i32(ptr addrspace(1) %out
ret void
}
-; FUNC-LABEL: {{^}}sext_in_reg_to_illegal_type:
-; GCN: buffer_load_sbyte
-; GCN: v_max_i32
-; GCN-NOT: bfe
-; GCN: buffer_store_short
define amdgpu_kernel void @sext_in_reg_to_illegal_type(ptr addrspace(1) nocapture %out, ptr addrspace(1) nocapture %src) #0 {
+; SI-LABEL: sext_in_reg_to_illegal_type:
+; SI: ; %bb.0:
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
+; SI-NEXT: s_mov_b32 s10, s6
+; SI-NEXT: s_mov_b32 s11, s7
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s8, s2
+; SI-NEXT: s_mov_b32 s9, s3
+; SI-NEXT: buffer_load_sbyte v0, off, s[8:11], 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_max_i32_e32 v0, 0, v0
+; SI-NEXT: buffer_store_short v0, off, s[4:7], 0
+; SI-NEXT: s_endpgm
+;
+; EG-LABEL: sext_in_reg_to_illegal_type:
+; EG: ; %bb.0:
+; EG-NEXT: ALU 0, @8, KC0[CB0:0-32], KC1[]
+; EG-NEXT: TEX 0 @6
+; EG-NEXT: ALU 12, @9, KC0[CB0:0-32], KC1[]
+; EG-NEXT: MEM_RAT MSKOR T0.XW, T1.X
+; EG-NEXT: CF_END
+; EG-NEXT: PAD
+; EG-NEXT: Fetch clause starting at 6:
+; EG-NEXT: VTX_READ_8 T0.X, T0.X, 0, #1
+; EG-NEXT: ALU clause starting at 8:
+; EG-NEXT: MOV * T0.X, KC0[2].Z,
+; EG-NEXT: ALU clause starting at 9:
+; EG-NEXT: BFE_INT T0.W, T0.X, 0.0, literal.x,
+; EG-NEXT: AND_INT * T1.W, KC0[2].Y, literal.y,
+; EG-NEXT: 8(1.121039e-44), 3(4.203895e-45)
+; EG-NEXT: MAX_INT T0.W, PV.W, 0.0,
+; EG-NEXT: LSHL * T1.W, PS, literal.x,
+; EG-NEXT: 3(4.203895e-45), 0(0.000000e+00)
+; EG-NEXT: LSHL T0.X, PV.W, PS,
+; EG-NEXT: LSHL * T0.W, literal.x, PS,
+; EG-NEXT: 65535(9.183409e-41), 0(0.000000e+00)
+; EG-NEXT: MOV T0.Y, 0.0,
+; EG-NEXT: MOV * T0.Z, 0.0,
+; EG-NEXT: LSHR * T1.X, KC0[2].Y, literal.x,
+; EG-NEXT: 2(2.802597e-45), 0(0.000000e+00)
+;
+; GFX11-LABEL: sext_in_reg_to_illegal_type:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT: s_mov_b32 s6, -1
+; GFX11-NEXT: s_mov_b32 s7, 0x31016000
+; GFX11-NEXT: s_mov_b32 s10, s6
+; GFX11-NEXT: s_mov_b32 s11, s7
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: s_mov_b32 s8, s2
+; GFX11-NEXT: s_mov_b32 s9, s3
+; GFX11-NEXT: s_mov_b32 s4, s0
+; GFX11-NEXT: buffer_load_i8 v0, off, s[8:11], 0
+; GFX11-NEXT: s_mov_b32 s5, s1
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_max_i32_e32 v0, 0, v0
+; GFX11-NEXT: buffer_store_b16 v0, off, s[4:7], 0
+; GFX11-NEXT: s_endpgm
%tmp5 = load i8, ptr addrspace(1) %src, align 1
%tmp2 = sext i8 %tmp5 to i32
%tmp2.5 = icmp sgt i32 %tmp2, 0
@@ -459,20 +1619,76 @@ define amdgpu_kernel void @sext_in_reg_to_illegal_type(ptr addrspace(1) nocaptur
; Make sure we propagate the VALUness to users of a moved scalar BFE.
-; FUNC-LABEL: {{^}}v_sext_in_reg_i1_to_i64_move_use:
-; SI: buffer_load_dwordx2
-; SI: v_lshl_b64 v[[[VAL_LO:[0-9]+]]:[[VAL_HI:[0-9]+]]]
-; GFX89: {{flat|global}}_load_dwordx2
-; GFX89: v_lshlrev_b64 v[[[VAL_LO:[0-9]+]]:[[VAL_HI:[0-9]+]]]
-; GCN-DAG: v_bfe_i32 v[[LO:[0-9]+]], v[[VAL_LO]], 0, 1
-; GCN-DAG: v_ashrrev_i32_e32 v[[HI:[0-9]+]], 31, v[[LO]]
-; GCN-DAG: v_and_b32_e32 v[[RESULT_LO:[0-9]+]], s{{[0-9]+}}, v[[LO]]
-; GCN-DAG: v_and_b32_e32 v[[RESULT_HI:[0-9]+]], s{{[0-9]+}}, v[[HI]]
-; SI: buffer_store_dwordx2 v[[[RESULT_LO]]:[[RESULT_HI]]]
-; GFX89: {{flat|global}}_store_dwordx2 v{{.+}}, v[[[RESULT_LO]]:[[RESULT_HI]]]
define amdgpu_kernel void @v_sext_in_reg_i1_to_i64_move_use(ptr addrspace(1) %out, ptr addrspace(1) %aptr, ptr addrspace(1) %bptr, i64 %s.val) #0 {
+; SI-LABEL: v_sext_in_reg_i1_to_i64_move_use:
+; SI: ; %bb.0:
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0xf
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, 0
+; SI-NEXT: v_lshlrev_b32_e32 v0, 3, v0
+; SI-NEXT: v_mov_b32_e32 v1, 0
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b64 s[4:5], s[2:3]
+; SI-NEXT: buffer_load_dwordx2 v[2:3], v[0:1], s[4:7], 0 addr64
+; SI-NEXT: s_mov_b64 s[2:3], s[6:7]
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshl_b64 v[2:3], v[2:3], v2
+; SI-NEXT: v_bfe_i32 v2, v2, 0, 1
+; SI-NEXT: v_ashrrev_i32_e32 v3, 31, v2
+; SI-NEXT: v_and_b32_e32 v3, s9, v3
+; SI-NEXT: v_and_b32_e32 v2, s8, v2
+; SI-NEXT: buffer_store_dwordx2 v[2:3], v[0:1], s[0:3], 0 addr64
+; SI-NEXT: s_endpgm
+;
+; EG-LABEL: v_sext_in_reg_i1_to_i64_move_use:
+; EG: ; %bb.0:
+; EG-NEXT: ALU 2, @8, KC0[CB0:0-32], KC1[]
+; EG-NEXT: TEX 0 @6
+; EG-NEXT: ALU 11, @11, KC0[CB0:0-32], KC1[]
+; EG-NEXT: MEM_RAT_CACHELESS STORE_RAW T0.XY, T1.X, 1
+; EG-NEXT: CF_END
+; EG-NEXT: PAD
+; EG-NEXT: Fetch clause starting at 6:
+; EG-NEXT: VTX_READ_32 T0.X, T0.X, 0, #1
+; EG-NEXT: ALU clause starting at 8:
+; EG-NEXT: LSHL * T0.W, T0.X, literal.x,
+; EG-NEXT: 3(4.203895e-45), 0(0.000000e+00)
+; EG-NEXT: ADD_INT * T0.X, KC0[2].Z, PV.W,
+; EG-NEXT: ALU clause starting at 11:
+; EG-NEXT: AND_INT * T1.W, T0.X, literal.x,
+; EG-NEXT: 31(4.344025e-44), 0(0.000000e+00)
+; EG-NEXT: LSHL T1.W, T0.X, PV.W,
+; EG-NEXT: AND_INT * T2.W, T0.X, literal.x,
+; EG-NEXT: 32(4.484155e-44), 0(0.000000e+00)
+; EG-NEXT: CNDE_INT * T1.W, PS, PV.W, 0.0,
+; EG-NEXT: BFE_INT * T1.W, PV.W, 0.0, 1,
+; EG-NEXT: AND_INT * T0.Y, PV.W, KC0[3].Z,
+; EG-NEXT: AND_INT T0.X, T1.W, KC0[3].Y,
+; EG-NEXT: ADD_INT * T0.W, KC0[2].Y, T0.W,
+; EG-NEXT: LSHR * T1.X, PV.W, literal.x,
+; EG-NEXT: 2(2.802597e-45), 0(0.000000e+00)
+;
+; GFX11-LABEL: v_sext_in_reg_i1_to_i64_move_use:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-NEXT: s_load_b64 s[4:5], s[4:5], 0x3c
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshlrev_b32_e32 v2, 3, v0
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: global_load_b64 v[0:1], v2, s[2:3]
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_lshlrev_b64 v[0:1], v0, v[0:1]
+; GFX11-NEXT: v_bfe_i32 v0, v0, 0, 1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_ashrrev_i32_e32 v1, 31, v0
+; GFX11-NEXT: v_and_b32_e32 v0, s4, v0
+; GFX11-NEXT: v_and_b32_e32 v1, s5, v1
+; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX11-NEXT: s_endpgm
%tid = call i32 @llvm.amdgcn.workitem.id.x()
%a.gep = getelementptr i64, ptr addrspace(1) %aptr, i32 %tid
%b.gep = getelementptr i64, ptr addrspace(1) %aptr, i32 %tid
@@ -489,20 +1705,76 @@ define amdgpu_kernel void @v_sext_in_reg_i1_to_i64_move_use(ptr addrspace(1) %ou
ret void
}
-; FUNC-LABEL: {{^}}v_sext_in_reg_i32_to_i64_move_use:
-; SI: buffer_load_dwordx2
-; SI: v_lshl_b64 v[[[LO:[0-9]+]]:[[HI:[0-9]+]]],
-; GFX89: {{flat|global}}_load_dwordx2
-; GFX89: v_lshlrev_b64 v[[[LO:[0-9]+]]:[[HI:[0-9]+]]],
-; GCN-DAG: v_ashrrev_i32_e32 v[[SHR:[0-9]+]], 31, v[[LO]]
-; GCN-DAG: v_and_b32_e32 v[[RESULT_LO:[0-9]+]], s{{[0-9]+}}, v[[LO]]
-; GCN-DAG: v_and_b32_e32 v[[RESULT_HI:[0-9]+]], s{{[0-9]+}}, v[[SHR]]
-; SI: buffer_store_dwordx2 v[[[RESULT_LO]]:[[RESULT_HI]]]
-; GFX89: {{flat|global}}_store_dwordx2 v{{.+}}, v[[[RESULT_LO]]:[[RESULT_HI]]]
define amdgpu_kernel void @v_sext_in_reg_i32_to_i64_move_use(ptr addrspace(1) %out, ptr addrspace(1) %aptr, ptr addrspace(1) %bptr, i64 %s.val) #0 {
+; SI-LABEL: v_sext_in_reg_i32_to_i64_move_use:
+; SI: ; %bb.0:
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0xf
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, 0
+; SI-NEXT: v_lshlrev_b32_e32 v0, 3, v0
+; SI-NEXT: v_mov_b32_e32 v1, 0
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b64 s[4:5], s[2:3]
+; SI-NEXT: buffer_load_dwordx2 v[2:3], v[0:1], s[4:7], 0 addr64
+; SI-NEXT: s_mov_b64 s[2:3], s[6:7]
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshl_b64 v[2:3], v[2:3], v2
+; SI-NEXT: v_ashrrev_i32_e32 v3, 31, v2
+; SI-NEXT: v_and_b32_e32 v3, s9, v3
+; SI-NEXT: v_and_b32_e32 v2, s8, v2
+; SI-NEXT: buffer_store_dwordx2 v[2:3], v[0:1], s[0:3], 0 addr64
+; SI-NEXT: s_endpgm
+;
+; EG-LABEL: v_sext_in_reg_i32_to_i64_move_use:
+; EG: ; %bb.0:
+; EG-NEXT: ALU 2, @8, KC0[CB0:0-32], KC1[]
+; EG-NEXT: TEX 0 @6
+; EG-NEXT: ALU 12, @11, KC0[CB0:0-32], KC1[]
+; EG-NEXT: MEM_RAT_CACHELESS STORE_RAW T0.XY, T1.X, 1
+; EG-NEXT: CF_END
+; EG-NEXT: PAD
+; EG-NEXT: Fetch clause starting at 6:
+; EG-NEXT: VTX_READ_32 T0.X, T0.X, 0, #1
+; EG-NEXT: ALU clause starting at 8:
+; EG-NEXT: LSHL * T0.W, T0.X, literal.x,
+; EG-NEXT: 3(4.203895e-45), 0(0.000000e+00)
+; EG-NEXT: ADD_INT * T0.X, KC0[2].Z, PV.W,
+; EG-NEXT: ALU clause starting at 11:
+; EG-NEXT: AND_INT * T1.W, T0.X, literal.x,
+; EG-NEXT: 31(4.344025e-44), 0(0.000000e+00)
+; EG-NEXT: LSHL T1.W, T0.X, PV.W,
+; EG-NEXT: AND_INT * T2.W, T0.X, literal.x,
+; EG-NEXT: 32(4.484155e-44), 0(0.000000e+00)
+; EG-NEXT: CNDE_INT * T1.W, PS, PV.W, 0.0,
+; EG-NEXT: AND_INT T0.X, PV.W, KC0[3].Y,
+; EG-NEXT: ASHR T1.W, PV.W, literal.x,
+; EG-NEXT: ADD_INT * T0.W, KC0[2].Y, T0.W,
+; EG-NEXT: 31(4.344025e-44), 0(0.000000e+00)
+; EG-NEXT: LSHR T1.X, PS, literal.x,
+; EG-NEXT: AND_INT * T0.Y, PV.W, KC0[3].Z,
+; EG-NEXT: 2(2.802597e-45), 0(0.000000e+00)
+;
+; GFX11-LABEL: v_sext_in_reg_i32_to_i64_move_use:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-NEXT: s_load_b64 s[4:5], s[4:5], 0x3c
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshlrev_b32_e32 v2, 3, v0
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: global_load_b64 v[0:1], v2, s[2:3]
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_lshlrev_b64 v[0:1], v0, v[0:1]
+; GFX11-NEXT: v_ashrrev_i32_e32 v1, 31, v0
+; GFX11-NEXT: v_and_b32_e32 v0, s4, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-NEXT: v_and_b32_e32 v1, s5, v1
+; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX11-NEXT: s_endpgm
%tid = call i32 @llvm.amdgcn.workitem.id.x()
%a.gep = getelementptr i64, ptr addrspace(1) %aptr, i32 %tid
%b.gep = getelementptr i64, ptr addrspace(1) %aptr, i32 %tid
@@ -518,14 +1790,62 @@ define amdgpu_kernel void @v_sext_in_reg_i32_to_i64_move_use(ptr addrspace(1) %o
ret void
}
-; FUNC-LABEL: {{^}}s_sext_in_reg_i1_i16:
-; GCN: s_load_dword [[VAL:s[0-9]+]]
-; GCN: s_bfe_i32 [[BFE:s[0-9]+]], [[VAL]], 0x10000
-; GCN: v_mov_b32_e32 [[VBFE:v[0-9]+]], [[BFE]]
-; GCN: buffer_store_short [[VBFE]]
define amdgpu_kernel void @s_sext_in_reg_i1_i16(ptr addrspace(1) %out, ptr addrspace(4) %ptr) #0 {
+; SI-LABEL: s_sext_in_reg_i1_i16:
+; SI: ; %bb.0:
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_load_dword s2, s[2:3], 0x0
+; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_bfe_i32 s4, s2, 0x10000
+; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: v_mov_b32_e32 v0, s4
+; SI-NEXT: buffer_store_short v0, off, s[0:3], 0
+; SI-NEXT: s_endpgm
+;
+; EG-LABEL: s_sext_in_reg_i1_i16:
+; EG: ; %bb.0:
+; EG-NEXT: ALU 0, @8, KC0[CB0:0-32], KC1[]
+; EG-NEXT: TEX 0 @6
+; EG-NEXT: ALU 12, @9, KC0[CB0:0-32], KC1[]
+; EG-NEXT: MEM_RAT MSKOR T0.XW, T1.X
+; EG-NEXT: CF_END
+; EG-NEXT: PAD
+; EG-NEXT: Fetch clause starting at 6:
+; EG-NEXT: VTX_READ_32 T0.X, T0.X, 0, #1
+; EG-NEXT: ALU clause starting at 8:
+; EG-NEXT: MOV * T0.X, KC0[2].Z,
+; EG-NEXT: ALU clause starting at 9:
+; EG-NEXT: BFE_INT T0.W, T0.X, 0.0, 1,
+; EG-NEXT: AND_INT * T1.W, KC0[2].Y, literal.x,
+; EG-NEXT: 3(4.203895e-45), 0(0.000000e+00)
+; EG-NEXT: AND_INT T0.W, PV.W, literal.x,
+; EG-NEXT: LSHL * T1.W, PS, literal.y,
+; EG-NEXT: 65535(9.183409e-41), 3(4.203895e-45)
+; EG-NEXT: LSHL T0.X, PV.W, PS,
+; EG-NEXT: LSHL * T0.W, literal.x, PS,
+; EG-NEXT: 65535(9.183409e-41), 0(0.000000e+00)
+; EG-NEXT: MOV T0.Y, 0.0,
+; EG-NEXT: MOV * T0.Z, 0.0,
+; EG-NEXT: LSHR * T1.X, KC0[2].Y, literal.x,
+; EG-NEXT: 2(2.802597e-45), 0(0.000000e+00)
+;
+; GFX11-LABEL: s_sext_in_reg_i1_i16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: s_load_b32 s2, s[2:3], 0x0
+; GFX11-NEXT: s_mov_b32 s3, 0x31016000
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: s_bfe_i32 s2, s2, 0x10000
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: v_mov_b32_e32 v0, s2
+; GFX11-NEXT: s_mov_b32 s2, -1
+; GFX11-NEXT: buffer_store_b16 v0, off, s[0:3], 0
+; GFX11-NEXT: s_endpgm
%ld = load i32, ptr addrspace(4) %ptr
%in = trunc i32 %ld to i16
%shl = shl i16 %in, 15
@@ -534,17 +1854,67 @@ define amdgpu_kernel void @s_sext_in_reg_i1_i16(ptr addrspace(1) %out, ptr addrs
ret void
}
-; FUNC-LABEL: {{^}}s_sext_in_reg_i2_i16:
-; GCN: s_load_dword [[VAL:s[0-9]+]]
-; SI: s_bfe_i32 [[BFE:s[0-9]+]], [[VAL]], 0x20000
-; SI: v_mov_b32_e32 [[VBFE:v[0-9]+]], [[BFE]]
-; SI: buffer_store_short [[VBFE]]
-; GFX89: s_lshl_b32 s{{[0-9]+}}, s{{[0-9]+}}, 14
-; GFX89: s_sext_i32_i16 s{{[0-9]+}}, s{{[0-9]+}}
-; GFX89: s_lshr_b32 s{{[0-9]+}}, s{{[0-9]+}}, 14
define amdgpu_kernel void @s_sext_in_reg_i2_i16(ptr addrspace(1) %out, ptr addrspace(4) %ptr) #0 {
+; SI-LABEL: s_sext_in_reg_i2_i16:
+; SI: ; %bb.0:
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_load_dword s2, s[2:3], 0x0
+; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_bfe_i32 s4, s2, 0x20000
+; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: v_mov_b32_e32 v0, s4
+; SI-NEXT: buffer_store_short v0, off, s[0:3], 0
+; SI-NEXT: s_endpgm
+;
+; EG-LABEL: s_sext_in_reg_i2_i16:
+; EG: ; %bb.0:
+; EG-NEXT: ALU 0, @8, KC0[CB0:0-32], KC1[]
+; EG-NEXT: TEX 0 @6
+; EG-NEXT: ALU 14, @9, KC0[CB0:0-32], KC1[]
+; EG-NEXT: MEM_RAT MSKOR T0.XW, T1.X
+; EG-NEXT: CF_END
+; EG-NEXT: PAD
+; EG-NEXT: Fetch clause starting at 6:
+; EG-NEXT: VTX_READ_32 T0.X, T0.X, 0, #1
+; EG-NEXT: ALU clause starting at 8:
+; EG-NEXT: MOV * T0.X, KC0[2].Z,
+; EG-NEXT: ALU clause starting at 9:
+; EG-NEXT: LSHL T0.W, T0.X, literal.x,
+; EG-NEXT: AND_INT * T1.W, KC0[2].Y, literal.y,
+; EG-NEXT: 30(4.203895e-44), 3(4.203895e-45)
+; EG-NEXT: ASHR * T0.W, PV.W, literal.x,
+; EG-NEXT: 30(4.203895e-44), 0(0.000000e+00)
+; EG-NEXT: AND_INT T0.W, PV.W, literal.x,
+; EG-NEXT: LSHL * T1.W, T1.W, literal.y,
+; EG-NEXT: 65535(9.183409e-41), 3(4.203895e-45)
+; EG-NEXT: LSHL T0.X, PV.W, PS,
+; EG-NEXT: LSHL * T0.W, literal.x, PS,
+; EG-NEXT: 65535(9.183409e-41), 0(0.000000e+00)
+; EG-NEXT: MOV T0.Y, 0.0,
+; EG-NEXT: MOV * T0.Z, 0.0,
+; EG-NEXT: LSHR * T1.X, KC0[2].Y, literal.x,
+; EG-NEXT: 2(2.802597e-45), 0(0.000000e+00)
+;
+; GFX11-LABEL: s_sext_in_reg_i2_i16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: s_load_b32 s2, s[2:3], 0x0
+; GFX11-NEXT: s_mov_b32 s3, 0x31016000
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: s_lshl_b32 s2, s2, 14
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_sext_i32_i16 s2, s2
+; GFX11-NEXT: s_lshr_b32 s2, s2, 14
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: v_mov_b32_e32 v0, s2
+; GFX11-NEXT: s_mov_b32 s2, -1
+; GFX11-NEXT: buffer_store_b16 v0, off, s[0:3], 0
+; GFX11-NEXT: s_endpgm
%ld = load i32, ptr addrspace(4) %ptr
%in = trunc i32 %ld to i16
%shl = shl i16 %in, 14
@@ -553,12 +1923,71 @@ define amdgpu_kernel void @s_sext_in_reg_i2_i16(ptr addrspace(1) %out, ptr addrs
ret void
}
-; FUNC-LABEL: {{^}}v_sext_in_reg_i1_i16:
-; GCN: {{buffer|flat|global}}_load_ushort [[VAL:v[0-9]+]]
-; GCN: v_bfe_i32 [[BFE:v[0-9]+]], [[VAL]], 0, 1{{$}}
-; GCN: ds_write_b16 v{{[0-9]+}}, [[BFE]]
define amdgpu_kernel void @v_sext_in_reg_i1_i16(ptr addrspace(3) %out, ptr addrspace(1) %ptr) #0 {
+; SI-LABEL: v_sext_in_reg_i1_i16:
+; SI: ; %bb.0:
+; SI-NEXT: s_load_dword s6, s[4:5], 0x9
+; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xb
+; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_mov_b32 s2, 0
+; SI-NEXT: v_lshlrev_b32_e32 v0, 1, v0
+; SI-NEXT: v_mov_b32_e32 v1, 0
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: buffer_load_ushort v1, v[0:1], s[0:3], 0 addr64
+; SI-NEXT: v_add_i32_e32 v0, vcc, s6, v0
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_bfe_i32 v1, v1, 0, 1
+; SI-NEXT: s_mov_b32 m0, -1
+; SI-NEXT: ds_write_b16 v0, v1
+; SI-NEXT: s_endpgm
+;
+; EG-LABEL: v_sext_in_reg_i1_i16:
+; EG: ; %bb.0:
+; EG-NEXT: ALU 1, @41, KC0[CB0:0-32], KC1[]
+; EG-NEXT: LSHL * T0.W, T0.X, 1,
+; EG-NEXT: ADD_INT * T0.X, KC0[2].Z, PV.W,
+; EG-NEXT: TEX 0 @0
+; EG-NEXT: VTX_READ_16 T0.X, T0.X, 0, #1
+; EG-NEXT: ALU 2, @42, KC0[CB0:0-32], KC1[]
+; EG-NEXT: BFE_INT T1.W, T0.X, 0.0, 1,
+; EG-NEXT: ADD_INT * T0.W, KC0[2].Y, T0.W,
+; EG-NEXT: LDS_SHORT_WRITE * T0.W, T1.W,
+; EG-NEXT: RETURN
+;
+; GFX11-TRUE16-LABEL: v_sext_in_reg_i1_i16:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x2c
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 1, v0
+; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT: global_load_d16_b16 v1, v0, s[0:1]
+; GFX11-TRUE16-NEXT: s_load_b32 s0, s[4:5], 0x24
+; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, s0, v0
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_bfe_i32 v1, v1, 0, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v1.l
+; GFX11-TRUE16-NEXT: ds_store_b16 v2, v0
+; GFX11-TRUE16-NEXT: s_endpgm
+;
+; GFX11-FAKE16-LABEL: v_sext_in_reg_i1_i16:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x2c
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 1, v0
+; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT: global_load_u16 v1, v0, s[0:1]
+; GFX11-FAKE16-NEXT: s_load_b32 s0, s[4:5], 0x24
+; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v0, s0, v0
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_bfe_i32 v1, v1, 0, 1
+; GFX11-FAKE16-NEXT: ds_store_b16 v0, v1
+; GFX11-FAKE16-NEXT: s_endpgm
%tid = call i32 @llvm.amdgcn.workitem.id.x()
%gep = getelementptr i16, ptr addrspace(1) %ptr, i32 %tid
%out.gep = getelementptr i16, ptr addrspace(3) %out, i32 %tid
@@ -570,16 +1999,91 @@ define amdgpu_kernel void @v_sext_in_reg_i1_i16(ptr addrspace(3) %out, ptr addrs
ret void
}
-; FUNC-LABEL: {{^}}v_sext_in_reg_i1_i16_nonload:
-; GCN: {{buffer|flat|global}}_load_ushort [[VAL0:v[0-9]+]]
-; GCN: {{buffer|flat|global}}_load_ushort [[VAL1:v[0-9]+]]
-; SI: v_lshlrev_b32_e32 [[REG:v[0-9]+]], [[VAL1]], [[VAL0]]
-; GFX89: v_lshlrev_b16_e32 [[REG:v[0-9]+]], [[VAL1]], [[VAL0]]
-; GCN: v_bfe_i32 [[BFE:v[0-9]+]], [[REG]], 0, 1{{$}}
-; GCN: ds_write_b16 v{{[0-9]+}}, [[BFE]]
define amdgpu_kernel void @v_sext_in_reg_i1_i16_nonload(ptr addrspace(3) %out, ptr addrspace(1) %aptr, ptr addrspace(1) %bptr, i16 %s.val) nounwind {
+; SI-LABEL: v_sext_in_reg_i1_i16_nonload:
+; SI: ; %bb.0:
+; SI-NEXT: s_load_dword s12, s[4:5], 0x9
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0xb
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, 0
+; SI-NEXT: v_lshlrev_b32_e32 v0, 1, v0
+; SI-NEXT: v_mov_b32_e32 v1, 0
+; SI-NEXT: s_mov_b64 s[10:11], s[6:7]
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b64 s[4:5], s[0:1]
+; SI-NEXT: s_mov_b64 s[8:9], s[2:3]
+; SI-NEXT: buffer_load_ushort v2, v[0:1], s[4:7], 0 addr64 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: buffer_load_ushort v1, v[0:1], s[8:11], 0 addr64 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_add_i32_e32 v0, vcc, s12, v0
+; SI-NEXT: v_lshlrev_b32_e32 v1, v1, v2
+; SI-NEXT: v_bfe_i32 v1, v1, 0, 1
+; SI-NEXT: s_mov_b32 m0, -1
+; SI-NEXT: ds_write_b16 v0, v1
+; SI-NEXT: s_endpgm
+;
+; EG-LABEL: v_sext_in_reg_i1_i16_nonload:
+; EG: ; %bb.0:
+; EG-NEXT: ALU 1, @43, KC0[CB0:0-32], KC1[]
+; EG-NEXT: LSHL * T0.W, T0.X, 1,
+; EG-NEXT: ADD_INT * T0.X, KC0[2].Z, PV.W,
+; EG-NEXT: TEX 0 @0
+; EG-NEXT: VTX_READ_16 T0.X, T0.X, 0, #1
+; EG-NEXT: ALU 0, @44, KC0[CB0:0-32], KC1[]
+; EG-NEXT: ADD_INT * T1.X, KC0[2].W, T0.W,
+; EG-NEXT: TEX 0 @0
+; EG-NEXT: VTX_READ_16 T1.X, T1.X, 0, #1
+; EG-NEXT: ALU 5, @45, KC0[CB0:0-32], KC1[]
+; EG-NEXT: AND_INT * T1.W, T1.X, literal.x,
+; EG-NEXT: 65535(9.183409e-41), 0(0.000000e+00)
+; EG-NEXT: LSHL * T1.W, T0.X, PV.W,
+; EG-NEXT: BFE_INT T1.W, PV.W, 0.0, 1,
+; EG-NEXT: ADD_INT * T0.W, KC0[2].Y, T0.W,
+; EG-NEXT: LDS_SHORT_WRITE * T0.W, T1.W,
+; EG-NEXT: RETURN
+;
+; GFX11-TRUE16-LABEL: v_sext_in_reg_i1_i16_nonload:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x2c
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 1, v0
+; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT: global_load_d16_b16 v0, v1, s[0:1] glc dlc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: global_load_d16_hi_b16 v0, v1, s[2:3] glc dlc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: s_load_b32 s0, s[4:5], 0x24
+; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, s0, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.l, v0.h, v0.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_bfe_i32 v0, v0, 0, 1
+; GFX11-TRUE16-NEXT: ds_store_b16 v1, v0
+; GFX11-TRUE16-NEXT: s_endpgm
+;
+; GFX11-FAKE16-LABEL: v_sext_in_reg_i1_i16_nonload:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x2c
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 1, v0
+; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT: global_load_u16 v1, v0, s[0:1] glc dlc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: global_load_u16 v2, v0, s[2:3] glc dlc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: s_load_b32 s0, s[4:5], 0x24
+; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v0, s0, v0
+; GFX11-FAKE16-NEXT: v_lshlrev_b16 v1, v2, v1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_bfe_i32 v1, v1, 0, 1
+; GFX11-FAKE16-NEXT: ds_store_b16 v0, v1
+; GFX11-FAKE16-NEXT: s_endpgm
%tid = call i32 @llvm.amdgcn.workitem.id.x()
%a.gep = getelementptr i16, ptr addrspace(1) %aptr, i32 %tid
%b.gep = getelementptr i16, ptr addrspace(1) %bptr, i32 %tid
@@ -595,60 +2099,260 @@ define amdgpu_kernel void @v_sext_in_reg_i1_i16_nonload(ptr addrspace(3) %out, p
ret void
}
-; FUNC-LABEL: {{^}}s_sext_in_reg_i2_i16_arg:
-; GCN: s_load_dword [[VAL:s[0-9]+]]
-; SI: s_bfe_i32 [[BFE:s[0-9]+]], [[VAL]], 0x20000
-; SI: v_mov_b32_e32 [[VBFE:v[0-9]+]], [[BFE]]
-; SI: buffer_store_short [[VBFE]]
-; GFX89: s_lshl_b32 s{{[0-9]+}}, s{{[0-9]+}}, 14{{$}}
-; GFX89: s_sext_i32_i16 s{{[0-9]+}}, s{{[0-9]+}}
-; GFX89: s_lshr_b32 s{{[0-9]+}}, s{{[0-9]+}}, 14{{$}}
define amdgpu_kernel void @s_sext_in_reg_i2_i16_arg(ptr addrspace(1) %out, i16 %in) #0 {
+; SI-LABEL: s_sext_in_reg_i2_i16_arg:
+; SI: ; %bb.0:
+; SI-NEXT: s_load_dword s2, s[4:5], 0xb
+; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_bfe_i32 s4, s2, 0x20000
+; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: v_mov_b32_e32 v0, s4
+; SI-NEXT: buffer_store_short v0, off, s[0:3], 0
+; SI-NEXT: s_endpgm
+;
+; EG-LABEL: s_sext_in_reg_i2_i16_arg:
+; EG: ; %bb.0:
+; EG-NEXT: ALU 0, @8, KC0[], KC1[]
+; EG-NEXT: TEX 0 @6
+; EG-NEXT: ALU 14, @9, KC0[CB0:0-32], KC1[]
+; EG-NEXT: MEM_RAT MSKOR T0.XW, T1.X
+; EG-NEXT: CF_END
+; EG-NEXT: PAD
+; EG-NEXT: Fetch clause starting at 6:
+; EG-NEXT: VTX_READ_16 T0.X, T0.X, 40, #3
+; EG-NEXT: ALU clause starting at 8:
+; EG-NEXT: MOV * T0.X, 0.0,
+; EG-NEXT: ALU clause starting at 9:
+; EG-NEXT: LSHL T0.W, T0.X, literal.x,
+; EG-NEXT: AND_INT * T1.W, KC0[2].Y, literal.y,
+; EG-NEXT: 30(4.203895e-44), 3(4.203895e-45)
+; EG-NEXT: ASHR * T0.W, PV.W, literal.x,
+; EG-NEXT: 30(4.203895e-44), 0(0.000000e+00)
+; EG-NEXT: AND_INT T0.W, PV.W, literal.x,
+; EG-NEXT: LSHL * T1.W, T1.W, literal.y,
+; EG-NEXT: 65535(9.183409e-41), 3(4.203895e-45)
+; EG-NEXT: LSHL T0.X, PV.W, PS,
+; EG-NEXT: LSHL * T0.W, literal.x, PS,
+; EG-NEXT: 65535(9.183409e-41), 0(0.000000e+00)
+; EG-NEXT: MOV T0.Y, 0.0,
+; EG-NEXT: MOV * T0.Z, 0.0,
+; EG-NEXT: LSHR * T1.X, KC0[2].Y, literal.x,
+; EG-NEXT: 2(2.802597e-45), 0(0.000000e+00)
+;
+; GFX11-LABEL: s_sext_in_reg_i2_i16_arg:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_clause 0x1
+; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT: s_mov_b32 s3, 0x31016000
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: s_lshl_b32 s2, s2, 14
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_sext_i32_i16 s2, s2
+; GFX11-NEXT: s_lshr_b32 s2, s2, 14
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: v_mov_b32_e32 v0, s2
+; GFX11-NEXT: s_mov_b32 s2, -1
+; GFX11-NEXT: buffer_store_b16 v0, off, s[0:3], 0
+; GFX11-NEXT: s_endpgm
%shl = shl i16 %in, 14
%sext = ashr i16 %shl, 14
store i16 %sext, ptr addrspace(1) %out
ret void
}
-; FUNC-LABEL: {{^}}s_sext_in_reg_i8_i16_arg:
-; GCN: s_load_dword [[VAL:s[0-9]+]]
-; SI: s_sext_i32_i8 [[SSEXT:s[0-9]+]], [[VAL]]
-; SI: v_mov_b32_e32 [[VSEXT:v[0-9]+]], [[SSEXT]]
-; SI: buffer_store_short [[VSEXT]]
-; GFX89: s_bfe_i32 s{{[0-9]+}}, s{{[0-9]+}}, 0x80000
define amdgpu_kernel void @s_sext_in_reg_i8_i16_arg(ptr addrspace(1) %out, i16 %in) #0 {
+; SI-LABEL: s_sext_in_reg_i8_i16_arg:
+; SI: ; %bb.0:
+; SI-NEXT: s_load_dword s2, s[4:5], 0xb
+; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_sext_i32_i8 s4, s2
+; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: v_mov_b32_e32 v0, s4
+; SI-NEXT: buffer_store_short v0, off, s[0:3], 0
+; SI-NEXT: s_endpgm
+;
+; EG-LABEL: s_sext_in_reg_i8_i16_arg:
+; EG: ; %bb.0:
+; EG-NEXT: ALU 0, @8, KC0[], KC1[]
+; EG-NEXT: TEX 0 @6
+; EG-NEXT: ALU 12, @9, KC0[CB0:0-32], KC1[]
+; EG-NEXT: MEM_RAT MSKOR T0.XW, T1.X
+; EG-NEXT: CF_END
+; EG-NEXT: PAD
+; EG-NEXT: Fetch clause starting at 6:
+; EG-NEXT: VTX_READ_8 T0.X, T0.X, 40, #3
+; EG-NEXT: ALU clause starting at 8:
+; EG-NEXT: MOV * T0.X, 0.0,
+; EG-NEXT: ALU clause starting at 9:
+; EG-NEXT: BFE_INT T0.W, T0.X, 0.0, literal.x,
+; EG-NEXT: AND_INT * T1.W, KC0[2].Y, literal.y,
+; EG-NEXT: 8(1.121039e-44), 3(4.203895e-45)
+; EG-NEXT: AND_INT T0.W, PV.W, literal.x,
+; EG-NEXT: LSHL * T1.W, PS, literal.y,
+; EG-NEXT: 65535(9.183409e-41), 3(4.203895e-45)
+; EG-NEXT: LSHL T0.X, PV.W, PS,
+; EG-NEXT: LSHL * T0.W, literal.x, PS,
+; EG-NEXT: 65535(9.183409e-41), 0(0.000000e+00)
+; EG-NEXT: MOV T0.Y, 0.0,
+; EG-NEXT: MOV * T0.Z, 0.0,
+; EG-NEXT: LSHR * T1.X, KC0[2].Y, literal.x,
+; EG-NEXT: 2(2.802597e-45), 0(0.000000e+00)
+;
+; GFX11-LABEL: s_sext_in_reg_i8_i16_arg:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_clause 0x1
+; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT: s_mov_b32 s3, 0x31016000
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: s_bfe_i32 s2, s2, 0x80000
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: v_mov_b32_e32 v0, s2
+; GFX11-NEXT: s_mov_b32 s2, -1
+; GFX11-NEXT: buffer_store_b16 v0, off, s[0:3], 0
+; GFX11-NEXT: s_endpgm
%shl = shl i16 %in, 8
%sext = ashr i16 %shl, 8
store i16 %sext, ptr addrspace(1) %out
ret void
}
-; FUNC-LABEL: {{^}}s_sext_in_reg_i15_i16_arg:
-; GCN: s_load_dword [[VAL:s[0-9]+]]
-; SI: s_bfe_i32 [[BFE:s[0-9]+]], [[VAL]], 0xf0000
-; SI: v_mov_b32_e32 [[VBFE:v[0-9]+]], [[BFE]]
-; SI: buffer_store_short [[VBFE]]
-; GFX89: s_lshl_b32 s{{[0-9]+}}, s{{[0-9]+}}, 1{{$}}
-; GFX89: s_sext_i32_i16 s{{[0-9]+}}, s{{[0-9]+}}
-; GFX89: s_lshr_b32 s{{[0-9]+}}, s{{[0-9]+}}, 1{{$}}
define amdgpu_kernel void @s_sext_in_reg_i15_i16_arg(ptr addrspace(1) %out, i16 %in) #0 {
+; SI-LABEL: s_sext_in_reg_i15_i16_arg:
+; SI: ; %bb.0:
+; SI-NEXT: s_load_dword s2, s[4:5], 0xb
+; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_bfe_i32 s4, s2, 0xf0000
+; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: v_mov_b32_e32 v0, s4
+; SI-NEXT: buffer_store_short v0, off, s[0:3], 0
+; SI-NEXT: s_endpgm
+;
+; EG-LABEL: s_sext_in_reg_i15_i16_arg:
+; EG: ; %bb.0:
+; EG-NEXT: ALU 0, @8, KC0[], KC1[]
+; EG-NEXT: TEX 0 @6
+; EG-NEXT: ALU 14, @9, KC0[CB0:0-32], KC1[]
+; EG-NEXT: MEM_RAT MSKOR T0.XW, T1.X
+; EG-NEXT: CF_END
+; EG-NEXT: PAD
+; EG-NEXT: Fetch clause starting at 6:
+; EG-NEXT: VTX_READ_16 T0.X, T0.X, 40, #3
+; EG-NEXT: ALU clause starting at 8:
+; EG-NEXT: MOV * T0.X, 0.0,
+; EG-NEXT: ALU clause starting at 9:
+; EG-NEXT: LSHL T0.W, T0.X, literal.x,
+; EG-NEXT: AND_INT * T1.W, KC0[2].Y, literal.y,
+; EG-NEXT: 17(2.382207e-44), 3(4.203895e-45)
+; EG-NEXT: ASHR * T0.W, PV.W, literal.x,
+; EG-NEXT: 17(2.382207e-44), 0(0.000000e+00)
+; EG-NEXT: AND_INT T0.W, PV.W, literal.x,
+; EG-NEXT: LSHL * T1.W, T1.W, literal.y,
+; EG-NEXT: 65535(9.183409e-41), 3(4.203895e-45)
+; EG-NEXT: LSHL T0.X, PV.W, PS,
+; EG-NEXT: LSHL * T0.W, literal.x, PS,
+; EG-NEXT: 65535(9.183409e-41), 0(0.000000e+00)
+; EG-NEXT: MOV T0.Y, 0.0,
+; EG-NEXT: MOV * T0.Z, 0.0,
+; EG-NEXT: LSHR * T1.X, KC0[2].Y, literal.x,
+; EG-NEXT: 2(2.802597e-45), 0(0.000000e+00)
+;
+; GFX11-LABEL: s_sext_in_reg_i15_i16_arg:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_clause 0x1
+; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT: s_mov_b32 s3, 0x31016000
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: s_lshl_b32 s2, s2, 1
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_sext_i32_i16 s2, s2
+; GFX11-NEXT: s_lshr_b32 s2, s2, 1
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: v_mov_b32_e32 v0, s2
+; GFX11-NEXT: s_mov_b32 s2, -1
+; GFX11-NEXT: buffer_store_b16 v0, off, s[0:3], 0
+; GFX11-NEXT: s_endpgm
%shl = shl i16 %in, 1
%sext = ashr i16 %shl, 1
store i16 %sext, ptr addrspace(1) %out
ret void
}
-; FUNC-LABEL: {{^}}sext_in_reg_v2i1_to_v2i16:
-; GFX9: v_pk_add_u16 [[ADD:v[0-9]+]]
-; GFX9: v_pk_lshlrev_b16 [[SHL:v[0-9]+]], 15, [[ADD]]
-; GFX9: v_pk_ashrrev_i16 [[SRA:v[0-9]+]], 15, [[SHL]]
define amdgpu_kernel void @sext_in_reg_v2i1_to_v2i16(ptr addrspace(1) %out, <2 x i16> %a, <2 x i16> %b) #0 {
+; SI-LABEL: sext_in_reg_v2i1_to_v2i16:
+; SI: ; %bb.0:
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_lshr_b32 s4, s2, 16
+; SI-NEXT: s_lshr_b32 s5, s3, 16
+; SI-NEXT: s_add_i32 s2, s2, s3
+; SI-NEXT: s_add_i32 s4, s4, s5
+; SI-NEXT: s_bfe_i32 s2, s2, 0x10000
+; SI-NEXT: s_bfe_i32 s3, s4, 0x10000
+; SI-NEXT: s_and_b32 s2, s2, 0xffff
+; SI-NEXT: s_lshl_b32 s3, s3, 16
+; SI-NEXT: s_or_b32 s2, s2, s3
+; SI-NEXT: s_mov_b32 s6, -1
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_mov_b32_e32 v0, s2
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
+; SI-NEXT: s_endpgm
+;
+; EG-LABEL: sext_in_reg_v2i1_to_v2i16:
+; EG: ; %bb.0:
+; EG-NEXT: ALU 0, @14, KC0[], KC1[]
+; EG-NEXT: TEX 3 @6
+; EG-NEXT: ALU 9, @15, KC0[CB0:0-32], KC1[]
+; EG-NEXT: MEM_RAT_CACHELESS STORE_RAW T4.X, T5.X, 1
+; EG-NEXT: CF_END
+; EG-NEXT: PAD
+; EG-NEXT: Fetch clause starting at 6:
+; EG-NEXT: VTX_READ_16 T5.X, T4.X, 42, #3
+; EG-NEXT: VTX_READ_16 T6.X, T4.X, 46, #3
+; EG-NEXT: VTX_READ_16 T7.X, T4.X, 40, #3
+; EG-NEXT: VTX_READ_16 T4.X, T4.X, 44, #3
+; EG-NEXT: ALU clause starting at 14:
+; EG-NEXT: MOV * T4.X, 0.0,
+; EG-NEXT: ALU clause starting at 15:
+; EG-NEXT: ADD_INT * T0.W, T5.X, T6.X,
+; EG-NEXT: ADD_INT * T1.W, T7.X, T4.X,
+; EG-NEXT: BFE_INT T0.Z, PV.W, 0.0, 1,
+; EG-NEXT: BFE_INT * T0.W, T0.W, 0.0, 1,
+; EG-NEXT: AND_INT T0.W, PV.W, literal.x,
+; EG-NEXT: AND_INT * T1.W, PV.Z, literal.y,
+; EG-NEXT: -65536(nan), 65535(9.183409e-41)
+; EG-NEXT: OR_INT T4.X, PV.W, PS,
+; EG-NEXT: LSHR * T5.X, KC0[2].Y, literal.x,
+; EG-NEXT: 2(2.802597e-45), 0(0.000000e+00)
+;
+; GFX11-LABEL: sext_in_reg_v2i1_to_v2i16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: v_pk_add_u16 v0, s2, s3
+; GFX11-NEXT: s_mov_b32 s3, 0x31016000
+; GFX11-NEXT: s_mov_b32 s2, -1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_pk_lshlrev_b16 v0, 15, v0 op_sel_hi:[0,1]
+; GFX11-NEXT: v_pk_ashrrev_i16 v0, 15, v0 op_sel_hi:[0,1]
+; GFX11-NEXT: buffer_store_b32 v0, off, s[0:3], 0
+; GFX11-NEXT: s_endpgm
%c = add <2 x i16> %a, %b ; add to prevent folding into extload
%shl = shl <2 x i16> %c, <i16 15, i16 15>
%ashr = ashr <2 x i16> %shl, <i16 15, i16 15>
@@ -656,14 +2360,89 @@ define amdgpu_kernel void @sext_in_reg_v2i1_to_v2i16(ptr addrspace(1) %out, <2 x
ret void
}
-; FUNC-LABEL: {{^}}sext_in_reg_v3i1_to_v3i16:
-; GFX9: v_pk_add_u16
-; GFX9: v_pk_add_u16
-; GFX9: v_pk_lshlrev_b16 v{{[0-9]+}}, 15, v{{[0-9]+}}
-; GFX9: v_pk_lshlrev_b16 v{{[0-9]+}}, 15, v{{[0-9]+}}
-; GFX9: v_pk_ashrrev_i16 v{{[0-9]+}}, 15, v{{[0-9]+}}
-; GFX9: v_pk_ashrrev_i16 v{{[0-9]+}}, 15, v{{[0-9]+}}
define amdgpu_kernel void @sext_in_reg_v3i1_to_v3i16(ptr addrspace(1) %out, <3 x i16> %a, <3 x i16> %b) #0 {
+; SI-LABEL: sext_in_reg_v3i1_to_v3i16:
+; SI: ; %bb.0:
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0xb
+; SI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_lshr_b32 s8, s0, 16
+; SI-NEXT: s_lshr_b32 s9, s2, 16
+; SI-NEXT: s_add_i32 s1, s1, s3
+; SI-NEXT: s_add_i32 s0, s0, s2
+; SI-NEXT: s_add_i32 s8, s8, s9
+; SI-NEXT: s_bfe_i32 s0, s0, 0x10000
+; SI-NEXT: s_bfe_i32 s1, s1, 0x10000
+; SI-NEXT: s_bfe_i32 s2, s8, 0x10000
+; SI-NEXT: s_and_b32 s0, s0, 0xffff
+; SI-NEXT: v_mov_b32_e32 v0, s1
+; SI-NEXT: s_lshl_b32 s1, s2, 16
+; SI-NEXT: s_or_b32 s0, s0, s1
+; SI-NEXT: buffer_store_short v0, off, s[4:7], 0 offset:4
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_mov_b32_e32 v0, s0
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
+; SI-NEXT: s_endpgm
+;
+; EG-LABEL: sext_in_reg_v3i1_to_v3i16:
+; EG: ; %bb.0:
+; EG-NEXT: ALU 0, @14, KC0[], KC1[]
+; EG-NEXT: TEX 3 @6
+; EG-NEXT: ALU 19, @15, KC0[CB0:0-32], KC1[]
+; EG-NEXT: MEM_RAT_CACHELESS STORE_RAW T1.X, T2.X, 0
+; EG-NEXT: MEM_RAT MSKOR T3.XW, T0.X
+; EG-NEXT: CF_END
+; EG-NEXT: Fetch clause starting at 6:
+; EG-NEXT: VTX_READ_16 T1.X, T0.X, 44, #3
+; EG-NEXT: VTX_READ_16 T2.X, T0.X, 52, #3
+; EG-NEXT: VTX_READ_16 T3.X, T0.X, 48, #3
+; EG-NEXT: VTX_READ_16 T0.X, T0.X, 56, #3
+; EG-NEXT: ALU clause starting at 14:
+; EG-NEXT: MOV * T0.X, 0.0,
+; EG-NEXT: ALU clause starting at 15:
+; EG-NEXT: ADD_INT T0.W, KC0[2].Y, literal.x,
+; EG-NEXT: ADD_INT * T1.W, T3.X, T0.X,
+; EG-NEXT: 4(5.605194e-45), 0(0.000000e+00)
+; EG-NEXT: BFE_INT T1.W, PS, 0.0, 1,
+; EG-NEXT: AND_INT * T2.W, PV.W, literal.x,
+; EG-NEXT: 3(4.203895e-45), 0(0.000000e+00)
+; EG-NEXT: AND_INT T1.W, PV.W, literal.x,
+; EG-NEXT: LSHL * T2.W, PS, literal.y,
+; EG-NEXT: 65535(9.183409e-41), 3(4.203895e-45)
+; EG-NEXT: LSHL T3.X, PV.W, PS,
+; EG-NEXT: LSHL * T3.W, literal.x, PS,
+; EG-NEXT: 65535(9.183409e-41), 0(0.000000e+00)
+; EG-NEXT: MOV T3.Y, 0.0,
+; EG-NEXT: MOV * T3.Z, 0.0,
+; EG-NEXT: LSHR T0.X, T0.W, literal.x,
+; EG-NEXT: ADD_INT * T0.W, T1.X, T2.X,
+; EG-NEXT: 2(2.802597e-45), 0(0.000000e+00)
+; EG-NEXT: BFE_INT T1.X, PV.W, 0.0, 1,
+; EG-NEXT: LSHR * T2.X, KC0[2].Y, literal.x,
+; EG-NEXT: 2(2.802597e-45), 0(0.000000e+00)
+;
+; GFX11-LABEL: sext_in_reg_v3i1_to_v3i16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_clause 0x1
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x2c
+; GFX11-NEXT: s_load_b64 s[4:5], s[4:5], 0x24
+; GFX11-NEXT: s_mov_b32 s7, 0x31016000
+; GFX11-NEXT: s_mov_b32 s6, -1
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: v_pk_add_u16 v0, s1, s3
+; GFX11-NEXT: v_pk_add_u16 v1, s0, s2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_pk_lshlrev_b16 v0, 15, v0 op_sel_hi:[0,1]
+; GFX11-NEXT: v_pk_lshlrev_b16 v1, 15, v1 op_sel_hi:[0,1]
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_pk_ashrrev_i16 v0, 15, v0 op_sel_hi:[0,1]
+; GFX11-NEXT: v_pk_ashrrev_i16 v1, 15, v1 op_sel_hi:[0,1]
+; GFX11-NEXT: s_clause 0x1
+; GFX11-NEXT: buffer_store_b16 v0, off, s[4:7], 0 offset:4
+; GFX11-NEXT: buffer_store_b32 v1, off, s[4:7], 0
+; GFX11-NEXT: s_endpgm
%c = add <3 x i16> %a, %b ; add to prevent folding into extload
%shl = shl <3 x i16> %c, <i16 15, i16 15, i16 15>
%ashr = ashr <3 x i16> %shl, <i16 15, i16 15, i16 15>
@@ -671,11 +2450,71 @@ define amdgpu_kernel void @sext_in_reg_v3i1_to_v3i16(ptr addrspace(1) %out, <3 x
ret void
}
-; FUNC-LABEL: {{^}}sext_in_reg_v2i2_to_v2i16:
-; GFX9: v_pk_add_u16 [[ADD:v[0-9]+]]
-; GFX9: v_pk_lshlrev_b16 [[SHL:v[0-9]+]], 14, [[ADD]]
-; GFX9: v_pk_ashrrev_i16 [[SRA:v[0-9]+]], 14, [[SHL]]
define amdgpu_kernel void @sext_in_reg_v2i2_to_v2i16(ptr addrspace(1) %out, <2 x i16> %a, <2 x i16> %b) #0 {
+; SI-LABEL: sext_in_reg_v2i2_to_v2i16:
+; SI: ; %bb.0:
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_lshr_b32 s4, s2, 16
+; SI-NEXT: s_lshr_b32 s5, s3, 16
+; SI-NEXT: s_add_i32 s2, s2, s3
+; SI-NEXT: s_add_i32 s4, s4, s5
+; SI-NEXT: s_bfe_i32 s2, s2, 0x20000
+; SI-NEXT: s_bfe_i32 s3, s4, 0x20000
+; SI-NEXT: s_and_b32 s2, s2, 0xffff
+; SI-NEXT: s_lshl_b32 s3, s3, 16
+; SI-NEXT: s_or_b32 s2, s2, s3
+; SI-NEXT: s_mov_b32 s6, -1
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_mov_b32_e32 v0, s2
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
+; SI-NEXT: s_endpgm
+;
+; EG-LABEL: sext_in_reg_v2i2_to_v2i16:
+; EG: ; %bb.0:
+; EG-NEXT: ALU 0, @14, KC0[], KC1[]
+; EG-NEXT: TEX 3 @6
+; EG-NEXT: ALU 13, @15, KC0[CB0:0-32], KC1[]
+; EG-NEXT: MEM_RAT_CACHELESS STORE_RAW T4.X, T5.X, 1
+; EG-NEXT: CF_END
+; EG-NEXT: PAD
+; EG-NEXT: Fetch clause starting at 6:
+; EG-NEXT: VTX_READ_16 T5.X, T4.X, 40, #3
+; EG-NEXT: VTX_READ_16 T6.X, T4.X, 44, #3
+; EG-NEXT: VTX_READ_16 T7.X, T4.X, 42, #3
+; EG-NEXT: VTX_READ_16 T4.X, T4.X, 46, #3
+; EG-NEXT: ALU clause starting at 14:
+; EG-NEXT: MOV * T4.X, 0.0,
+; EG-NEXT: ALU clause starting at 15:
+; EG-NEXT: ADD_INT * T0.W, T5.X, T6.X,
+; EG-NEXT: ADD_INT * T1.W, T7.X, T4.X,
+; EG-NEXT: LSHL T1.W, PV.W, literal.x,
+; EG-NEXT: LSHL * T0.W, T0.W, literal.x,
+; EG-NEXT: 30(4.203895e-44), 0(0.000000e+00)
+; EG-NEXT: ASHR T0.W, PS, literal.x,
+; EG-NEXT: ASHR * T1.W, PV.W, literal.x,
+; EG-NEXT: 30(4.203895e-44), 0(0.000000e+00)
+; EG-NEXT: LSHL T1.W, PS, literal.x,
+; EG-NEXT: AND_INT * T0.W, PV.W, literal.y,
+; EG-NEXT: 16(2.242078e-44), 65535(9.183409e-41)
+; EG-NEXT: OR_INT T4.X, PV.W, PS,
+; EG-NEXT: LSHR * T5.X, KC0[2].Y, literal.x,
+; EG-NEXT: 2(2.802597e-45), 0(0.000000e+00)
+;
+; GFX11-LABEL: sext_in_reg_v2i2_to_v2i16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: v_pk_add_u16 v0, s2, s3
+; GFX11-NEXT: s_mov_b32 s3, 0x31016000
+; GFX11-NEXT: s_mov_b32 s2, -1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_pk_lshlrev_b16 v0, 14, v0 op_sel_hi:[0,1]
+; GFX11-NEXT: v_pk_ashrrev_i16 v0, 14, v0 op_sel_hi:[0,1]
+; GFX11-NEXT: buffer_store_b32 v0, off, s[0:3], 0
+; GFX11-NEXT: s_endpgm
%c = add <2 x i16> %a, %b ; add to prevent folding into extload
%shl = shl <2 x i16> %c, <i16 14, i16 14>
%ashr = ashr <2 x i16> %shl, <i16 14, i16 14>
@@ -683,11 +2522,68 @@ define amdgpu_kernel void @sext_in_reg_v2i2_to_v2i16(ptr addrspace(1) %out, <2 x
ret void
}
-; FUNC-LABEL: {{^}}sext_in_reg_v2i8_to_v2i16:
-; GFX9: v_pk_add_u16 [[ADD:v[0-9]+]]
-; GFX9: v_pk_lshlrev_b16 [[SHL:v[0-9]+]], 8, [[ADD]]
-; GFX9: v_pk_ashrrev_i16 [[SRA:v[0-9]+]], 8, [[SHL]]
define amdgpu_kernel void @sext_in_reg_v2i8_to_v2i16(ptr addrspace(1) %out, <2 x i16> %a, <2 x i16> %b) #0 {
+; SI-LABEL: sext_in_reg_v2i8_to_v2i16:
+; SI: ; %bb.0:
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_lshr_b32 s4, s2, 16
+; SI-NEXT: s_lshr_b32 s5, s3, 16
+; SI-NEXT: s_add_i32 s2, s2, s3
+; SI-NEXT: s_add_i32 s4, s4, s5
+; SI-NEXT: s_sext_i32_i8 s2, s2
+; SI-NEXT: s_sext_i32_i8 s3, s4
+; SI-NEXT: s_and_b32 s2, s2, 0xffff
+; SI-NEXT: s_lshl_b32 s3, s3, 16
+; SI-NEXT: s_or_b32 s2, s2, s3
+; SI-NEXT: s_mov_b32 s6, -1
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_mov_b32_e32 v0, s2
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
+; SI-NEXT: s_endpgm
+;
+; EG-LABEL: sext_in_reg_v2i8_to_v2i16:
+; EG: ; %bb.0:
+; EG-NEXT: ALU 0, @14, KC0[], KC1[]
+; EG-NEXT: TEX 3 @6
+; EG-NEXT: ALU 10, @15, KC0[CB0:0-32], KC1[]
+; EG-NEXT: MEM_RAT_CACHELESS STORE_RAW T4.X, T5.X, 1
+; EG-NEXT: CF_END
+; EG-NEXT: PAD
+; EG-NEXT: Fetch clause starting at 6:
+; EG-NEXT: VTX_READ_16 T5.X, T4.X, 42, #3
+; EG-NEXT: VTX_READ_16 T6.X, T4.X, 46, #3
+; EG-NEXT: VTX_READ_16 T7.X, T4.X, 40, #3
+; EG-NEXT: VTX_READ_16 T4.X, T4.X, 44, #3
+; EG-NEXT: ALU clause starting at 14:
+; EG-NEXT: MOV * T4.X, 0.0,
+; EG-NEXT: ALU clause starting at 15:
+; EG-NEXT: ADD_INT * T0.W, T5.X, T6.X,
+; EG-NEXT: ADD_INT * T1.W, T7.X, T4.X,
+; EG-NEXT: BFE_INT T0.Z, PV.W, 0.0, literal.x,
+; EG-NEXT: BFE_INT * T0.W, T0.W, 0.0, literal.x,
+; EG-NEXT: 8(1.121039e-44), 0(0.000000e+00)
+; EG-NEXT: LSHL T0.W, PV.W, literal.x,
+; EG-NEXT: AND_INT * T1.W, PV.Z, literal.y,
+; EG-NEXT: 16(2.242078e-44), 65535(9.183409e-41)
+; EG-NEXT: OR_INT T4.X, PV.W, PS,
+; EG-NEXT: LSHR * T5.X, KC0[2].Y, literal.x,
+; EG-NEXT: 2(2.802597e-45), 0(0.000000e+00)
+;
+; GFX11-LABEL: sext_in_reg_v2i8_to_v2i16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: v_pk_add_u16 v0, s2, s3
+; GFX11-NEXT: s_mov_b32 s3, 0x31016000
+; GFX11-NEXT: s_mov_b32 s2, -1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_pk_lshlrev_b16 v0, 8, v0 op_sel_hi:[0,1]
+; GFX11-NEXT: v_pk_ashrrev_i16 v0, 8, v0 op_sel_hi:[0,1]
+; GFX11-NEXT: buffer_store_b32 v0, off, s[0:3], 0
+; GFX11-NEXT: s_endpgm
%c = add <2 x i16> %a, %b ; add to prevent folding into extload
%shl = shl <2 x i16> %c, <i16 8, i16 8>
%ashr = ashr <2 x i16> %shl, <i16 8, i16 8>
@@ -695,14 +2591,89 @@ define amdgpu_kernel void @sext_in_reg_v2i8_to_v2i16(ptr addrspace(1) %out, <2 x
ret void
}
-; FUNC-LABEL: {{^}}sext_in_reg_v3i8_to_v3i16:
-; GFX9: v_pk_add_u16
-; GFX9: v_pk_add_u16
-; GFX9: v_pk_lshlrev_b16 v{{[0-9]+}}, 8, v{{[0-9]+}}
-; GFX9: v_pk_lshlrev_b16 v{{[0-9]+}}, 8, v{{[0-9]+}}
-; GFX9: v_pk_ashrrev_i16 v{{[0-9]+}}, 8, v{{[0-9]+}}
-; GFX9: v_pk_ashrrev_i16 v{{[0-9]+}}, 8, v{{[0-9]+}}
define amdgpu_kernel void @sext_in_reg_v3i8_to_v3i16(ptr addrspace(1) %out, <3 x i16> %a, <3 x i16> %b) #0 {
+; SI-LABEL: sext_in_reg_v3i8_to_v3i16:
+; SI: ; %bb.0:
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0xb
+; SI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_lshr_b32 s8, s0, 16
+; SI-NEXT: s_lshr_b32 s9, s2, 16
+; SI-NEXT: s_add_i32 s1, s1, s3
+; SI-NEXT: s_add_i32 s0, s0, s2
+; SI-NEXT: s_add_i32 s8, s8, s9
+; SI-NEXT: s_sext_i32_i8 s0, s0
+; SI-NEXT: s_sext_i32_i8 s1, s1
+; SI-NEXT: s_sext_i32_i8 s2, s8
+; SI-NEXT: s_and_b32 s0, s0, 0xffff
+; SI-NEXT: v_mov_b32_e32 v0, s1
+; SI-NEXT: s_lshl_b32 s1, s2, 16
+; SI-NEXT: s_or_b32 s0, s0, s1
+; SI-NEXT: buffer_store_short v0, off, s[4:7], 0 offset:4
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_mov_b32_e32 v0, s0
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
+; SI-NEXT: s_endpgm
+;
+; EG-LABEL: sext_in_reg_v3i8_to_v3i16:
+; EG: ; %bb.0:
+; EG-NEXT: ALU 0, @14, KC0[], KC1[]
+; EG-NEXT: TEX 3 @6
+; EG-NEXT: ALU 19, @15, KC0[CB0:0-32], KC1[]
+; EG-NEXT: MEM_RAT_CACHELESS STORE_RAW T1.X, T2.X, 0
+; EG-NEXT: MEM_RAT MSKOR T3.XW, T0.X
+; EG-NEXT: CF_END
+; EG-NEXT: Fetch clause starting at 6:
+; EG-NEXT: VTX_READ_16 T1.X, T0.X, 44, #3
+; EG-NEXT: VTX_READ_16 T2.X, T0.X, 52, #3
+; EG-NEXT: VTX_READ_16 T3.X, T0.X, 48, #3
+; EG-NEXT: VTX_READ_16 T0.X, T0.X, 56, #3
+; EG-NEXT: ALU clause starting at 14:
+; EG-NEXT: MOV * T0.X, 0.0,
+; EG-NEXT: ALU clause starting at 15:
+; EG-NEXT: ADD_INT T0.W, KC0[2].Y, literal.x,
+; EG-NEXT: ADD_INT * T1.W, T3.X, T0.X,
+; EG-NEXT: 4(5.605194e-45), 0(0.000000e+00)
+; EG-NEXT: BFE_INT T1.W, PS, 0.0, literal.x,
+; EG-NEXT: AND_INT * T2.W, PV.W, literal.y,
+; EG-NEXT: 8(1.121039e-44), 3(4.203895e-45)
+; EG-NEXT: AND_INT T1.W, PV.W, literal.x,
+; EG-NEXT: LSHL * T2.W, PS, literal.y,
+; EG-NEXT: 65535(9.183409e-41), 3(4.203895e-45)
+; EG-NEXT: LSHL T3.X, PV.W, PS,
+; EG-NEXT: LSHL * T3.W, literal.x, PS,
+; EG-NEXT: 65535(9.183409e-41), 0(0.000000e+00)
+; EG-NEXT: MOV T3.Y, 0.0,
+; EG-NEXT: MOV * T3.Z, 0.0,
+; EG-NEXT: LSHR T0.X, T0.W, literal.x,
+; EG-NEXT: ADD_INT * T0.W, T1.X, T2.X,
+; EG-NEXT: 2(2.802597e-45), 0(0.000000e+00)
+; EG-NEXT: BFE_INT T1.X, PV.W, 0.0, literal.x,
+; EG-NEXT: LSHR * T2.X, KC0[2].Y, literal.y,
+; EG-NEXT: 8(1.121039e-44), 2(2.802597e-45)
+;
+; GFX11-LABEL: sext_in_reg_v3i8_to_v3i16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_clause 0x1
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x2c
+; GFX11-NEXT: s_load_b64 s[4:5], s[4:5], 0x24
+; GFX11-NEXT: s_mov_b32 s7, 0x31016000
+; GFX11-NEXT: s_mov_b32 s6, -1
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: v_pk_add_u16 v0, s1, s3
+; GFX11-NEXT: v_pk_add_u16 v1, s0, s2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_pk_lshlrev_b16 v0, 8, v0 op_sel_hi:[0,1]
+; GFX11-NEXT: v_pk_lshlrev_b16 v1, 8, v1 op_sel_hi:[0,1]
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_pk_ashrrev_i16 v0, 8, v0 op_sel_hi:[0,1]
+; GFX11-NEXT: v_pk_ashrrev_i16 v1, 8, v1 op_sel_hi:[0,1]
+; GFX11-NEXT: s_clause 0x1
+; GFX11-NEXT: buffer_store_b16 v0, off, s[4:7], 0 offset:4
+; GFX11-NEXT: buffer_store_b32 v1, off, s[4:7], 0
+; GFX11-NEXT: s_endpgm
%c = add <3 x i16> %a, %b ; add to prevent folding into extload
%shl = shl <3 x i16> %c, <i16 8, i16 8, i16 8>
%ashr = ashr <3 x i16> %shl, <i16 8, i16 8, i16 8>
@@ -714,3 +2685,8 @@ declare i32 @llvm.amdgcn.workitem.id.x() #1
attributes #0 = { nounwind }
attributes #1 = { nounwind readnone }
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; FUNC: {{.*}}
+; GCN: {{.*}}
+; GFX89: {{.*}}
+; GFX9: {{.*}}
More information about the llvm-commits
mailing list