[llvm] Main true16 fix gisel copy (PR #215401)
Guo Chen via llvm-commits
llvm-commits at lists.llvm.org
Mon Aug 10 14:09:50 PDT 2026
https://github.com/broxigarchen created https://github.com/llvm/llvm-project/pull/215401
None
>From ff717a913982c2a7cc65dcf6c9570e26c1001bc0 Mon Sep 17 00:00:00 2001
From: guochen2 <guochen2 at amd.com>
Date: Mon, 10 Aug 2026 13:47:41 -0400
Subject: [PATCH 1/2] fold sgpr32 via vgpr16
---
llvm/lib/Target/AMDGPU/SIFoldOperands.cpp | 39 +++++++++
llvm/test/CodeGen/AMDGPU/GlobalISel/fshl.ll | 84 ++++++-------------
llvm/test/CodeGen/AMDGPU/GlobalISel/fshr.ll | 15 +---
.../inst-select-amdgcn.class.s16.mir | 3 +-
.../CodeGen/AMDGPU/llvm.amdgcn.alignbyte.ll | 4 +-
llvm/test/CodeGen/AMDGPU/true16-fold.mir | 62 ++++++++++++++
6 files changed, 133 insertions(+), 74 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp b/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
index 04a9ed487d655..19ad398dab427 100644
--- a/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
+++ b/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
@@ -1508,6 +1508,45 @@ bool SIFoldOperandsImpl::foldOperand(
return true;
}
+ // Look through Lo16 Copy
+ // OpToFold: %0
+ // %1:vgpr32 = copy %0:sreg32/sregxx:sub_x
+ // %2:vgpr16 = copy %1.lo16:vgpr32 (UseMI)
+ // VALU16 %2:vgpr16 ...
+ // =>
+ // VALU16 %0:sreg32/sregxx:sub_x...
+ // Another hack to allow 32-bit SGPRs to be folded into True16 instructions
+ // Remove this if 16-bit SGPRs (i.e. SGPR_LO16) are added to the
+ // VS_16RegClass
+ if (UseMI->isCopy() && OpToFold.isReg() &&
+ UseMI->getOperand(0).getReg().isVirtual() &&
+ TRI->isSGPRReg(*MRI, OpToFold.getReg()) &&
+ TII->getOpSize(*UseMI, 0) == 2 &&
+ UseMI->getOperand(1).getSubReg() == AMDGPU::lo16 &&
+ OpToFold.DefMI->implicit_operands().empty()) {
+
+ // Append Users of the UseMI instead
+ SmallVector<MachineOperand *, 4> UsesToProcess(llvm::make_pointer_range(
+ MRI->use_nodbg_operands(UseMI->getOperand(0).getReg())));
+ for (auto *U : UsesToProcess) {
+ MachineInstr *NextMI = U->getParent();
+
+ const MCInstrDesc &UseDesc = NextMI->getDesc();
+ if (UseDesc.isVariadic() || U->isImplicit() ||
+ UseDesc.operands()[NextMI->getOperandNo(U)].RegClass == -1)
+ continue;
+
+ FoldableDef NextOpToFold(*OpToFold.OpToFold, OpToFold.DefRC,
+ U->getSubReg());
+ LLVM_DEBUG(dbgs() << "Folding " << *NextOpToFold.OpToFold
+ << "\n through" << *UseMI << " into " << *NextMI);
+
+ Changed |= tryAddToFoldList(FoldList, NextMI, NextMI->getOperandNo(U),
+ NextOpToFold);
+ }
+ return Changed;
+ }
+
unsigned UseOpc = UseMI->getOpcode();
if (UseOpc == AMDGPU::V_READFIRSTLANE_B32 ||
(UseOpc == AMDGPU::V_READLANE_B32 &&
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fshl.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fshl.ll
index 6844b698473c7..14294565c0f63 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fshl.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fshl.ll
@@ -3028,28 +3028,16 @@ define amdgpu_ps i32 @s_fshl_i32(i32 inreg %lhs, i32 inreg %rhs, i32 inreg %amt)
; GFX10-NEXT: v_readfirstlane_b32 s0, v0
; GFX10-NEXT: ; return to shader part epilog
;
-; GFX11-TRUE16-LABEL: s_fshl_i32:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_not_b32 s2, s2
-; GFX11-TRUE16-NEXT: v_alignbit_b32 v0, s0, s1, 1
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, s2
-; GFX11-TRUE16-NEXT: s_lshr_b32 s0, s0, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_alignbit_b32 v0, s0, v0, v1.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
-; GFX11-TRUE16-NEXT: ; return to shader part epilog
-;
-; GFX11-FAKE16-LABEL: s_fshl_i32:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: v_alignbit_b32 v0, s0, s1, 1
-; GFX11-FAKE16-NEXT: s_lshr_b32 s0, s0, 1
-; GFX11-FAKE16-NEXT: s_not_b32 s1, s2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: v_alignbit_b32 v0, s0, v0, s1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
-; GFX11-FAKE16-NEXT: ; return to shader part epilog
+; GFX11-LABEL: s_fshl_i32:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: v_alignbit_b32 v0, s0, s1, 1
+; GFX11-NEXT: s_lshr_b32 s0, s0, 1
+; GFX11-NEXT: s_not_b32 s1, s2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: v_alignbit_b32 v0, s0, v0, s1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-NEXT: ; return to shader part epilog
%result = call i32 @llvm.fshl.i32(i32 %lhs, i32 %rhs, i32 %amt)
ret i32 %result
}
@@ -3289,24 +3277,14 @@ define amdgpu_ps float @v_fshl_i32_svs(i32 inreg %lhs, i32 %rhs, i32 inreg %amt)
; GFX10-NEXT: v_alignbit_b32 v0, s0, v0, s1
; GFX10-NEXT: ; return to shader part epilog
;
-; GFX11-TRUE16-LABEL: v_fshl_i32_svs:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_not_b32 s1, s1
-; GFX11-TRUE16-NEXT: v_alignbit_b32 v0, s0, v0, 1
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, s1
-; GFX11-TRUE16-NEXT: s_lshr_b32 s0, s0, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_alignbit_b32 v0, s0, v0, v1.l
-; GFX11-TRUE16-NEXT: ; return to shader part epilog
-;
-; GFX11-FAKE16-LABEL: v_fshl_i32_svs:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: v_alignbit_b32 v0, s0, v0, 1
-; GFX11-FAKE16-NEXT: s_lshr_b32 s0, s0, 1
-; GFX11-FAKE16-NEXT: s_not_b32 s1, s1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: v_alignbit_b32 v0, s0, v0, s1
-; GFX11-FAKE16-NEXT: ; return to shader part epilog
+; GFX11-LABEL: v_fshl_i32_svs:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: v_alignbit_b32 v0, s0, v0, 1
+; GFX11-NEXT: s_lshr_b32 s0, s0, 1
+; GFX11-NEXT: s_not_b32 s1, s1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: v_alignbit_b32 v0, s0, v0, s1
+; GFX11-NEXT: ; return to shader part epilog
%result = call i32 @llvm.fshl.i32(i32 %lhs, i32 %rhs, i32 %amt)
%cast.result = bitcast i32 %result to float
ret float %cast.result
@@ -3351,24 +3329,14 @@ define amdgpu_ps float @v_fshl_i32_vss(i32 inreg %lhs, i32 inreg %rhs, i32 inreg
; GFX10-NEXT: v_alignbit_b32 v0, s0, v0, s1
; GFX10-NEXT: ; return to shader part epilog
;
-; GFX11-TRUE16-LABEL: v_fshl_i32_vss:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_not_b32 s2, s2
-; GFX11-TRUE16-NEXT: v_alignbit_b32 v0, s0, s1, 1
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, s2
-; GFX11-TRUE16-NEXT: s_lshr_b32 s0, s0, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_alignbit_b32 v0, s0, v0, v1.l
-; GFX11-TRUE16-NEXT: ; return to shader part epilog
-;
-; GFX11-FAKE16-LABEL: v_fshl_i32_vss:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: v_alignbit_b32 v0, s0, s1, 1
-; GFX11-FAKE16-NEXT: s_lshr_b32 s0, s0, 1
-; GFX11-FAKE16-NEXT: s_not_b32 s1, s2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: v_alignbit_b32 v0, s0, v0, s1
-; GFX11-FAKE16-NEXT: ; return to shader part epilog
+; GFX11-LABEL: v_fshl_i32_vss:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: v_alignbit_b32 v0, s0, s1, 1
+; GFX11-NEXT: s_lshr_b32 s0, s0, 1
+; GFX11-NEXT: s_not_b32 s1, s2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: v_alignbit_b32 v0, s0, v0, s1
+; GFX11-NEXT: ; return to shader part epilog
%result = call i32 @llvm.fshl.i32(i32 %lhs, i32 %rhs, i32 %amt)
%cast.result = bitcast i32 %result to float
ret float %cast.result
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fshr.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fshr.ll
index 11b62d825c965..ee863cc06eb6d 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fshr.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fshr.ll
@@ -3257,17 +3257,10 @@ define amdgpu_ps float @v_fshr_i32_svs(i32 inreg %lhs, i32 %rhs, i32 inreg %amt)
; GFX10-NEXT: v_alignbit_b32 v0, s0, v0, s1
; GFX10-NEXT: ; return to shader part epilog
;
-; GFX11-TRUE16-LABEL: v_fshr_i32_svs:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, s1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_alignbit_b32 v0, s0, v0, v1.l
-; GFX11-TRUE16-NEXT: ; return to shader part epilog
-;
-; GFX11-FAKE16-LABEL: v_fshr_i32_svs:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: v_alignbit_b32 v0, s0, v0, s1
-; GFX11-FAKE16-NEXT: ; return to shader part epilog
+; GFX11-LABEL: v_fshr_i32_svs:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: v_alignbit_b32 v0, s0, v0, s1
+; GFX11-NEXT: ; return to shader part epilog
%result = call i32 @llvm.fshr.i32(i32 %lhs, i32 %rhs, i32 %amt)
%cast.result = bitcast i32 %result to float
ret float %cast.result
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-amdgcn.class.s16.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-amdgcn.class.s16.mir
index b0587821eb116..df9d756bdd0bc 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-amdgcn.class.s16.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-amdgcn.class.s16.mir
@@ -136,8 +136,7 @@ body: |
; GFX11-FOLD-TRUE16-NEXT: {{ $}}
; GFX11-FOLD-TRUE16-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; GFX11-FOLD-TRUE16-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr0
- ; GFX11-FOLD-TRUE16-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[COPY1]]
- ; GFX11-FOLD-TRUE16-NEXT: [[V_CMP_CLASS_F16_t16_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_CLASS_F16_t16_e64 0, [[COPY]].lo16, 0, [[COPY2]].lo16, 0, implicit $exec
+ ; GFX11-FOLD-TRUE16-NEXT: [[V_CMP_CLASS_F16_t16_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_CLASS_F16_t16_e64 0, [[COPY]].lo16, 0, [[COPY1]], 0, implicit $exec
; GFX11-FOLD-TRUE16-NEXT: S_ENDPGM 0, implicit [[V_CMP_CLASS_F16_t16_e64_]]
;
; GFX11-FOLD-FAKE16-LABEL: name: class_f16_vcc_vs
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.alignbyte.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.alignbyte.ll
index de05fe56b8428..db8687edb2642 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.alignbyte.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.alignbyte.ll
@@ -294,9 +294,7 @@ define amdgpu_kernel void @v_alignbyte_b32_2(ptr addrspace(1) %out, ptr addrspac
; GFX11-TRUE16-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-GISEL-NEXT: s_load_b32 s2, s[4:5], 0x3c
; GFX11-TRUE16-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-TRUE16-GISEL-NEXT: v_mov_b32_e32 v2, s2
-; GFX11-TRUE16-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-GISEL-NEXT: v_alignbyte_b32 v0, v1, v0, v2.l
+; GFX11-TRUE16-GISEL-NEXT: v_alignbyte_b32 v0, v1, v0, s2
; GFX11-TRUE16-GISEL-NEXT: v_mov_b32_e32 v1, 0
; GFX11-TRUE16-GISEL-NEXT: global_store_b32 v1, v0, s[0:1]
; GFX11-TRUE16-GISEL-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/true16-fold.mir b/llvm/test/CodeGen/AMDGPU/true16-fold.mir
index 29be6c55e6f30..5a12046abc61c 100644
--- a/llvm/test/CodeGen/AMDGPU/true16-fold.mir
+++ b/llvm/test/CodeGen/AMDGPU/true16-fold.mir
@@ -265,3 +265,65 @@ body: |
$vgpr0 = COPY %5
SI_RETURN implicit $vgpr0
...
+
+---
+name: fold_sreg32_cross_lo16_copy_1
+tracksRegLiveness: true
+registers:
+body: |
+ bb.0:
+ liveins: $sgpr0
+ ; CHECK-LABEL: name: fold_sreg32_cross_lo16_copy_1
+ ; CHECK: liveins: $sgpr0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+ ; CHECK-NEXT: [[V_MAX_F16_t16_e64_:%[0-9]+]]:vgpr_16 = nofpexcept V_MAX_F16_t16_e64 0, [[COPY]], 0, [[COPY]], -1, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0
+ %0:sreg_32 = COPY $sgpr0
+ %1:vgpr_32 = COPY %0:sreg_32
+ %2:vgpr_16 = COPY %1.lo16:vgpr_32
+ %3:vgpr_16 = nofpexcept V_MAX_F16_t16_e64 0, %2, 0, %2, -1, 0, 0, implicit $mode, implicit $exec
+ S_ENDPGM 0
+...
+
+---
+name: fold_sreg32_cross_lo16_copy_2
+tracksRegLiveness: true
+registers:
+body: |
+ bb.0:
+ liveins: $sgpr0
+ ; CHECK-LABEL: name: fold_sreg32_cross_lo16_copy_2
+ ; CHECK: liveins: $sgpr0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[V_MAX_F16_t16_e64_:%[0-9]+]]:vgpr_16 = nofpexcept V_MAX_F16_t16_e64 0, [[DEF]].sub1, 0, [[DEF]].sub1, -1, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0
+ %0:sreg_64 = IMPLICIT_DEF
+ %1:vgpr_32 = COPY %0.sub1:sreg_64
+ %2:vgpr_16 = COPY %1.lo16:vgpr_32
+ %3:vgpr_16 = nofpexcept V_MAX_F16_t16_e64 0, %2, 0, %2, -1, 0, 0, implicit $mode, implicit $exec
+ S_ENDPGM 0
+...
+
+# Should not fold
+---
+name: fold_sreg32_cross_hi16
+tracksRegLiveness: true
+registers:
+body: |
+ bb.0:
+ liveins: $sgpr0
+ ; CHECK-LABEL: name: fold_sreg32_cross_hi16
+ ; CHECK: liveins: $sgpr0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[COPY]]
+ ; CHECK-NEXT: [[V_MAX_F16_t16_e64_:%[0-9]+]]:vgpr_16 = nofpexcept V_MAX_F16_t16_e64 0, [[COPY1]].hi16, 0, [[COPY1]].hi16, -1, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0
+ %0:sreg_32 = COPY $sgpr0
+ %1:vgpr_32 = COPY %0:sreg_32
+ %2:vgpr_16 = COPY %1.hi16:vgpr_32
+ %3:vgpr_16 = nofpexcept V_MAX_F16_t16_e64 0, %2, 0, %2, -1, 0, 0, implicit $mode, implicit $exec
+ S_ENDPGM 0
+...
>From c736246ce1936e43feb04b3fde7a2bd25500d239 Mon Sep 17 00:00:00 2001
From: guochen2 <guochen2 at amd.com>
Date: Sun, 9 Aug 2026 18:44:24 -0400
Subject: [PATCH 2/2] Legalize sgpr16 copy in gisel
---
.../AMDGPU/AMDGPUInstructionSelector.cpp | 45 ++
.../Target/AMDGPU/AMDGPUInstructionSelector.h | 1 +
llvm/test/CodeGen/AMDGPU/GlobalISel/bswap.ll | 6 -
llvm/test/CodeGen/AMDGPU/GlobalISel/fma.ll | 2 +-
llvm/test/CodeGen/AMDGPU/GlobalISel/fmed3.ll | 4 +-
.../AMDGPU/GlobalISel/fp-int-conversions.ll | 4 +-
.../AMDGPU/GlobalISel/insertelement.i8.ll | 6 +-
.../AMDGPU/GlobalISel/load-uniform-in-vgpr.ll | 96 ++-
.../CodeGen/AMDGPU/GlobalISel/load-uniform.ll | 95 +--
.../GlobalISel/merge-values-s16-true16.ll | 9 +-
...alize-amdgcn.ptr.s.buffer.load.subdword.ll | 6 +-
...klegalize-amdgcn.s.buffer.load.subdword.ll | 6 +-
.../AMDGPU/GlobalISel/store-local.128.ll | 64 +-
.../AMDGPU/GlobalISel/store-local.96.ll | 54 +-
.../AMDGPU/GlobalISel/strict_fma.f16.ll | 16 +-
llvm/test/CodeGen/AMDGPU/bitreverse.ll | 83 +--
.../CodeGen/AMDGPU/dagcombine-fmul-sel.ll | 601 ++++++++++++------
llvm/test/CodeGen/AMDGPU/fmaximum.ll | 2 +-
llvm/test/CodeGen/AMDGPU/fmaxnum.ll | 4 +-
llvm/test/CodeGen/AMDGPU/fminimum.ll | 2 +-
llvm/test/CodeGen/AMDGPU/fminnum.ll | 4 +-
llvm/test/CodeGen/AMDGPU/fp_to_sint.ll | 2 +-
llvm/test/CodeGen/AMDGPU/fp_to_uint.ll | 2 +-
llvm/test/CodeGen/AMDGPU/fptrunc.f16.ll | 16 +-
llvm/test/CodeGen/AMDGPU/fptrunc.ll | 222 ++-----
.../test/CodeGen/AMDGPU/llvm.amdgcn.bitop3.ll | 2 +-
.../CodeGen/AMDGPU/llvm.amdgcn.cvt.f16.fp8.ll | 14 +-
.../AMDGPU/llvm.amdgcn.fdot2.bf16.bf16.ll | 2 +-
.../llvm.amdgcn.ptr.s.buffer.load-gfx12.ll | 4 +-
.../AMDGPU/llvm.amdgcn.raw.buffer.load.ll | 2 +-
.../CodeGen/AMDGPU/llvm.amdgcn.reduce.add.ll | 96 +--
.../CodeGen/AMDGPU/llvm.amdgcn.reduce.and.ll | 79 +--
.../CodeGen/AMDGPU/llvm.amdgcn.reduce.max.ll | 78 +--
.../CodeGen/AMDGPU/llvm.amdgcn.reduce.min.ll | 78 +--
.../CodeGen/AMDGPU/llvm.amdgcn.reduce.or.ll | 79 +--
.../CodeGen/AMDGPU/llvm.amdgcn.reduce.sub.ll | 108 ++--
.../CodeGen/AMDGPU/llvm.amdgcn.reduce.umax.ll | 79 +--
.../CodeGen/AMDGPU/llvm.amdgcn.reduce.umin.ll | 79 +--
.../CodeGen/AMDGPU/llvm.amdgcn.reduce.xor.ll | 109 ++--
.../CodeGen/AMDGPU/llvm.amdgcn.rsq.f16.ll | 2 +-
.../test/CodeGen/AMDGPU/llvm.fptrunc.round.ll | 43 +-
llvm/test/CodeGen/AMDGPU/llvm.sqrt.f16.ll | 2 +-
llvm/test/CodeGen/AMDGPU/minimummaximum.ll | 39 +-
llvm/test/CodeGen/AMDGPU/minmax.ll | 105 +--
44 files changed, 1057 insertions(+), 1295 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
index 677312f4f0f52..d9abc7d7d6793 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
@@ -219,6 +219,47 @@ bool AMDGPUInstructionSelector::selectCOPY(MachineInstr &I) const {
continue;
RBI.constrainGenericRegister(MO.getReg(), *RC, *MRI);
}
+
+ // Crossbank s16 copy between sgpr and vgpr
+ // sgpr(s16) = COPY vgpr(s16)
+ // =>
+ // vgpr32 = REG_SEQUENCE vgpr16
+ // sreg32 = readfirstlane vgpr32
+ //
+ // vgpr(s16) = COPY sgpr(s16)
+ // =>
+ // vgpr32 = COPY sreg32
+ // vgpr16 = COPY vgpr32.lo16
+
+ LLT DstTy = MRI->getType(DstReg);
+ bool IsS16Copy = DstTy == LLT::scalar(16);
+ if (Subtarget->useRealTrue16Insts() && IsS16Copy && !SrcReg.isPhysical() &&
+ !DstReg.isPhysical()) {
+ if (isSGPR(DstReg) && isVGPR(SrcReg)) {
+ Register Reg16 = MRI->createVirtualRegister(&AMDGPU::VGPR_16RegClass);
+ Register Reg32 = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass);
+ BuildMI(*BB, &I, DL, TII.get(AMDGPU::IMPLICIT_DEF), Reg16);
+ BuildMI(*BB, &I, DL, TII.get(AMDGPU::REG_SEQUENCE), Reg32)
+ .addReg(SrcReg)
+ .addImm(AMDGPU::lo16)
+ .addReg(Reg16)
+ .addImm(AMDGPU::hi16);
+ BuildMI(*BB, &I, DL, TII.get(AMDGPU::V_READFIRSTLANE_B32), DstReg)
+ .addReg(Reg32);
+ I.eraseFromParent();
+ return true;
+ }
+
+ if (isSGPR(SrcReg) && isVGPR(DstReg)) {
+ Register Reg32 = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass);
+ BuildMI(*BB, &I, DL, TII.get(AMDGPU::COPY), Reg32).addReg(SrcReg);
+ BuildMI(*BB, &I, DL, TII.get(AMDGPU::COPY), DstReg)
+ .addReg(Reg32, {}, AMDGPU::lo16);
+ I.eraseFromParent();
+ return true;
+ }
+ }
+
return true;
}
@@ -3128,6 +3169,10 @@ bool AMDGPUInstructionSelector::isSGPR(Register Reg) const {
return RBI.getRegBank(Reg, *MRI, TRI)->getID() == AMDGPU::SGPRRegBankID;
}
+bool AMDGPUInstructionSelector::isVGPR(Register Reg) const {
+ return RBI.getRegBank(Reg, *MRI, TRI)->getID() == AMDGPU::VGPRRegBankID;
+}
+
bool AMDGPUInstructionSelector::isInstrUniform(const MachineInstr &MI) const {
if (!MI.hasOneMemOperand())
return false;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h
index 1fb7a231a6829..795f04516708c 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h
@@ -70,6 +70,7 @@ class AMDGPUInstructionSelector final : public InstructionSelector {
};
bool isSGPR(Register Reg) const;
+ bool isVGPR(Register Reg) const;
bool isInstrUniform(const MachineInstr &MI) const;
bool isVCC(Register Reg, const MachineRegisterInfo &MRI) const;
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/bswap.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/bswap.ll
index 5738dd30112c5..40360cbaf3adc 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/bswap.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/bswap.ll
@@ -402,12 +402,6 @@ define amdgpu_ps i16 @s_bswap_i16(i16 inreg %src) {
; GFX9-NEXT: v_perm_b32 v0, 0, v0, s0
; GFX9-NEXT: v_readfirstlane_b32 s0, v0
; GFX9-NEXT: ; return to shader part epilog
-;
-; GFX10-LABEL: s_bswap_i16:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: v_perm_b32 v0, 0, s0, 0xc0c0001
-; GFX10-NEXT: v_readfirstlane_b32 s0, v0
-; GFX10-NEXT: ; return to shader part epilog
%bswap = call i16 @llvm.bswap.i16(i16 %src)
ret i16 %bswap
}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fma.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fma.ll
index 7505234fae8d2..e6a93c653cedf 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fma.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fma.ll
@@ -1526,7 +1526,7 @@ define amdgpu_ps half @fma_s16_uniform(half inreg %a, half inreg %b, half inreg
;
; GFX11-TRUE16-LABEL: fma_s16_uniform:
; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, s2
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, s2
; GFX11-TRUE16-NEXT: v_fma_f16 v0.l, s1, s0, v0.l
; GFX11-TRUE16-NEXT: ; return to shader part epilog
;
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fmed3.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fmed3.ll
index 0baba45e541a1..81ddff1dd2a4f 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fmed3.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fmed3.ll
@@ -12,7 +12,7 @@ define amdgpu_ps half @fmed3_s16_uniform(half inreg %a, half inreg %b, half inre
;
; GFX12-LABEL: fmed3_s16_uniform:
; GFX12: ; %bb.0:
-; GFX12-NEXT: v_mov_b16_e32 v0.l, s2
+; GFX12-NEXT: v_mov_b32_e32 v0, s2
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_med3_num_f16 v0.l, s0, s1, v0.l
; GFX12-NEXT: ; return to shader part epilog
@@ -31,7 +31,7 @@ define amdgpu_ps half @fmed3_s16_uniform_salu_use(half inreg %a, half inreg %b,
;
; GFX12-LABEL: fmed3_s16_uniform_salu_use:
; GFX12: ; %bb.0:
-; GFX12-NEXT: v_mov_b16_e32 v0.l, s2
+; GFX12-NEXT: v_mov_b32_e32 v0, s2
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-NEXT: v_med3_num_f16 v0.l, s0, s1, v0.l
; GFX12-NEXT: v_readfirstlane_b32 s0, v0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fp-int-conversions.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fp-int-conversions.ll
index cdc83b0aab25e..7e63a2e5c0834 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fp-int-conversions.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fp-int-conversions.ll
@@ -531,7 +531,7 @@ define amdgpu_ps void @s_uitofp_i32_to_f16(i32 inreg %x, ptr addrspace(1) %out)
; GFX12: ; %bb.0:
; GFX12: s_cvt_f32_u32 s0, s0
; GFX12: s_cvt_f16_f32 s0, s0
-; GFX12: v_mov_b16_e32 v2.l, s0
+; GFX12: v_mov_b32_e32 v2, s0
%result = uitofp i32 %x to half
store half %result, ptr addrspace(1) %out
ret void
@@ -552,7 +552,7 @@ define amdgpu_ps void @s_sitofp_i32_to_f16(i32 inreg %x, ptr addrspace(1) %out)
; GFX12: ; %bb.0:
; GFX12: s_cvt_f32_i32 s0, s0
; GFX12: s_cvt_f16_f32 s0, s0
-; GFX12: v_mov_b16_e32 v2.l, s0
+; GFX12: v_mov_b32_e32 v2, s0
%result = sitofp i32 %x to half
store half %result, ptr addrspace(1) %out
ret void
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/insertelement.i8.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/insertelement.i8.ll
index 620cba152f5a2..373f80379998d 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/insertelement.i8.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/insertelement.i8.ll
@@ -90,10 +90,10 @@ define amdgpu_ps void @insertelement_s_v2i8_s_s(ptr addrspace(4) inreg %ptr, i8
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 0
; GFX11-TRUE16-NEXT: s_mov_b32 m0, s5
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, 0
; GFX11-TRUE16-NEXT: global_load_u16 v0, v0, s[2:3]
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, 0
; GFX11-TRUE16-NEXT: s_and_b32 s1, 0xffff, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_lshr_b32 s1, s1, 8
@@ -103,8 +103,8 @@ define amdgpu_ps void @insertelement_s_v2i8_s_s(ptr addrspace(4) inreg %ptr, i8
; GFX11-TRUE16-NEXT: s_lshl_b32 s1, s1, 8
; GFX11-TRUE16-NEXT: s_or_b32 s0, s0, s1
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
-; GFX11-TRUE16-NEXT: global_store_b16 v[1:2], v0, off
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, s0
+; GFX11-TRUE16-NEXT: global_store_b16 v[0:1], v2, off
; GFX11-TRUE16-NEXT: s_endpgm
;
; GFX11-FAKE16-LABEL: insertelement_s_v2i8_s_s:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/load-uniform-in-vgpr.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/load-uniform-in-vgpr.ll
index 730443a56b003..16cbe0d79c773 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/load-uniform-in-vgpr.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/load-uniform-in-vgpr.ll
@@ -39,7 +39,7 @@ define amdgpu_ps void @load_uniform_P1_i16_b16_gfx12(ptr addrspace(1) inreg %ptr
; GFX11-True16-NEXT: v_readfirstlane_b32 s1, v2
; GFX11-True16-NEXT: s_add_i32 s0, s0, s1
; GFX11-True16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-True16-NEXT: v_mov_b16_e32 v2.l, s0
+; GFX11-True16-NEXT: v_mov_b32_e32 v2, s0
; GFX11-True16-NEXT: global_store_b16 v[0:1], v2, off
; GFX11-True16-NEXT: s_endpgm
;
@@ -69,7 +69,7 @@ define amdgpu_ps void @load_uniform_P1_i16_b16_gfx12(ptr addrspace(1) inreg %ptr
; GFX12-True16-NEXT: v_readfirstlane_b32 s1, v2
; GFX12-True16-NEXT: s_add_co_i32 s0, s0, s1
; GFX12-True16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-True16-NEXT: v_mov_b16_e32 v2.l, s0
+; GFX12-True16-NEXT: v_mov_b32_e32 v2, s0
; GFX12-True16-NEXT: global_store_b16 v[0:1], v2, off
; GFX12-True16-NEXT: s_endpgm
;
@@ -123,7 +123,7 @@ define amdgpu_ps void @load_uniform_P1_i16_b16_gfx11(ptr addrspace(1) inreg %ptr
; GFX11-True16-NEXT: v_readfirstlane_b32 s1, v2
; GFX11-True16-NEXT: s_add_i32 s0, s0, s1
; GFX11-True16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-True16-NEXT: v_mov_b16_e32 v2.l, s0
+; GFX11-True16-NEXT: v_mov_b32_e32 v2, s0
; GFX11-True16-NEXT: global_store_b16 v[0:1], v2, off
; GFX11-True16-NEXT: s_endpgm
;
@@ -152,7 +152,7 @@ define amdgpu_ps void @load_uniform_P1_i16_b16_gfx11(ptr addrspace(1) inreg %ptr
; GFX12-True16-NEXT: s_wait_kmcnt 0x0
; GFX12-True16-NEXT: s_add_co_i32 s0, s2, s0
; GFX12-True16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-True16-NEXT: v_mov_b16_e32 v2.l, s0
+; GFX12-True16-NEXT: v_mov_b32_e32 v2, s0
; GFX12-True16-NEXT: global_store_b16 v[0:1], v2, off
; GFX12-True16-NEXT: s_endpgm
;
@@ -205,7 +205,7 @@ define amdgpu_ps void @load_uniform_P1_i16_anyextending_gfx12(ptr addrspace(1) i
; GFX11-True16-NEXT: v_readfirstlane_b32 s1, v2
; GFX11-True16-NEXT: s_add_i32 s0, s0, s1
; GFX11-True16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-True16-NEXT: v_mov_b16_e32 v2.l, s0
+; GFX11-True16-NEXT: v_mov_b32_e32 v2, s0
; GFX11-True16-NEXT: global_store_b16 v[0:1], v2, off
; GFX11-True16-NEXT: s_endpgm
;
@@ -235,7 +235,7 @@ define amdgpu_ps void @load_uniform_P1_i16_anyextending_gfx12(ptr addrspace(1) i
; GFX12-True16-NEXT: v_readfirstlane_b32 s1, v2
; GFX12-True16-NEXT: s_add_co_i32 s0, s0, s1
; GFX12-True16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-True16-NEXT: v_mov_b16_e32 v2.l, s0
+; GFX12-True16-NEXT: v_mov_b32_e32 v2, s0
; GFX12-True16-NEXT: global_store_b16 v[0:1], v2, off
; GFX12-True16-NEXT: s_endpgm
;
@@ -289,7 +289,7 @@ define amdgpu_ps void @load_uniform_P1_i16_anyextending_gfx11(ptr addrspace(1) i
; GFX11-True16-NEXT: v_readfirstlane_b32 s1, v2
; GFX11-True16-NEXT: s_add_i32 s0, s0, s1
; GFX11-True16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-True16-NEXT: v_mov_b16_e32 v2.l, s0
+; GFX11-True16-NEXT: v_mov_b32_e32 v2, s0
; GFX11-True16-NEXT: global_store_b16 v[0:1], v2, off
; GFX11-True16-NEXT: s_endpgm
;
@@ -318,7 +318,7 @@ define amdgpu_ps void @load_uniform_P1_i16_anyextending_gfx11(ptr addrspace(1) i
; GFX12-True16-NEXT: s_wait_kmcnt 0x0
; GFX12-True16-NEXT: s_add_co_i32 s0, s2, s0
; GFX12-True16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-True16-NEXT: v_mov_b16_e32 v2.l, s0
+; GFX12-True16-NEXT: v_mov_b32_e32 v2, s0
; GFX12-True16-NEXT: global_store_b16 v[0:1], v2, off
; GFX12-True16-NEXT: s_endpgm
;
@@ -1260,7 +1260,7 @@ define amdgpu_ps void @load_uniform_P4_i16_b16_gfx12(ptr addrspace(4) inreg %ptr
; GFX11-True16-NEXT: v_readfirstlane_b32 s1, v2
; GFX11-True16-NEXT: s_add_i32 s0, s0, s1
; GFX11-True16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-True16-NEXT: v_mov_b16_e32 v2.l, s0
+; GFX11-True16-NEXT: v_mov_b32_e32 v2, s0
; GFX11-True16-NEXT: global_store_b16 v[0:1], v2, off
; GFX11-True16-NEXT: s_endpgm
;
@@ -1290,7 +1290,7 @@ define amdgpu_ps void @load_uniform_P4_i16_b16_gfx12(ptr addrspace(4) inreg %ptr
; GFX12-True16-NEXT: s_wait_kmcnt 0x0
; GFX12-True16-NEXT: s_add_co_i32 s0, s1, s0
; GFX12-True16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-True16-NEXT: v_mov_b16_e32 v2.l, s0
+; GFX12-True16-NEXT: v_mov_b32_e32 v2, s0
; GFX12-True16-NEXT: global_store_b16 v[0:1], v2, off
; GFX12-True16-NEXT: s_endpgm
;
@@ -1342,7 +1342,7 @@ define amdgpu_ps void @load_uniform_P4_i16_b16_gfx11(ptr addrspace(4) inreg %ptr
; GFX11-True16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-True16-NEXT: s_add_i32 s0, s1, s0
; GFX11-True16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-True16-NEXT: v_mov_b16_e32 v2.l, s0
+; GFX11-True16-NEXT: v_mov_b32_e32 v2, s0
; GFX11-True16-NEXT: global_store_b16 v[0:1], v2, off
; GFX11-True16-NEXT: s_endpgm
;
@@ -1360,29 +1360,17 @@ define amdgpu_ps void @load_uniform_P4_i16_b16_gfx11(ptr addrspace(4) inreg %ptr
; GFX11-NoTrue16-NEXT: global_store_b16 v[0:1], v2, off
; GFX11-NoTrue16-NEXT: s_endpgm
;
-; GFX12-True16-LABEL: load_uniform_P4_i16_b16_gfx11:
-; GFX12-True16: ; %bb.0:
-; GFX12-True16-NEXT: s_clause 0x1
-; GFX12-True16-NEXT: s_load_u16 s2, s[0:1], 0x0
-; GFX12-True16-NEXT: s_load_u16 s0, s[0:1], 0x0
-; GFX12-True16-NEXT: s_wait_kmcnt 0x0
-; GFX12-True16-NEXT: s_add_co_i32 s0, s2, s0
-; GFX12-True16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-True16-NEXT: v_mov_b16_e32 v2.l, s0
-; GFX12-True16-NEXT: global_store_b16 v[0:1], v2, off
-; GFX12-True16-NEXT: s_endpgm
-;
-; GFX12-NoTrue16-LABEL: load_uniform_P4_i16_b16_gfx11:
-; GFX12-NoTrue16: ; %bb.0:
-; GFX12-NoTrue16-NEXT: s_clause 0x1
-; GFX12-NoTrue16-NEXT: s_load_u16 s2, s[0:1], 0x0
-; GFX12-NoTrue16-NEXT: s_load_u16 s0, s[0:1], 0x0
-; GFX12-NoTrue16-NEXT: s_wait_kmcnt 0x0
-; GFX12-NoTrue16-NEXT: s_add_co_i32 s0, s2, s0
-; GFX12-NoTrue16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NoTrue16-NEXT: v_mov_b32_e32 v2, s0
-; GFX12-NoTrue16-NEXT: global_store_b16 v[0:1], v2, off
-; GFX12-NoTrue16-NEXT: s_endpgm
+; GFX12-LABEL: load_uniform_P4_i16_b16_gfx11:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_clause 0x1
+; GFX12-NEXT: s_load_u16 s2, s[0:1], 0x0
+; GFX12-NEXT: s_load_u16 s0, s[0:1], 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_add_co_i32 s0, s2, s0
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT: v_mov_b32_e32 v2, s0
+; GFX12-NEXT: global_store_b16 v[0:1], v2, off
+; GFX12-NEXT: s_endpgm
%a = load i16, ptr addrspace(4) %ptra
%b = load volatile i16, ptr addrspace(4) %ptra, align 4
%sum = add i16 %a, %b
@@ -1421,7 +1409,7 @@ define amdgpu_ps void @load_uniform_P4_i16_anyextending_gfx12(ptr addrspace(4) i
; GFX11-True16-NEXT: v_readfirstlane_b32 s1, v2
; GFX11-True16-NEXT: s_add_i32 s0, s0, s1
; GFX11-True16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-True16-NEXT: v_mov_b16_e32 v2.l, s0
+; GFX11-True16-NEXT: v_mov_b32_e32 v2, s0
; GFX11-True16-NEXT: global_store_b16 v[0:1], v2, off
; GFX11-True16-NEXT: s_endpgm
;
@@ -1451,7 +1439,7 @@ define amdgpu_ps void @load_uniform_P4_i16_anyextending_gfx12(ptr addrspace(4) i
; GFX12-True16-NEXT: s_wait_kmcnt 0x0
; GFX12-True16-NEXT: s_add_co_i32 s0, s1, s0
; GFX12-True16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-True16-NEXT: v_mov_b16_e32 v2.l, s0
+; GFX12-True16-NEXT: v_mov_b32_e32 v2, s0
; GFX12-True16-NEXT: global_store_b16 v[0:1], v2, off
; GFX12-True16-NEXT: s_endpgm
;
@@ -1503,7 +1491,7 @@ define amdgpu_ps void @load_uniform_P4_i16_anyextending_gfx11(ptr addrspace(4) i
; GFX11-True16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-True16-NEXT: s_add_i32 s0, s1, s0
; GFX11-True16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-True16-NEXT: v_mov_b16_e32 v2.l, s0
+; GFX11-True16-NEXT: v_mov_b32_e32 v2, s0
; GFX11-True16-NEXT: global_store_b16 v[0:1], v2, off
; GFX11-True16-NEXT: s_endpgm
;
@@ -1521,29 +1509,17 @@ define amdgpu_ps void @load_uniform_P4_i16_anyextending_gfx11(ptr addrspace(4) i
; GFX11-NoTrue16-NEXT: global_store_b16 v[0:1], v2, off
; GFX11-NoTrue16-NEXT: s_endpgm
;
-; GFX12-True16-LABEL: load_uniform_P4_i16_anyextending_gfx11:
-; GFX12-True16: ; %bb.0:
-; GFX12-True16-NEXT: s_clause 0x1
-; GFX12-True16-NEXT: s_load_u16 s2, s[0:1], 0x0
-; GFX12-True16-NEXT: s_load_u16 s0, s[0:1], 0x0
-; GFX12-True16-NEXT: s_wait_kmcnt 0x0
-; GFX12-True16-NEXT: s_add_co_i32 s0, s2, s0
-; GFX12-True16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-True16-NEXT: v_mov_b16_e32 v2.l, s0
-; GFX12-True16-NEXT: global_store_b16 v[0:1], v2, off
-; GFX12-True16-NEXT: s_endpgm
-;
-; GFX12-NoTrue16-LABEL: load_uniform_P4_i16_anyextending_gfx11:
-; GFX12-NoTrue16: ; %bb.0:
-; GFX12-NoTrue16-NEXT: s_clause 0x1
-; GFX12-NoTrue16-NEXT: s_load_u16 s2, s[0:1], 0x0
-; GFX12-NoTrue16-NEXT: s_load_u16 s0, s[0:1], 0x0
-; GFX12-NoTrue16-NEXT: s_wait_kmcnt 0x0
-; GFX12-NoTrue16-NEXT: s_add_co_i32 s0, s2, s0
-; GFX12-NoTrue16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NoTrue16-NEXT: v_mov_b32_e32 v2, s0
-; GFX12-NoTrue16-NEXT: global_store_b16 v[0:1], v2, off
-; GFX12-NoTrue16-NEXT: s_endpgm
+; GFX12-LABEL: load_uniform_P4_i16_anyextending_gfx11:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_clause 0x1
+; GFX12-NEXT: s_load_u16 s2, s[0:1], 0x0
+; GFX12-NEXT: s_load_u16 s0, s[0:1], 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_add_co_i32 s0, s2, s0
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT: v_mov_b32_e32 v2, s0
+; GFX12-NEXT: global_store_b16 v[0:1], v2, off
+; GFX12-NEXT: s_endpgm
%a = load i16, ptr addrspace(4) %ptra
%b = load volatile i16, ptr addrspace(4) %ptra, align 4
%sum = add i16 %a, %b
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/load-uniform.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/load-uniform.ll
index 5dd0626feec2c..33e325dc228ad 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/load-uniform.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/load-uniform.ll
@@ -16,21 +16,13 @@ define amdgpu_ps void @load_uniform_P1_i16_gfx12(ptr addrspace(1) inreg %ptra, p
; GFX11-NEXT: global_store_b16 v[0:1], v2, off
; GFX11-NEXT: s_endpgm
;
-; GFX12-True16-LABEL: load_uniform_P1_i16_gfx12:
-; GFX12-True16: ; %bb.0:
-; GFX12-True16-NEXT: s_load_u16 s0, s[0:1], 0x0
-; GFX12-True16-NEXT: s_wait_kmcnt 0x0
-; GFX12-True16-NEXT: v_mov_b16_e32 v2.l, s0
-; GFX12-True16-NEXT: global_store_b16 v[0:1], v2, off
-; GFX12-True16-NEXT: s_endpgm
-;
-; GFX12-NoTrue16-LABEL: load_uniform_P1_i16_gfx12:
-; GFX12-NoTrue16: ; %bb.0:
-; GFX12-NoTrue16-NEXT: s_load_u16 s0, s[0:1], 0x0
-; GFX12-NoTrue16-NEXT: s_wait_kmcnt 0x0
-; GFX12-NoTrue16-NEXT: v_mov_b32_e32 v2, s0
-; GFX12-NoTrue16-NEXT: global_store_b16 v[0:1], v2, off
-; GFX12-NoTrue16-NEXT: s_endpgm
+; GFX12-LABEL: load_uniform_P1_i16_gfx12:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_load_u16 s0, s[0:1], 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_mov_b32_e32 v2, s0
+; GFX12-NEXT: global_store_b16 v[0:1], v2, off
+; GFX12-NEXT: s_endpgm
%a = load i16, ptr addrspace(1) %ptra
store i16 %a, ptr addrspace(1) %out
ret void
@@ -44,25 +36,17 @@ define amdgpu_ps void @load_uniform_P1_i16_align4_widen_mmo_gfx11(ptr addrspace(
; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b32 s0, s[0:1], 0x0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_mov_b16_e32 v2.l, s0
+; GFX11-NEXT: v_mov_b32_e32 v2, s0
; GFX11-NEXT: global_store_b16 v[0:1], v2, off
; GFX11-NEXT: s_endpgm
;
-; GFX12-True16-LABEL: load_uniform_P1_i16_align4_widen_mmo_gfx11:
-; GFX12-True16: ; %bb.0:
-; GFX12-True16-NEXT: s_load_u16 s0, s[0:1], 0x0
-; GFX12-True16-NEXT: s_wait_kmcnt 0x0
-; GFX12-True16-NEXT: v_mov_b16_e32 v2.l, s0
-; GFX12-True16-NEXT: global_store_b16 v[0:1], v2, off
-; GFX12-True16-NEXT: s_endpgm
-;
-; GFX12-NoTrue16-LABEL: load_uniform_P1_i16_align4_widen_mmo_gfx11:
-; GFX12-NoTrue16: ; %bb.0:
-; GFX12-NoTrue16-NEXT: s_load_u16 s0, s[0:1], 0x0
-; GFX12-NoTrue16-NEXT: s_wait_kmcnt 0x0
-; GFX12-NoTrue16-NEXT: v_mov_b32_e32 v2, s0
-; GFX12-NoTrue16-NEXT: global_store_b16 v[0:1], v2, off
-; GFX12-NoTrue16-NEXT: s_endpgm
+; GFX12-LABEL: load_uniform_P1_i16_align4_widen_mmo_gfx11:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_load_u16 s0, s[0:1], 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_mov_b32_e32 v2, s0
+; GFX12-NEXT: global_store_b16 v[0:1], v2, off
+; GFX12-NEXT: s_endpgm
%a = load i16, ptr addrspace(1) %ptra, align 4
store i16 %a, ptr addrspace(1) %out
ret void
@@ -312,21 +296,13 @@ define amdgpu_ps void @load_uniform_P4_i16_gfx12(ptr addrspace(4) inreg %ptra, p
; GFX11-NEXT: global_store_b16 v[0:1], v2, off
; GFX11-NEXT: s_endpgm
;
-; GFX12-True16-LABEL: load_uniform_P4_i16_gfx12:
-; GFX12-True16: ; %bb.0:
-; GFX12-True16-NEXT: s_load_u16 s0, s[0:1], 0x0
-; GFX12-True16-NEXT: s_wait_kmcnt 0x0
-; GFX12-True16-NEXT: v_mov_b16_e32 v2.l, s0
-; GFX12-True16-NEXT: global_store_b16 v[0:1], v2, off
-; GFX12-True16-NEXT: s_endpgm
-;
-; GFX12-NoTrue16-LABEL: load_uniform_P4_i16_gfx12:
-; GFX12-NoTrue16: ; %bb.0:
-; GFX12-NoTrue16-NEXT: s_load_u16 s0, s[0:1], 0x0
-; GFX12-NoTrue16-NEXT: s_wait_kmcnt 0x0
-; GFX12-NoTrue16-NEXT: v_mov_b32_e32 v2, s0
-; GFX12-NoTrue16-NEXT: global_store_b16 v[0:1], v2, off
-; GFX12-NoTrue16-NEXT: s_endpgm
+; GFX12-LABEL: load_uniform_P4_i16_gfx12:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_load_u16 s0, s[0:1], 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_mov_b32_e32 v2, s0
+; GFX12-NEXT: global_store_b16 v[0:1], v2, off
+; GFX12-NEXT: s_endpgm
%a = load i16, ptr addrspace(4) %ptra
store i16 %a, ptr addrspace(1) %out
ret void
@@ -337,25 +313,17 @@ define amdgpu_ps void @load_uniform_P4_i16_align4_widen_mmo_gfx11(ptr addrspace(
; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b32 s0, s[0:1], 0x0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_mov_b16_e32 v2.l, s0
+; GFX11-NEXT: v_mov_b32_e32 v2, s0
; GFX11-NEXT: global_store_b16 v[0:1], v2, off
; GFX11-NEXT: s_endpgm
;
-; GFX12-True16-LABEL: load_uniform_P4_i16_align4_widen_mmo_gfx11:
-; GFX12-True16: ; %bb.0:
-; GFX12-True16-NEXT: s_load_u16 s0, s[0:1], 0x0
-; GFX12-True16-NEXT: s_wait_kmcnt 0x0
-; GFX12-True16-NEXT: v_mov_b16_e32 v2.l, s0
-; GFX12-True16-NEXT: global_store_b16 v[0:1], v2, off
-; GFX12-True16-NEXT: s_endpgm
-;
-; GFX12-NoTrue16-LABEL: load_uniform_P4_i16_align4_widen_mmo_gfx11:
-; GFX12-NoTrue16: ; %bb.0:
-; GFX12-NoTrue16-NEXT: s_load_u16 s0, s[0:1], 0x0
-; GFX12-NoTrue16-NEXT: s_wait_kmcnt 0x0
-; GFX12-NoTrue16-NEXT: v_mov_b32_e32 v2, s0
-; GFX12-NoTrue16-NEXT: global_store_b16 v[0:1], v2, off
-; GFX12-NoTrue16-NEXT: s_endpgm
+; GFX12-LABEL: load_uniform_P4_i16_align4_widen_mmo_gfx11:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_load_u16 s0, s[0:1], 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_mov_b32_e32 v2, s0
+; GFX12-NEXT: global_store_b16 v[0:1], v2, off
+; GFX12-NEXT: s_endpgm
%a = load i16, ptr addrspace(4) %ptra, align 4
store i16 %a, ptr addrspace(1) %out
ret void
@@ -594,3 +562,6 @@ define amdgpu_ps void @load_uniform_P4_v16i32(ptr addrspace(4) inreg %ptra, ptr
store <16 x i32> %a, ptr addrspace(1) %out
ret void
}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; GFX12-NoTrue16: {{.*}}
+; GFX12-True16: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/merge-values-s16-true16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/merge-values-s16-true16.ll
index fc1be777b09a3..9dd8a1f9b5c59 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/merge-values-s16-true16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/merge-values-s16-true16.ll
@@ -29,19 +29,20 @@ define amdgpu_kernel void @store_i136_divergent(ptr %p) {
; GFX1150: ; %bb.0:
; GFX1150-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
; GFX1150-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_and_b32 v0, 0x3ff, v0
-; GFX1150-NEXT: v_mov_b32_e32 v2, 0
; GFX1150-NEXT: v_mov_b32_e32 v6, 0
+; GFX1150-NEXT: v_mov_b32_e32 v2, 0
; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX1150-NEXT: v_lshrrev_b32_e32 v1, 8, v0
; GFX1150-NEXT: v_mov_b16_e32 v0.h, 0
; GFX1150-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX1150-NEXT: v_lshlrev_b16 v4.l, 8, v1.l
-; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX1150-NEXT: v_mov_b16_e32 v1.l, v0.h
+; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1150-NEXT: v_mov_b16_e32 v1.h, v0.h
; GFX1150-NEXT: v_or_b16 v0.l, v0.l, v4.l
; GFX1150-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1150-NEXT: v_dual_mov_b32 v5, s1 :: v_dual_mov_b32 v4, s0
+; GFX1150-NEXT: v_mov_b32_e32 v5, s1
+; GFX1150-NEXT: v_mov_b16_e32 v1.l, v6.l
+; GFX1150-NEXT: v_mov_b32_e32 v4, s0
; GFX1150-NEXT: s_clause 0x1
; GFX1150-NEXT: flat_store_b128 v[4:5], v[0:3]
; GFX1150-NEXT: flat_store_b8 v[4:5], v6 offset:16
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbanklegalize-amdgcn.ptr.s.buffer.load.subdword.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbanklegalize-amdgcn.ptr.s.buffer.load.subdword.ll
index e3f267238dd95..221461c47f56a 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbanklegalize-amdgcn.ptr.s.buffer.load.subdword.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbanklegalize-amdgcn.ptr.s.buffer.load.subdword.ll
@@ -41,11 +41,10 @@ define amdgpu_ps void @ptr_s_buffer_load_i16_sgpr_rsrc_sgpr_offset(ptr addrspace
; GFX12-LABEL: ptr_s_buffer_load_i16_sgpr_rsrc_sgpr_offset:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_buffer_load_u16 s0, s[0:3], s4 offset:0x0
-; GFX12-NEXT: v_mov_b32_e32 v1, 0
; GFX12-NEXT: s_mov_b32 s8, s5
; GFX12-NEXT: s_mov_b32 s9, s6
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX12-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s0
; GFX12-NEXT: global_store_b16 v1, v0, s[8:9]
; GFX12-NEXT: s_endpgm
%val = call i16 @llvm.amdgcn.ptr.s.buffer.load.i16(ptr addrspace(8) %rsrc, i32 %soffset, i32 0), !invariant.load !0
@@ -58,11 +57,10 @@ define amdgpu_ps void @ptr_s_buffer_load_u16_sgpr_rsrc_sgpr_offset(ptr addrspace
; GFX12-LABEL: ptr_s_buffer_load_u16_sgpr_rsrc_sgpr_offset:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_buffer_load_u16 s0, s[0:3], s4 offset:0x0
-; GFX12-NEXT: v_mov_b32_e32 v1, 0
; GFX12-NEXT: s_mov_b32 s8, s5
; GFX12-NEXT: s_mov_b32 s9, s6
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX12-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s0
; GFX12-NEXT: global_store_b16 v1, v0, s[8:9]
; GFX12-NEXT: s_endpgm
%val = call i16 @llvm.amdgcn.ptr.s.buffer.load.u16(ptr addrspace(8) %rsrc, i32 %soffset, i32 0), !invariant.load !0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbanklegalize-amdgcn.s.buffer.load.subdword.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbanklegalize-amdgcn.s.buffer.load.subdword.ll
index 42779b9803ece..e72f9a1d44f91 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbanklegalize-amdgcn.s.buffer.load.subdword.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbanklegalize-amdgcn.s.buffer.load.subdword.ll
@@ -41,11 +41,10 @@ define amdgpu_ps void @s_buffer_load_i16_sgpr_rsrc_sgpr_offset(<4 x i32> inreg %
; GFX12-LABEL: s_buffer_load_i16_sgpr_rsrc_sgpr_offset:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_buffer_load_u16 s0, s[0:3], s4 offset:0x0
-; GFX12-NEXT: v_mov_b32_e32 v1, 0
; GFX12-NEXT: s_mov_b32 s8, s5
; GFX12-NEXT: s_mov_b32 s9, s6
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX12-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s0
; GFX12-NEXT: global_store_b16 v1, v0, s[8:9]
; GFX12-NEXT: s_endpgm
%val = call i16 @llvm.amdgcn.s.buffer.load.i16(<4 x i32> %rsrc, i32 %soffset, i32 0)
@@ -58,11 +57,10 @@ define amdgpu_ps void @s_buffer_load_u16_sgpr_rsrc_sgpr_offset(<4 x i32> inreg %
; GFX12-LABEL: s_buffer_load_u16_sgpr_rsrc_sgpr_offset:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_buffer_load_u16 s0, s[0:3], s4 offset:0x0
-; GFX12-NEXT: v_mov_b32_e32 v1, 0
; GFX12-NEXT: s_mov_b32 s8, s5
; GFX12-NEXT: s_mov_b32 s9, s6
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX12-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s0
; GFX12-NEXT: global_store_b16 v1, v0, s[8:9]
; GFX12-NEXT: s_endpgm
%val = call i16 @llvm.amdgcn.s.buffer.load.u16(<4 x i32> %rsrc, i32 %soffset, i32 0)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/store-local.128.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/store-local.128.ll
index 439d9f7912b30..2587d868e8676 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/store-local.128.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/store-local.128.ll
@@ -239,53 +239,47 @@ define amdgpu_kernel void @store_lds_v4i32_align1(ptr addrspace(3) %out, <4 x i3
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_and_b32 s6, 0xffff, s0
; GFX11-NEXT: s_lshr_b32 s5, s0, 16
-; GFX11-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s4
; GFX11-NEXT: s_lshr_b32 s0, s0, 24
+; GFX11-NEXT: s_lshr_b32 s4, s1, 16
; GFX11-NEXT: s_and_b32 s7, 0xffff, s1
; GFX11-NEXT: s_lshr_b32 s6, s6, 8
-; GFX11-NEXT: v_mov_b32_e32 v5, s4
-; GFX11-NEXT: v_mov_b16_e32 v2.l, s0
-; GFX11-NEXT: s_lshr_b32 s0, s7, 8
-; GFX11-NEXT: v_mov_b16_e32 v3.l, s6
-; GFX11-NEXT: v_mov_b16_e32 v1.h, s5
-; GFX11-NEXT: s_lshr_b32 s4, s1, 16
-; GFX11-NEXT: v_mov_b16_e32 v0.h, s1
+; GFX11-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v3, s2
; GFX11-NEXT: s_lshr_b32 s1, s1, 24
+; GFX11-NEXT: v_dual_mov_b32 v4, s5 :: v_dual_mov_b32 v5, s0
+; GFX11-NEXT: s_lshr_b32 s0, s7, 8
+; GFX11-NEXT: v_dual_mov_b32 v6, s4 :: v_dual_mov_b32 v7, s6
; GFX11-NEXT: s_and_b32 s9, 0xffff, s2
-; GFX11-NEXT: v_mov_b16_e32 v4.l, s0
+; GFX11-NEXT: v_dual_mov_b32 v8, s1 :: v_dual_mov_b32 v9, s0
; GFX11-NEXT: s_lshr_b32 s0, s2, 24
-; GFX11-NEXT: v_mov_b16_e32 v2.h, s4
-; GFX11-NEXT: v_mov_b16_e32 v3.h, s1
; GFX11-NEXT: s_lshr_b32 s1, s9, 8
-; GFX11-NEXT: ds_store_b8 v5, v0
-; GFX11-NEXT: ds_store_b8 v5, v3 offset:1
-; GFX11-NEXT: ds_store_b8_d16_hi v5, v1 offset:2
-; GFX11-NEXT: ds_store_b8 v5, v2 offset:3
-; GFX11-NEXT: ds_store_b8_d16_hi v5, v0 offset:4
-; GFX11-NEXT: ds_store_b8 v5, v4 offset:5
-; GFX11-NEXT: ds_store_b8_d16_hi v5, v2 offset:6
-; GFX11-NEXT: ds_store_b8_d16_hi v5, v3 offset:7
-; GFX11-NEXT: v_mov_b16_e32 v1.h, s0
+; GFX11-NEXT: ds_store_b8 v1, v0
+; GFX11-NEXT: ds_store_b8 v1, v7 offset:1
+; GFX11-NEXT: ds_store_b8 v1, v4 offset:2
+; GFX11-NEXT: ds_store_b8 v1, v5 offset:3
+; GFX11-NEXT: ds_store_b8 v1, v2 offset:4
+; GFX11-NEXT: ds_store_b8 v1, v9 offset:5
+; GFX11-NEXT: ds_store_b8 v1, v6 offset:6
+; GFX11-NEXT: ds_store_b8 v1, v8 offset:7
+; GFX11-NEXT: v_mov_b32_e32 v4, s0
; GFX11-NEXT: s_and_b32 s0, 0xffff, s3
; GFX11-NEXT: s_lshr_b32 s8, s2, 16
-; GFX11-NEXT: v_mov_b16_e32 v1.l, s2
-; GFX11-NEXT: v_mov_b16_e32 v0.l, s1
+; GFX11-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v5, s3
; GFX11-NEXT: s_lshr_b32 s0, s0, 8
-; GFX11-NEXT: v_mov_b16_e32 v0.h, s8
; GFX11-NEXT: s_lshr_b32 s1, s3, 16
-; GFX11-NEXT: v_mov_b16_e32 v2.h, s0
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: v_dual_mov_b32 v2, s8 :: v_dual_mov_b32 v7, s1
+; GFX11-NEXT: v_mov_b32_e32 v6, s0
; GFX11-NEXT: s_lshr_b32 s0, s3, 24
-; GFX11-NEXT: v_mov_b16_e32 v2.l, s3
-; GFX11-NEXT: v_mov_b16_e32 v3.l, s1
-; GFX11-NEXT: v_mov_b16_e32 v3.h, s0
-; GFX11-NEXT: ds_store_b8 v5, v1 offset:8
-; GFX11-NEXT: ds_store_b8 v5, v0 offset:9
-; GFX11-NEXT: ds_store_b8_d16_hi v5, v0 offset:10
-; GFX11-NEXT: ds_store_b8_d16_hi v5, v1 offset:11
-; GFX11-NEXT: ds_store_b8 v5, v2 offset:12
-; GFX11-NEXT: ds_store_b8_d16_hi v5, v2 offset:13
-; GFX11-NEXT: ds_store_b8 v5, v3 offset:14
-; GFX11-NEXT: ds_store_b8_d16_hi v5, v3 offset:15
+; GFX11-NEXT: v_mov_b32_e32 v8, s0
+; GFX11-NEXT: ds_store_b8 v1, v3 offset:8
+; GFX11-NEXT: ds_store_b8 v1, v0 offset:9
+; GFX11-NEXT: ds_store_b8 v1, v2 offset:10
+; GFX11-NEXT: ds_store_b8 v1, v4 offset:11
+; GFX11-NEXT: ds_store_b8 v1, v5 offset:12
+; GFX11-NEXT: ds_store_b8 v1, v6 offset:13
+; GFX11-NEXT: ds_store_b8 v1, v7 offset:14
+; GFX11-NEXT: ds_store_b8 v1, v8 offset:15
; GFX11-NEXT: s_endpgm
store <4 x i32> %x, ptr addrspace(3) %out, align 1
ret void
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/store-local.96.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/store-local.96.ll
index 3b6106b915e03..43d39edc824d4 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/store-local.96.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/store-local.96.ll
@@ -60,7 +60,6 @@ define amdgpu_kernel void @store_lds_v3i32(ptr addrspace(3) %out, <3 x i32> %x)
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-NEXT: ds_store_b96 v3, v[0:2]
; GFX11-NEXT: s_endpgm
-;
; GFX6-LABEL: store_lds_v3i32:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x4
@@ -223,45 +222,38 @@ define amdgpu_kernel void @store_lds_v3i32_align1(ptr addrspace(3) %out, <3 x i3
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_load_b32 s3, s[4:5], 0x0
; GFX11-NEXT: s_and_b32 s5, 0xffff, s0
-; GFX11-NEXT: s_lshr_b32 s4, s0, 16
+; GFX11-NEXT: s_lshr_b32 s7, s2, 16
; GFX11-NEXT: s_lshr_b32 s5, s5, 8
-; GFX11-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX11-NEXT: s_lshr_b32 s4, s0, 16
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v6, s3
+; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s3
; GFX11-NEXT: s_lshr_b32 s0, s0, 24
; GFX11-NEXT: s_lshr_b32 s3, s1, 16
; GFX11-NEXT: s_and_b32 s6, 0xffff, s1
-; GFX11-NEXT: v_mov_b16_e32 v0.h, s1
+; GFX11-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v3, s2
; GFX11-NEXT: s_lshr_b32 s1, s1, 24
; GFX11-NEXT: s_and_b32 s8, 0xffff, s2
-; GFX11-NEXT: v_mov_b16_e32 v4.l, s5
-; GFX11-NEXT: v_mov_b16_e32 v1.h, s4
-; GFX11-NEXT: s_lshr_b32 s7, s2, 16
-; GFX11-NEXT: v_mov_b16_e32 v1.l, s2
+; GFX11-NEXT: v_dual_mov_b32 v8, s7 :: v_dual_mov_b32 v9, s5
+; GFX11-NEXT: v_dual_mov_b32 v4, s4 :: v_dual_mov_b32 v5, s0
; GFX11-NEXT: s_lshr_b32 s2, s2, 24
-; GFX11-NEXT: v_mov_b16_e32 v2.l, s0
; GFX11-NEXT: s_lshr_b32 s0, s6, 8
-; GFX11-NEXT: v_mov_b16_e32 v3.l, s1
+; GFX11-NEXT: v_dual_mov_b32 v6, s3 :: v_dual_mov_b32 v7, s1
; GFX11-NEXT: s_lshr_b32 s1, s8, 8
-; GFX11-NEXT: v_mov_b16_e32 v2.h, s3
-; GFX11-NEXT: v_mov_b16_e32 v3.h, s7
-; GFX11-NEXT: v_mov_b16_e32 v4.h, s2
-; GFX11-NEXT: v_mov_b16_e32 v5.l, s0
-; GFX11-NEXT: v_mov_b16_e32 v5.h, s1
-; GFX11-NEXT: ds_store_b8 v6, v0
-; GFX11-NEXT: ds_store_b8 v6, v4 offset:1
-; GFX11-NEXT: ds_store_b8_d16_hi v6, v1 offset:2
-; GFX11-NEXT: ds_store_b8 v6, v2 offset:3
-; GFX11-NEXT: ds_store_b8_d16_hi v6, v0 offset:4
-; GFX11-NEXT: ds_store_b8 v6, v5 offset:5
-; GFX11-NEXT: ds_store_b8_d16_hi v6, v2 offset:6
-; GFX11-NEXT: ds_store_b8 v6, v3 offset:7
-; GFX11-NEXT: ds_store_b8 v6, v1 offset:8
-; GFX11-NEXT: ds_store_b8_d16_hi v6, v5 offset:9
-; GFX11-NEXT: ds_store_b8_d16_hi v6, v3 offset:10
-; GFX11-NEXT: ds_store_b8_d16_hi v6, v4 offset:11
+; GFX11-NEXT: v_dual_mov_b32 v10, s2 :: v_dual_mov_b32 v11, s0
+; GFX11-NEXT: v_mov_b32_e32 v12, s1
+; GFX11-NEXT: ds_store_b8 v1, v0
+; GFX11-NEXT: ds_store_b8 v1, v9 offset:1
+; GFX11-NEXT: ds_store_b8 v1, v4 offset:2
+; GFX11-NEXT: ds_store_b8 v1, v5 offset:3
+; GFX11-NEXT: ds_store_b8 v1, v2 offset:4
+; GFX11-NEXT: ds_store_b8 v1, v11 offset:5
+; GFX11-NEXT: ds_store_b8 v1, v6 offset:6
+; GFX11-NEXT: ds_store_b8 v1, v7 offset:7
+; GFX11-NEXT: ds_store_b8 v1, v3 offset:8
+; GFX11-NEXT: ds_store_b8 v1, v12 offset:9
+; GFX11-NEXT: ds_store_b8 v1, v8 offset:10
+; GFX11-NEXT: ds_store_b8 v1, v10 offset:11
; GFX11-NEXT: s_endpgm
-;
; GFX6-LABEL: store_lds_v3i32_align1:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x4
@@ -426,7 +418,6 @@ define amdgpu_kernel void @store_lds_v3i32_align2(ptr addrspace(3) %out, <3 x i3
; GFX11-NEXT: ds_store_b16 v1, v4 offset:8
; GFX11-NEXT: ds_store_b16 v1, v6 offset:10
; GFX11-NEXT: s_endpgm
-;
; GFX6-LABEL: store_lds_v3i32_align2:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x4
@@ -522,7 +513,6 @@ define amdgpu_kernel void @store_lds_v3i32_align4(ptr addrspace(3) %out, <3 x i3
; GFX11-NEXT: ds_store_2addr_b32 v1, v0, v2 offset1:1
; GFX11-NEXT: ds_store_b32 v1, v3 offset:8
; GFX11-NEXT: s_endpgm
-;
; GFX6-LABEL: store_lds_v3i32_align4:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_load_dword s6, s[4:5], 0x0
@@ -597,7 +587,6 @@ define amdgpu_kernel void @store_lds_v3i32_align8(ptr addrspace(3) %out, <3 x i3
; GFX11-NEXT: ds_store_2addr_b32 v1, v0, v2 offset1:1
; GFX11-NEXT: ds_store_b32 v1, v3 offset:8
; GFX11-NEXT: s_endpgm
-;
; GFX6-LABEL: store_lds_v3i32_align8:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x4
@@ -671,7 +660,6 @@ define amdgpu_kernel void @store_lds_v3i32_align16(ptr addrspace(3) %out, <3 x i
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-NEXT: ds_store_b96 v3, v[0:2]
; GFX11-NEXT: s_endpgm
-;
; GFX6-LABEL: store_lds_v3i32_align16:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x4
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/strict_fma.f16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/strict_fma.f16.ll
index 893cb7f237055..2b26bfd7f7b09 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/strict_fma.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/strict_fma.f16.ll
@@ -39,7 +39,7 @@ define void @v_constained_fma_f16_fpexcept_strict_uni(half inreg %x, half inreg
; GFX11-LABEL: v_constained_fma_f16_fpexcept_strict_uni:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_mov_b16_e32 v2.l, s2
+; GFX11-NEXT: v_mov_b32_e32 v2, s2
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_fma_f16 v2.l, s0, s1, v2.l
; GFX11-NEXT: global_store_b16 v[0:1], v2, off
@@ -55,7 +55,7 @@ define void @v_constained_fma_f16_fpexcept_strict_uni(half inreg %x, half inreg
; GFX12-NEXT: s_fmac_f16 s2, s0, s1
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_2)
-; GFX12-NEXT: v_mov_b16_e32 v2.l, s2
+; GFX12-NEXT: v_mov_b32_e32 v2, s2
; GFX12-NEXT: global_store_b16 v[0:1], v2, off
; GFX12-NEXT: s_setpc_b64 s[30:31]
%val = call half @llvm.experimental.constrained.fma.f16(half %x, half %y, half %z, metadata !"round.tonearest", metadata !"fpexcept.strict")
@@ -511,7 +511,7 @@ define void @v_constained_fma_f16_fpexcept_strict_fneg_uni(half inreg %x, half i
; GFX11-LABEL: v_constained_fma_f16_fpexcept_strict_fneg_uni:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_mov_b16_e32 v2.l, s1
+; GFX11-NEXT: v_mov_b32_e32 v2, s1
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_fma_f16 v2.l, s0, v2.l, -s2
; GFX11-NEXT: global_store_b16 v[0:1], v2, off
@@ -529,7 +529,7 @@ define void @v_constained_fma_f16_fpexcept_strict_fneg_uni(half inreg %x, half i
; GFX12-NEXT: s_fmac_f16 s2, s0, s1
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_2)
-; GFX12-NEXT: v_mov_b16_e32 v2.l, s2
+; GFX12-NEXT: v_mov_b32_e32 v2, s2
; GFX12-NEXT: global_store_b16 v[0:1], v2, off
; GFX12-NEXT: s_setpc_b64 s[30:31]
%neg.z = fneg half %z
@@ -622,7 +622,7 @@ define void @v_constained_fma_f16_fpexcept_strict_fneg_fneg_uni(half inreg %x, h
; GFX11-LABEL: v_constained_fma_f16_fpexcept_strict_fneg_fneg_uni:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_mov_b16_e32 v2.l, s2
+; GFX11-NEXT: v_mov_b32_e32 v2, s2
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_fma_f16 v2.l, -s0, -s1, v2.l
; GFX11-NEXT: global_store_b16 v[0:1], v2, off
@@ -641,7 +641,7 @@ define void @v_constained_fma_f16_fpexcept_strict_fneg_fneg_uni(half inreg %x, h
; GFX12-NEXT: s_fmac_f16 s2, s0, s1
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_2)
-; GFX12-NEXT: v_mov_b16_e32 v2.l, s2
+; GFX12-NEXT: v_mov_b32_e32 v2, s2
; GFX12-NEXT: global_store_b16 v[0:1], v2, off
; GFX12-NEXT: s_setpc_b64 s[30:31]
%neg.x = fneg half %x
@@ -736,7 +736,7 @@ define void @v_constained_fma_f16_fpexcept_strict_fabs_fabs_uni(half inreg %x, h
; GFX11-LABEL: v_constained_fma_f16_fpexcept_strict_fabs_fabs_uni:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_mov_b16_e32 v2.l, s2
+; GFX11-NEXT: v_mov_b32_e32 v2, s2
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_fma_f16 v2.l, |s0|, |s1|, v2.l
; GFX11-NEXT: global_store_b16 v[0:1], v2, off
@@ -755,7 +755,7 @@ define void @v_constained_fma_f16_fpexcept_strict_fabs_fabs_uni(half inreg %x, h
; GFX12-NEXT: s_fmac_f16 s2, s0, s1
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_2)
-; GFX12-NEXT: v_mov_b16_e32 v2.l, s2
+; GFX12-NEXT: v_mov_b32_e32 v2, s2
; GFX12-NEXT: global_store_b16 v[0:1], v2, off
; GFX12-NEXT: s_setpc_b64 s[30:31]
%neg.x = call half @llvm.fabs.f16(half %x) #0
diff --git a/llvm/test/CodeGen/AMDGPU/bitreverse.ll b/llvm/test/CodeGen/AMDGPU/bitreverse.ll
index a0eb268fe6d51..5554bbe44b4f1 100644
--- a/llvm/test/CodeGen/AMDGPU/bitreverse.ll
+++ b/llvm/test/CodeGen/AMDGPU/bitreverse.ll
@@ -72,33 +72,19 @@ define amdgpu_kernel void @s_brev_i16(ptr addrspace(1) noalias %out, i16 %val) #
; GFX11-FLAT-NEXT: global_store_d16_hi_b16 v0, v1, s[0:1]
; GFX11-FLAT-NEXT: s_endpgm
;
-; GFX11-GISEL-TRUE16-LABEL: s_brev_i16:
-; GFX11-GISEL-TRUE16: ; %bb.0:
-; GFX11-GISEL-TRUE16-NEXT: s_clause 0x1
-; GFX11-GISEL-TRUE16-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-GISEL-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX11-GISEL-TRUE16-NEXT: v_mov_b32_e32 v1, 0
-; GFX11-GISEL-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-GISEL-TRUE16-NEXT: s_brev_b32 s2, s2
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-GISEL-TRUE16-NEXT: s_lshr_b32 s2, s2, 16
-; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.l, s2
-; GFX11-GISEL-TRUE16-NEXT: global_store_b16 v1, v0, s[0:1]
-; GFX11-GISEL-TRUE16-NEXT: s_endpgm
-;
-; GFX11-GISEL-FAKE16-LABEL: s_brev_i16:
-; GFX11-GISEL-FAKE16: ; %bb.0:
-; GFX11-GISEL-FAKE16-NEXT: s_clause 0x1
-; GFX11-GISEL-FAKE16-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-GISEL-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX11-GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, 0
-; GFX11-GISEL-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-GISEL-FAKE16-NEXT: s_brev_b32 s2, s2
-; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-GISEL-FAKE16-NEXT: s_lshr_b32 s2, s2, 16
-; GFX11-GISEL-FAKE16-NEXT: v_mov_b32_e32 v0, s2
-; GFX11-GISEL-FAKE16-NEXT: global_store_b16 v1, v0, s[0:1]
-; GFX11-GISEL-FAKE16-NEXT: s_endpgm
+; GFX11-GISEL-LABEL: s_brev_i16:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: s_clause 0x1
+; GFX11-GISEL-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GFX11-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: s_brev_b32 s2, s2
+; GFX11-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-GISEL-NEXT: s_lshr_b32 s2, s2, 16
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX11-GISEL-NEXT: global_store_b16 v1, v0, s[0:1]
+; GFX11-GISEL-NEXT: s_endpgm
%brev = call i16 @llvm.bitreverse.i16(i16 %val) #1
store i16 %brev, ptr addrspace(1) %out
ret void
@@ -175,35 +161,20 @@ define amdgpu_kernel void @v_brev_i16(ptr addrspace(1) noalias %out, ptr addrspa
; GFX11-FLAT-NEXT: global_store_d16_hi_b16 v1, v0, s[0:1]
; GFX11-FLAT-NEXT: s_endpgm
;
-; GFX11-GISEL-TRUE16-LABEL: v_brev_i16:
-; GFX11-GISEL-TRUE16: ; %bb.0:
-; GFX11-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-GISEL-TRUE16-NEXT: v_mov_b32_e32 v1, 0
-; GFX11-GISEL-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-GISEL-TRUE16-NEXT: global_load_u16 v0, v1, s[2:3]
-; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-GISEL-TRUE16-NEXT: v_readfirstlane_b32 s2, v0
-; GFX11-GISEL-TRUE16-NEXT: s_brev_b32 s2, s2
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-GISEL-TRUE16-NEXT: s_lshr_b32 s2, s2, 16
-; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.l, s2
-; GFX11-GISEL-TRUE16-NEXT: global_store_b16 v1, v0, s[0:1]
-; GFX11-GISEL-TRUE16-NEXT: s_endpgm
-;
-; GFX11-GISEL-FAKE16-LABEL: v_brev_i16:
-; GFX11-GISEL-FAKE16: ; %bb.0:
-; GFX11-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-GISEL-FAKE16-NEXT: v_mov_b32_e32 v0, 0
-; GFX11-GISEL-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-GISEL-FAKE16-NEXT: global_load_u16 v1, v0, s[2:3]
-; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-GISEL-FAKE16-NEXT: v_readfirstlane_b32 s2, v1
-; GFX11-GISEL-FAKE16-NEXT: s_brev_b32 s2, s2
-; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-GISEL-FAKE16-NEXT: s_lshr_b32 s2, s2, 16
-; GFX11-GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, s2
-; GFX11-GISEL-FAKE16-NEXT: global_store_b16 v0, v1, s[0:1]
-; GFX11-GISEL-FAKE16-NEXT: s_endpgm
+; GFX11-GISEL-LABEL: v_brev_i16:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: global_load_u16 v1, v0, s[2:3]
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-NEXT: v_readfirstlane_b32 s2, v1
+; GFX11-GISEL-NEXT: s_brev_b32 s2, s2
+; GFX11-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-GISEL-NEXT: s_lshr_b32 s2, s2, 16
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v1, s2
+; GFX11-GISEL-NEXT: global_store_b16 v0, v1, s[0:1]
+; GFX11-GISEL-NEXT: s_endpgm
%val = load i16, ptr addrspace(1) %valptr
%brev = call i16 @llvm.bitreverse.i16(i16 %val) #1
store i16 %brev, ptr addrspace(1) %out
diff --git a/llvm/test/CodeGen/AMDGPU/dagcombine-fmul-sel.ll b/llvm/test/CodeGen/AMDGPU/dagcombine-fmul-sel.ll
index e1c26a02f9a5d..c9de2001e024a 100644
--- a/llvm/test/CodeGen/AMDGPU/dagcombine-fmul-sel.ll
+++ b/llvm/test/CodeGen/AMDGPU/dagcombine-fmul-sel.ll
@@ -2875,25 +2875,46 @@ define bfloat @fmul_select_bf16_test1(bfloat %x, i32 %bool.arg1, i32 %bool.arg2)
; GFX11-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-GISEL-LABEL: fmul_select_bf16_test1:
-; GFX11-GISEL: ; %bb.0:
-; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-NEXT: v_dual_mov_b32 v3, 0x3f80 :: v_dual_lshlrev_b32 v0, 16, v0
-; GFX11-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-NEXT: v_cndmask_b32_e64 v1, v3, 0x4000, vcc_lo
-; GFX11-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-NEXT: v_mul_f32_e32 v0, v0, v1
-; GFX11-GISEL-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX11-GISEL-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX11-GISEL-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11-GISEL-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
+; GFX11-GISEL-TRUE16-LABEL: fmul_select_bf16_test1:
+; GFX11-GISEL-TRUE16: ; %bb.0:
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: v_dual_mov_b32 v3, 0x4000 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX11-GISEL-TRUE16-NEXT: v_mov_b32_e32 v4, 0x3f80
+; GFX11-GISEL-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b32_e32 v1, v4, v3, vcc_lo
+; GFX11-GISEL-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-TRUE16-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX11-GISEL-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX11-GISEL-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX11-GISEL-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-GISEL-FAKE16-LABEL: fmul_select_bf16_test1:
+; GFX11-GISEL-FAKE16: ; %bb.0:
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: v_dual_mov_b32 v3, 0x3f80 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX11-GISEL-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e64 v1, v3, 0x4000, vcc_lo
+; GFX11-GISEL-FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-FAKE16-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX11-GISEL-FAKE16-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX11-GISEL-FAKE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX11-GISEL-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-FAKE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
%bool = icmp eq i32 %bool.arg1, %bool.arg2
%y = select i1 %bool, bfloat 2.000000e+00, bfloat 1.000000e+00
%ldexp = fmul bfloat %x, %y
@@ -3041,25 +3062,46 @@ define bfloat @fmul_select_bf16_test2(bfloat %x, i32 %bool.arg1, i32 %bool.arg2)
; GFX11-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-GISEL-LABEL: fmul_select_bf16_test2:
-; GFX11-GISEL: ; %bb.0:
-; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-NEXT: v_dual_mov_b32 v3, 0x3f80 :: v_dual_lshlrev_b32 v0, 16, v0
-; GFX11-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-NEXT: v_cndmask_b32_e64 v1, v3, 0x3f00, vcc_lo
-; GFX11-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-NEXT: v_mul_f32_e32 v0, v0, v1
-; GFX11-GISEL-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX11-GISEL-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX11-GISEL-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11-GISEL-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
+; GFX11-GISEL-TRUE16-LABEL: fmul_select_bf16_test2:
+; GFX11-GISEL-TRUE16: ; %bb.0:
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: v_dual_mov_b32 v3, 0x3f00 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX11-GISEL-TRUE16-NEXT: v_mov_b32_e32 v4, 0x3f80
+; GFX11-GISEL-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b32_e32 v1, v4, v3, vcc_lo
+; GFX11-GISEL-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-TRUE16-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX11-GISEL-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX11-GISEL-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX11-GISEL-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-GISEL-FAKE16-LABEL: fmul_select_bf16_test2:
+; GFX11-GISEL-FAKE16: ; %bb.0:
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: v_dual_mov_b32 v3, 0x3f80 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX11-GISEL-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e64 v1, v3, 0x3f00, vcc_lo
+; GFX11-GISEL-FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-FAKE16-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX11-GISEL-FAKE16-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX11-GISEL-FAKE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX11-GISEL-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-FAKE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
%bool = icmp eq i32 %bool.arg1, %bool.arg2
%y = select i1 %bool, bfloat 5.000000e-01, bfloat 1.000000e+00
%ldexp = fmul bfloat %x, %y
@@ -3300,36 +3342,38 @@ define <2 x bfloat> @fmul_select_v2bf16_test3(<2 x bfloat> %x, <2 x i32> %bool.a
; GFX11-GISEL-TRUE16-LABEL: fmul_select_v2bf16_test3:
; GFX11-GISEL-TRUE16: ; %bb.0:
; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-TRUE16-NEXT: v_mov_b32_e32 v5, 0x3f80
+; GFX11-GISEL-TRUE16-NEXT: v_mov_b32_e32 v5, 0x4000
+; GFX11-GISEL-TRUE16-NEXT: v_mov_b32_e32 v6, 0x3f80
; GFX11-GISEL-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v4
; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b32_e64 v2, v5, 0x4000, vcc_lo
+; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b32_e32 v2, v6, v5, vcc_lo
; GFX11-GISEL-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v3
; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.h
; GFX11-GISEL-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX11-GISEL-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b32_e64 v1, v5, 0x4000, vcc_lo
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-GISEL-TRUE16-NEXT: v_dual_mul_f32 v0, v0, v1 :: v_dual_lshlrev_b32 v3, 16, v3
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-GISEL-TRUE16-NEXT: v_dual_cndmask_b32 v1, v6, v5 :: v_dual_mul_f32 v2, v3, v2
; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-GISEL-TRUE16-NEXT: v_mul_f32_e32 v2, v3, v2
-; GFX11-GISEL-TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-GISEL-TRUE16-NEXT: v_bfe_u32 v1, v2, 16, 1
+; GFX11-GISEL-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX11-GISEL-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v2
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-GISEL-TRUE16-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX11-GISEL-TRUE16-NEXT: v_bfe_u32 v1, v2, 16, 1
; GFX11-GISEL-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v2
+; GFX11-GISEL-TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-GISEL-TRUE16-NEXT: v_add3_u32 v1, v1, v2, 0x7fff
; GFX11-GISEL-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
; GFX11-GISEL-TRUE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
-; GFX11-GISEL-TRUE16-NEXT: v_add3_u32 v1, v1, v2, 0x7fff
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc_lo
; GFX11-GISEL-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
-; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-GISEL-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
; GFX11-GISEL-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
; GFX11-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -3605,36 +3649,38 @@ define <2 x bfloat> @fmul_select_v2bf16_test4(<2 x bfloat> %x, <2 x i32> %bool.a
; GFX11-GISEL-TRUE16-LABEL: fmul_select_v2bf16_test4:
; GFX11-GISEL-TRUE16: ; %bb.0:
; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-TRUE16-NEXT: v_mov_b32_e32 v5, 0x3f80
+; GFX11-GISEL-TRUE16-NEXT: v_mov_b32_e32 v5, 0x3f00
+; GFX11-GISEL-TRUE16-NEXT: v_mov_b32_e32 v6, 0x3f80
; GFX11-GISEL-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v4
; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b32_e64 v2, v5, 0x3f00, vcc_lo
+; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b32_e32 v2, v6, v5, vcc_lo
; GFX11-GISEL-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v3
; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.h
; GFX11-GISEL-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX11-GISEL-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b32_e64 v1, v5, 0x3f00, vcc_lo
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-GISEL-TRUE16-NEXT: v_dual_mul_f32 v0, v0, v1 :: v_dual_lshlrev_b32 v3, 16, v3
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-GISEL-TRUE16-NEXT: v_dual_cndmask_b32 v1, v6, v5 :: v_dual_mul_f32 v2, v3, v2
; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-GISEL-TRUE16-NEXT: v_mul_f32_e32 v2, v3, v2
-; GFX11-GISEL-TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-GISEL-TRUE16-NEXT: v_bfe_u32 v1, v2, 16, 1
+; GFX11-GISEL-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX11-GISEL-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v2
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-GISEL-TRUE16-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX11-GISEL-TRUE16-NEXT: v_bfe_u32 v1, v2, 16, 1
; GFX11-GISEL-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v2
+; GFX11-GISEL-TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-GISEL-TRUE16-NEXT: v_add3_u32 v1, v1, v2, 0x7fff
; GFX11-GISEL-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
; GFX11-GISEL-TRUE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
-; GFX11-GISEL-TRUE16-NEXT: v_add3_u32 v1, v1, v2, 0x7fff
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc_lo
; GFX11-GISEL-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
-; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-GISEL-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
; GFX11-GISEL-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
; GFX11-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -3818,25 +3864,46 @@ define bfloat @fmul_select_bf16_test5(bfloat %x, i32 %bool.arg1, i32 %bool.arg2)
; GFX11-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-GISEL-LABEL: fmul_select_bf16_test5:
-; GFX11-GISEL: ; %bb.0:
-; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-NEXT: v_dual_mov_b32 v3, 0x4100 :: v_dual_lshlrev_b32 v0, 16, v0
-; GFX11-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-NEXT: v_cndmask_b32_e64 v1, v3, 0x4000, vcc_lo
-; GFX11-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-NEXT: v_mul_f32_e32 v0, v0, v1
-; GFX11-GISEL-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX11-GISEL-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX11-GISEL-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11-GISEL-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
+; GFX11-GISEL-TRUE16-LABEL: fmul_select_bf16_test5:
+; GFX11-GISEL-TRUE16: ; %bb.0:
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: v_dual_mov_b32 v3, 0x4000 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX11-GISEL-TRUE16-NEXT: v_mov_b32_e32 v4, 0x4100
+; GFX11-GISEL-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b32_e32 v1, v4, v3, vcc_lo
+; GFX11-GISEL-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-TRUE16-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX11-GISEL-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX11-GISEL-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX11-GISEL-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-GISEL-FAKE16-LABEL: fmul_select_bf16_test5:
+; GFX11-GISEL-FAKE16: ; %bb.0:
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: v_dual_mov_b32 v3, 0x4100 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX11-GISEL-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e64 v1, v3, 0x4000, vcc_lo
+; GFX11-GISEL-FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-FAKE16-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX11-GISEL-FAKE16-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX11-GISEL-FAKE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX11-GISEL-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-FAKE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
%bool = icmp eq i32 %bool.arg1, %bool.arg2
%y = select i1 %bool, bfloat 2.000000e+00, bfloat 8.000000e+00
%ldexp = fmul bfloat %x, %y
@@ -3986,25 +4053,46 @@ define bfloat @fmul_select_bf16_test6(bfloat %x, i32 %bool.arg1, i32 %bool.arg2)
; GFX11-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-GISEL-LABEL: fmul_select_bf16_test6:
-; GFX11-GISEL: ; %bb.0:
-; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-NEXT: v_dual_mov_b32 v3, 0x4040 :: v_dual_lshlrev_b32 v0, 16, v0
-; GFX11-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-NEXT: v_cndmask_b32_e64 v1, v3, 0xc100, vcc_lo
-; GFX11-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-NEXT: v_mul_f32_e32 v0, v0, v1
-; GFX11-GISEL-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX11-GISEL-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX11-GISEL-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11-GISEL-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
+; GFX11-GISEL-TRUE16-LABEL: fmul_select_bf16_test6:
+; GFX11-GISEL-TRUE16: ; %bb.0:
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: v_dual_mov_b32 v3, 0xc100 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX11-GISEL-TRUE16-NEXT: v_mov_b32_e32 v4, 0x4040
+; GFX11-GISEL-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b32_e32 v1, v4, v3, vcc_lo
+; GFX11-GISEL-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-TRUE16-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX11-GISEL-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX11-GISEL-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX11-GISEL-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-GISEL-FAKE16-LABEL: fmul_select_bf16_test6:
+; GFX11-GISEL-FAKE16: ; %bb.0:
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: v_dual_mov_b32 v3, 0x4040 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX11-GISEL-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e64 v1, v3, 0xc100, vcc_lo
+; GFX11-GISEL-FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-FAKE16-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX11-GISEL-FAKE16-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX11-GISEL-FAKE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX11-GISEL-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-FAKE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
%bool = icmp eq i32 %bool.arg1, %bool.arg2
%y = select i1 %bool, bfloat -8.000000e+00, bfloat 3.000000e+00
%ldexp = fmul bfloat %x, %y
@@ -4153,25 +4241,46 @@ define bfloat @fmul_select_bf16_test7(bfloat %x, i32 %bool.arg1, i32 %bool.arg2)
; GFX11-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-GISEL-LABEL: fmul_select_bf16_test7:
-; GFX11-GISEL: ; %bb.0:
-; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-NEXT: v_dual_mov_b32 v3, 0xc080 :: v_dual_lshlrev_b32 v0, 16, v0
-; GFX11-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-NEXT: v_cndmask_b32_e64 v1, v3, 0x4100, vcc_lo
-; GFX11-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-NEXT: v_mul_f32_e32 v0, v0, v1
-; GFX11-GISEL-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX11-GISEL-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX11-GISEL-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11-GISEL-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
+; GFX11-GISEL-TRUE16-LABEL: fmul_select_bf16_test7:
+; GFX11-GISEL-TRUE16: ; %bb.0:
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: v_dual_mov_b32 v3, 0x4100 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX11-GISEL-TRUE16-NEXT: v_mov_b32_e32 v4, 0xc080
+; GFX11-GISEL-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b32_e32 v1, v4, v3, vcc_lo
+; GFX11-GISEL-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-TRUE16-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX11-GISEL-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX11-GISEL-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX11-GISEL-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-GISEL-FAKE16-LABEL: fmul_select_bf16_test7:
+; GFX11-GISEL-FAKE16: ; %bb.0:
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: v_dual_mov_b32 v3, 0xc080 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX11-GISEL-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e64 v1, v3, 0x4100, vcc_lo
+; GFX11-GISEL-FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-FAKE16-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX11-GISEL-FAKE16-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX11-GISEL-FAKE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX11-GISEL-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-FAKE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
%bool = icmp eq i32 %bool.arg1, %bool.arg2
%y = select i1 %bool, bfloat 8.000000e+00, bfloat -4.000000e+00
%ldexp = fmul bfloat %x, %y
@@ -4314,24 +4423,45 @@ define bfloat @fmul_select_bf16_test8(bfloat %x, i32 %bool.arg1, i32 %bool.arg2)
; GFX11-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-GISEL-LABEL: fmul_select_bf16_test8:
-; GFX11-GISEL: ; %bb.0:
-; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX11-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-GISEL-NEXT: v_cndmask_b32_e64 v1, 0, 0x8000, vcc_lo
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-GISEL-NEXT: v_mul_f32_e32 v0, v0, v1
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-GISEL-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX11-GISEL-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX11-GISEL-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
-; GFX11-GISEL-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
+; GFX11-GISEL-TRUE16-LABEL: fmul_select_bf16_test8:
+; GFX11-GISEL-TRUE16: ; %bb.0:
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: v_dual_mov_b32 v3, 0x8000 :: v_dual_mov_b32 v4, 0
+; GFX11-GISEL-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
+; GFX11-GISEL-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b32_e32 v1, v4, v3, vcc_lo
+; GFX11-GISEL-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-TRUE16-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX11-GISEL-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX11-GISEL-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX11-GISEL-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-GISEL-FAKE16-LABEL: fmul_select_bf16_test8:
+; GFX11-GISEL-FAKE16: ; %bb.0:
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
+; GFX11-GISEL-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e64 v1, 0, 0x8000, vcc_lo
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-GISEL-FAKE16-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-GISEL-FAKE16-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX11-GISEL-FAKE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX11-GISEL-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX11-GISEL-FAKE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
%bool = icmp eq i32 %bool.arg1, %bool.arg2
%y = select i1 %bool, bfloat -0.000000e+00, bfloat 0.000000e+00
%ldexp = fmul bfloat %x, %y
@@ -4481,25 +4611,46 @@ define bfloat @fmul_select_bf16_test9(bfloat %x, i32 %bool.arg1, i32 %bool.arg2)
; GFX11-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-GISEL-LABEL: fmul_select_bf16_test9:
-; GFX11-GISEL: ; %bb.0:
-; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-NEXT: v_dual_mov_b32 v3, 0xc200 :: v_dual_lshlrev_b32 v0, 16, v0
-; GFX11-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-NEXT: v_cndmask_b32_e64 v1, v3, 0xc180, vcc_lo
-; GFX11-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-NEXT: v_mul_f32_e32 v0, v0, v1
-; GFX11-GISEL-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX11-GISEL-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX11-GISEL-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11-GISEL-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
+; GFX11-GISEL-TRUE16-LABEL: fmul_select_bf16_test9:
+; GFX11-GISEL-TRUE16: ; %bb.0:
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: v_dual_mov_b32 v3, 0xc180 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX11-GISEL-TRUE16-NEXT: v_mov_b32_e32 v4, 0xc200
+; GFX11-GISEL-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b32_e32 v1, v4, v3, vcc_lo
+; GFX11-GISEL-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-TRUE16-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX11-GISEL-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX11-GISEL-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX11-GISEL-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-GISEL-FAKE16-LABEL: fmul_select_bf16_test9:
+; GFX11-GISEL-FAKE16: ; %bb.0:
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: v_dual_mov_b32 v3, 0xc200 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX11-GISEL-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e64 v1, v3, 0xc180, vcc_lo
+; GFX11-GISEL-FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-FAKE16-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX11-GISEL-FAKE16-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX11-GISEL-FAKE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX11-GISEL-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-FAKE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
%bool = icmp eq i32 %bool.arg1, %bool.arg2
%y = select i1 %bool, bfloat -1.600000e+01, bfloat -3.200000e+01
%ldexp = fmul bfloat %x, %y
@@ -4649,25 +4800,46 @@ define bfloat @fmul_select_bf16_test10_sel_log2val_pos65_pos56(bfloat %x, i32 %b
; GFX11-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-GISEL-LABEL: fmul_select_bf16_test10_sel_log2val_pos65_pos56:
-; GFX11-GISEL: ; %bb.0:
-; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-NEXT: v_dual_mov_b32 v3, 0xdb80 :: v_dual_lshlrev_b32 v0, 16, v0
-; GFX11-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-NEXT: v_cndmask_b32_e64 v1, v3, 0xe000, vcc_lo
-; GFX11-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-NEXT: v_mul_f32_e32 v0, v0, v1
-; GFX11-GISEL-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX11-GISEL-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX11-GISEL-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11-GISEL-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
+; GFX11-GISEL-TRUE16-LABEL: fmul_select_bf16_test10_sel_log2val_pos65_pos56:
+; GFX11-GISEL-TRUE16: ; %bb.0:
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: v_dual_mov_b32 v3, 0xe000 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX11-GISEL-TRUE16-NEXT: v_mov_b32_e32 v4, 0xdb80
+; GFX11-GISEL-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b32_e32 v1, v4, v3, vcc_lo
+; GFX11-GISEL-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-TRUE16-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX11-GISEL-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX11-GISEL-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX11-GISEL-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-GISEL-FAKE16-LABEL: fmul_select_bf16_test10_sel_log2val_pos65_pos56:
+; GFX11-GISEL-FAKE16: ; %bb.0:
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: v_dual_mov_b32 v3, 0xdb80 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX11-GISEL-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e64 v1, v3, 0xe000, vcc_lo
+; GFX11-GISEL-FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-FAKE16-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX11-GISEL-FAKE16-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX11-GISEL-FAKE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX11-GISEL-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-FAKE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
%bool = icmp eq i32 %bool.arg1, %bool.arg2
%y = select i1 %bool, bfloat 0xRE000, bfloat 0xRDB80
%ldexp = fmul bfloat %x, %y
@@ -4817,36 +4989,49 @@ define bfloat @fmul_select_bf16_test11_sel_log2val_neg22_pos25(bfloat %x, i32 %b
; GFX11-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-GISEL-LABEL: fmul_select_bf16_test11_sel_log2val_neg22_pos25:
-; GFX11-GISEL: ; %bb.0:
-; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-NEXT: v_dual_mov_b32 v3, 0x4c00 :: v_dual_lshlrev_b32 v0, 16, v0
-; GFX11-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-NEXT: v_cndmask_b32_e64 v1, v3, 0x3480, vcc_lo
-; GFX11-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-NEXT: v_mul_f32_e32 v0, v0, v1
-; GFX11-GISEL-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX11-GISEL-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX11-GISEL-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11-GISEL-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
+; GFX11-GISEL-TRUE16-LABEL: fmul_select_bf16_test11_sel_log2val_neg22_pos25:
+; GFX11-GISEL-TRUE16: ; %bb.0:
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: v_dual_mov_b32 v3, 0x3480 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX11-GISEL-TRUE16-NEXT: v_mov_b32_e32 v4, 0x4c00
+; GFX11-GISEL-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b32_e32 v1, v4, v3, vcc_lo
+; GFX11-GISEL-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-TRUE16-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX11-GISEL-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX11-GISEL-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX11-GISEL-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-GISEL-FAKE16-LABEL: fmul_select_bf16_test11_sel_log2val_neg22_pos25:
+; GFX11-GISEL-FAKE16: ; %bb.0:
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: v_dual_mov_b32 v3, 0x4c00 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX11-GISEL-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e64 v1, v3, 0x3480, vcc_lo
+; GFX11-GISEL-FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-FAKE16-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX11-GISEL-FAKE16-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX11-GISEL-FAKE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX11-GISEL-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, 0, v0
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-FAKE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-GISEL-FAKE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
%bool = icmp eq i32 %bool.arg1, %bool.arg2
%y = select i1 %bool, bfloat 0xR3480, bfloat 0xR4C00
%ldexp = fmul bfloat %x, %y
ret bfloat %ldexp
}
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; GFX10: {{.*}}
-; GFX11: {{.*}}
-; GFX11-GISEL-FAKE16: {{.*}}
-; GFX11-GISEL-TRUE16: {{.*}}
-; GFX11-SDAG: {{.*}}
-; GFX7: {{.*}}
-; GFX9: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/fmaximum.ll b/llvm/test/CodeGen/AMDGPU/fmaximum.ll
index 213d6ffeb9f00..6e2c47ac62c24 100644
--- a/llvm/test/CodeGen/AMDGPU/fmaximum.ll
+++ b/llvm/test/CodeGen/AMDGPU/fmaximum.ll
@@ -1530,7 +1530,7 @@ define amdgpu_kernel void @fmaximum_f16_move_to_valu(ptr addrspace(1) %out, ptr
; GFX12-GISEL-TRUE16-NEXT: v_readfirstlane_b32 s3, v1
; GFX12-GISEL-TRUE16-NEXT: s_maximum_f16 s2, s2, s3
; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
-; GFX12-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.l, s2
+; GFX12-GISEL-TRUE16-NEXT: v_mov_b32_e32 v0, s2
; GFX12-GISEL-TRUE16-NEXT: global_store_b16 v2, v0, s[0:1]
; GFX12-GISEL-TRUE16-NEXT: s_endpgm
;
diff --git a/llvm/test/CodeGen/AMDGPU/fmaxnum.ll b/llvm/test/CodeGen/AMDGPU/fmaxnum.ll
index 25090bf693552..a183ea4ad2a73 100644
--- a/llvm/test/CodeGen/AMDGPU/fmaxnum.ll
+++ b/llvm/test/CodeGen/AMDGPU/fmaxnum.ll
@@ -1943,7 +1943,7 @@ define amdgpu_kernel void @test_fmax_f16_v_ieee_on(ptr addrspace(1) %out, half %
; GFX12-GISEL-NEXT: s_mov_b32 s3, 0x31016000
; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-GISEL-NEXT: v_mov_b16_e32 v0.l, s2
+; GFX12-GISEL-NEXT: v_mov_b32_e32 v0, s2
; GFX12-GISEL-NEXT: s_mov_b32 s2, -1
; GFX12-GISEL-NEXT: buffer_store_b16 v0, off, s[0:3], null
; GFX12-GISEL-NEXT: s_endpgm
@@ -2064,7 +2064,7 @@ define amdgpu_kernel void @test_fmax_f16_s_ieee_on(ptr addrspace(1) %out, half i
; GFX12-GISEL-NEXT: s_mov_b32 s3, 0x31016000
; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-GISEL-NEXT: v_mov_b16_e32 v0.l, s2
+; GFX12-GISEL-NEXT: v_mov_b32_e32 v0, s2
; GFX12-GISEL-NEXT: s_mov_b32 s2, -1
; GFX12-GISEL-NEXT: buffer_store_b16 v0, off, s[0:3], null
; GFX12-GISEL-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/fminimum.ll b/llvm/test/CodeGen/AMDGPU/fminimum.ll
index 094dd76e0f945..2156fc6f2397c 100644
--- a/llvm/test/CodeGen/AMDGPU/fminimum.ll
+++ b/llvm/test/CodeGen/AMDGPU/fminimum.ll
@@ -1530,7 +1530,7 @@ define amdgpu_kernel void @fminimum_f16_move_to_valu(ptr addrspace(1) %out, ptr
; GFX12-GISEL-TRUE16-NEXT: v_readfirstlane_b32 s3, v1
; GFX12-GISEL-TRUE16-NEXT: s_minimum_f16 s2, s2, s3
; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
-; GFX12-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.l, s2
+; GFX12-GISEL-TRUE16-NEXT: v_mov_b32_e32 v0, s2
; GFX12-GISEL-TRUE16-NEXT: global_store_b16 v2, v0, s[0:1]
; GFX12-GISEL-TRUE16-NEXT: s_endpgm
;
diff --git a/llvm/test/CodeGen/AMDGPU/fminnum.ll b/llvm/test/CodeGen/AMDGPU/fminnum.ll
index 06fb492792ae8..86329738d2520 100644
--- a/llvm/test/CodeGen/AMDGPU/fminnum.ll
+++ b/llvm/test/CodeGen/AMDGPU/fminnum.ll
@@ -1856,7 +1856,7 @@ define amdgpu_kernel void @test_fmin_f16_v_ieee_on(ptr addrspace(1) %out, half %
; GFX12-GISEL-NEXT: s_mov_b32 s3, 0x31016000
; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-GISEL-NEXT: v_mov_b16_e32 v0.l, s2
+; GFX12-GISEL-NEXT: v_mov_b32_e32 v0, s2
; GFX12-GISEL-NEXT: s_mov_b32 s2, -1
; GFX12-GISEL-NEXT: buffer_store_b16 v0, off, s[0:3], null
; GFX12-GISEL-NEXT: s_endpgm
@@ -1977,7 +1977,7 @@ define amdgpu_kernel void @test_fmin_f16_s_ieee_on(ptr addrspace(1) %out, half i
; GFX12-GISEL-NEXT: s_mov_b32 s3, 0x31016000
; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-GISEL-NEXT: v_mov_b16_e32 v0.l, s2
+; GFX12-GISEL-NEXT: v_mov_b32_e32 v0, s2
; GFX12-GISEL-NEXT: s_mov_b32 s2, -1
; GFX12-GISEL-NEXT: buffer_store_b16 v0, off, s[0:3], null
; GFX12-GISEL-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/fp_to_sint.ll b/llvm/test/CodeGen/AMDGPU/fp_to_sint.ll
index 2618c670b8c23..61798f566c890 100644
--- a/llvm/test/CodeGen/AMDGPU/fp_to_sint.ll
+++ b/llvm/test/CodeGen/AMDGPU/fp_to_sint.ll
@@ -1298,7 +1298,7 @@ define amdgpu_kernel void @fp_to_sint_f32_i16(ptr addrspace(1) %out, float %in)
; GFX11-GISEL-NEXT: v_cvt_i32_f32_e32 v0, s2
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-GISEL-NEXT: v_readfirstlane_b32 s2, v0
-; GFX11-GISEL-NEXT: v_mov_b16_e32 v0.l, s2
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v0, s2
; GFX11-GISEL-NEXT: global_store_b16 v1, v0, s[0:1]
; GFX11-GISEL-NEXT: s_endpgm
;
diff --git a/llvm/test/CodeGen/AMDGPU/fp_to_uint.ll b/llvm/test/CodeGen/AMDGPU/fp_to_uint.ll
index 2fb9e0b764391..a0a37b4c02a42 100644
--- a/llvm/test/CodeGen/AMDGPU/fp_to_uint.ll
+++ b/llvm/test/CodeGen/AMDGPU/fp_to_uint.ll
@@ -1069,7 +1069,7 @@ define amdgpu_kernel void @fp_to_uint_f32_to_i16(ptr addrspace(1) %out, float %i
; GFX11-GISEL-NEXT: v_cvt_u32_f32_e32 v0, s2
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-GISEL-NEXT: v_readfirstlane_b32 s2, v0
-; GFX11-GISEL-NEXT: v_mov_b16_e32 v0.l, s2
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v0, s2
; GFX11-GISEL-NEXT: global_store_b16 v1, v0, s[0:1]
; GFX11-GISEL-NEXT: s_endpgm
;
diff --git a/llvm/test/CodeGen/AMDGPU/fptrunc.f16.ll b/llvm/test/CodeGen/AMDGPU/fptrunc.f16.ll
index c993f0ebb8a64..aa39c45ccccd8 100644
--- a/llvm/test/CodeGen/AMDGPU/fptrunc.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fptrunc.f16.ll
@@ -255,7 +255,7 @@ define amdgpu_kernel void @fptrunc_f32_to_f16(
; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-TRUE16-NEXT: s_cvt_f16_f32 s2, s2
; GFX1250-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
-; GFX1250-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.l, s2
+; GFX1250-GISEL-TRUE16-NEXT: v_mov_b32_e32 v0, s2
; GFX1250-GISEL-TRUE16-NEXT: s_mov_b32 s2, -1
; GFX1250-GISEL-TRUE16-NEXT: buffer_store_b16 v0, off, s[0:3], null
; GFX1250-GISEL-TRUE16-NEXT: s_endpgm
@@ -526,7 +526,7 @@ define amdgpu_kernel void @fptrunc_f32_to_f16_afn(ptr addrspace(1) %r,
; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-TRUE16-NEXT: s_cvt_f16_f32 s2, s2
; GFX1250-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
-; GFX1250-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.l, s2
+; GFX1250-GISEL-TRUE16-NEXT: v_mov_b32_e32 v0, s2
; GFX1250-GISEL-TRUE16-NEXT: s_mov_b32 s2, -1
; GFX1250-GISEL-TRUE16-NEXT: buffer_store_b16 v0, off, s[0:3], null
; GFX1250-GISEL-TRUE16-NEXT: s_endpgm
@@ -1193,7 +1193,7 @@ define amdgpu_kernel void @fptrunc_f64_to_f16(
; GFX11-GISEL-TRUE16-NEXT: s_and_b32 s3, s3, 0x8000
; GFX11-GISEL-TRUE16-NEXT: s_or_b32 s2, s3, s2
; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s3, 0x31016000
-; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.l, s2
+; GFX11-GISEL-TRUE16-NEXT: v_mov_b32_e32 v0, s2
; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s2, -1
; GFX11-GISEL-TRUE16-NEXT: buffer_store_b16 v0, off, s[0:3], 0
; GFX11-GISEL-TRUE16-NEXT: s_endpgm
@@ -1452,7 +1452,7 @@ define amdgpu_kernel void @fptrunc_f64_to_f16(
; GFX1250-GISEL-TRUE16-NEXT: s_and_b32 s3, s3, 0x8000
; GFX1250-GISEL-TRUE16-NEXT: s_or_b32 s2, s3, s2
; GFX1250-GISEL-TRUE16-NEXT: s_mov_b32 s3, 0x31016000
-; GFX1250-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.l, s2
+; GFX1250-GISEL-TRUE16-NEXT: v_mov_b32_e32 v0, s2
; GFX1250-GISEL-TRUE16-NEXT: s_mov_b32 s2, -1
; GFX1250-GISEL-TRUE16-NEXT: buffer_store_b16 v0, off, s[0:3], null
; GFX1250-GISEL-TRUE16-NEXT: s_endpgm
@@ -1786,7 +1786,7 @@ define amdgpu_kernel void @fptrunc_f64_to_f16_afn(
; GFX1250-GISEL-TRUE16-NEXT: v_readfirstlane_b32 s2, v0
; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-TRUE16-NEXT: s_cvt_f16_f32 s2, s2
-; GFX1250-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.l, s2
+; GFX1250-GISEL-TRUE16-NEXT: v_mov_b32_e32 v0, s2
; GFX1250-GISEL-TRUE16-NEXT: s_mov_b32 s2, -1
; GFX1250-GISEL-TRUE16-NEXT: buffer_store_b16 v0, off, s[0:3], null
; GFX1250-GISEL-TRUE16-NEXT: s_endpgm
@@ -4435,7 +4435,7 @@ define amdgpu_kernel void @fneg_fptrunc_f32_to_f16(
; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-TRUE16-NEXT: s_cvt_f16_f32 s2, s2
; GFX1250-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
-; GFX1250-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.l, s2
+; GFX1250-GISEL-TRUE16-NEXT: v_mov_b32_e32 v0, s2
; GFX1250-GISEL-TRUE16-NEXT: s_mov_b32 s2, -1
; GFX1250-GISEL-TRUE16-NEXT: buffer_store_b16 v0, off, s[0:3], null
; GFX1250-GISEL-TRUE16-NEXT: s_endpgm
@@ -4713,7 +4713,7 @@ define amdgpu_kernel void @fabs_fptrunc_f32_to_f16(
; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-TRUE16-NEXT: s_cvt_f16_f32 s2, s2
; GFX1250-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
-; GFX1250-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.l, s2
+; GFX1250-GISEL-TRUE16-NEXT: v_mov_b32_e32 v0, s2
; GFX1250-GISEL-TRUE16-NEXT: s_mov_b32 s2, -1
; GFX1250-GISEL-TRUE16-NEXT: buffer_store_b16 v0, off, s[0:3], null
; GFX1250-GISEL-TRUE16-NEXT: s_endpgm
@@ -4991,7 +4991,7 @@ define amdgpu_kernel void @fneg_fabs_fptrunc_f32_to_f16(
; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-TRUE16-NEXT: s_cvt_f16_f32 s2, s2
; GFX1250-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
-; GFX1250-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.l, s2
+; GFX1250-GISEL-TRUE16-NEXT: v_mov_b32_e32 v0, s2
; GFX1250-GISEL-TRUE16-NEXT: s_mov_b32 s2, -1
; GFX1250-GISEL-TRUE16-NEXT: buffer_store_b16 v0, off, s[0:3], null
; GFX1250-GISEL-TRUE16-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/fptrunc.ll b/llvm/test/CodeGen/AMDGPU/fptrunc.ll
index 685fc615812dc..fb85ae0bef264 100644
--- a/llvm/test/CodeGen/AMDGPU/fptrunc.ll
+++ b/llvm/test/CodeGen/AMDGPU/fptrunc.ll
@@ -479,173 +479,61 @@ define amdgpu_kernel void @fptrunc_f64_to_f16(ptr addrspace(1) %out, double %in)
; GFX11-SDAG-NEXT: buffer_store_b16 v0, off, s[0:3], 0
; GFX11-SDAG-NEXT: s_endpgm
;
-; GFX11-SAFE-GISEL-LABEL: fptrunc_f64_to_f16:
-; GFX11-SAFE-GISEL: ; %bb.0:
-; GFX11-SAFE-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-SAFE-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-SAFE-GISEL-NEXT: s_bfe_u32 s4, s3, 0xb0014
-; GFX11-SAFE-GISEL-NEXT: s_lshr_b32 s5, s3, 8
-; GFX11-SAFE-GISEL-NEXT: s_and_b32 s6, s3, 0x1ff
-; GFX11-SAFE-GISEL-NEXT: s_addk_i32 s4, 0xfc10
-; GFX11-SAFE-GISEL-NEXT: s_and_b32 s5, s5, 0xffe
-; GFX11-SAFE-GISEL-NEXT: s_or_b32 s2, s6, s2
-; GFX11-SAFE-GISEL-NEXT: s_cselect_b32 s2, 1, 0
-; GFX11-SAFE-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-SAFE-GISEL-NEXT: s_or_b32 s2, s5, s2
-; GFX11-SAFE-GISEL-NEXT: s_cselect_b32 s5, 1, 0
-; GFX11-SAFE-GISEL-NEXT: s_sub_i32 s6, 1, s4
-; GFX11-SAFE-GISEL-NEXT: s_or_b32 s8, s2, 0x1000
-; GFX11-SAFE-GISEL-NEXT: s_max_i32 s6, s6, 0
-; GFX11-SAFE-GISEL-NEXT: s_lshl_b32 s7, s4, 12
-; GFX11-SAFE-GISEL-NEXT: s_min_i32 s6, s6, 13
-; GFX11-SAFE-GISEL-NEXT: s_lshl_b32 s5, s5, 9
-; GFX11-SAFE-GISEL-NEXT: s_lshr_b32 s9, s8, s6
-; GFX11-SAFE-GISEL-NEXT: s_or_b32 s2, s2, s7
-; GFX11-SAFE-GISEL-NEXT: s_lshl_b32 s6, s9, s6
-; GFX11-SAFE-GISEL-NEXT: s_or_b32 s5, s5, 0x7c00
-; GFX11-SAFE-GISEL-NEXT: s_cmp_lg_u32 s6, s8
-; GFX11-SAFE-GISEL-NEXT: s_cselect_b32 s6, 1, 0
-; GFX11-SAFE-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX11-SAFE-GISEL-NEXT: s_or_b32 s6, s9, s6
-; GFX11-SAFE-GISEL-NEXT: s_cmp_lt_i32 s4, 1
-; GFX11-SAFE-GISEL-NEXT: s_cselect_b32 s2, s6, s2
-; GFX11-SAFE-GISEL-NEXT: s_and_b32 s6, s2, 7
-; GFX11-SAFE-GISEL-NEXT: s_lshr_b32 s2, s2, 2
-; GFX11-SAFE-GISEL-NEXT: s_cmp_eq_u32 s6, 3
-; GFX11-SAFE-GISEL-NEXT: s_cselect_b32 s7, 1, 0
-; GFX11-SAFE-GISEL-NEXT: s_cmp_gt_i32 s6, 5
-; GFX11-SAFE-GISEL-NEXT: s_cselect_b32 s6, 1, 0
-; GFX11-SAFE-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-SAFE-GISEL-NEXT: s_or_b32 s6, s7, s6
-; GFX11-SAFE-GISEL-NEXT: s_cmp_lg_u32 s6, 0
-; GFX11-SAFE-GISEL-NEXT: s_cselect_b32 s6, 1, 0
-; GFX11-SAFE-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-SAFE-GISEL-NEXT: s_add_i32 s2, s2, s6
-; GFX11-SAFE-GISEL-NEXT: s_cmp_gt_i32 s4, 30
-; GFX11-SAFE-GISEL-NEXT: s_cselect_b32 s2, 0x7c00, s2
-; GFX11-SAFE-GISEL-NEXT: s_cmpk_eq_i32 s4, 0x40f
-; GFX11-SAFE-GISEL-NEXT: s_cselect_b32 s2, s5, s2
-; GFX11-SAFE-GISEL-NEXT: s_lshr_b32 s3, s3, 16
-; GFX11-SAFE-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-SAFE-GISEL-NEXT: s_and_b32 s3, s3, 0x8000
-; GFX11-SAFE-GISEL-NEXT: s_or_b32 s2, s3, s2
-; GFX11-SAFE-GISEL-NEXT: s_mov_b32 s3, 0x31016000
-; GFX11-SAFE-GISEL-NEXT: v_mov_b16_e32 v0.l, s2
-; GFX11-SAFE-GISEL-NEXT: s_mov_b32 s2, -1
-; GFX11-SAFE-GISEL-NEXT: buffer_store_b16 v0, off, s[0:3], 0
-; GFX11-SAFE-GISEL-NEXT: s_endpgm
-;
-; GFX11-UNSAFE-GISEL-TRUE16-LABEL: fptrunc_f64_to_f16:
-; GFX11-UNSAFE-GISEL-TRUE16: ; %bb.0:
-; GFX11-UNSAFE-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-UNSAFE-GISEL-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-UNSAFE-GISEL-TRUE16-NEXT: s_bfe_u32 s4, s3, 0xb0014
-; GFX11-UNSAFE-GISEL-TRUE16-NEXT: s_lshr_b32 s5, s3, 8
-; GFX11-UNSAFE-GISEL-TRUE16-NEXT: s_and_b32 s6, s3, 0x1ff
-; GFX11-UNSAFE-GISEL-TRUE16-NEXT: s_addk_i32 s4, 0xfc10
-; GFX11-UNSAFE-GISEL-TRUE16-NEXT: s_and_b32 s5, s5, 0xffe
-; GFX11-UNSAFE-GISEL-TRUE16-NEXT: s_or_b32 s2, s6, s2
-; GFX11-UNSAFE-GISEL-TRUE16-NEXT: s_cselect_b32 s2, 1, 0
-; GFX11-UNSAFE-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-UNSAFE-GISEL-TRUE16-NEXT: s_or_b32 s2, s5, s2
-; GFX11-UNSAFE-GISEL-TRUE16-NEXT: s_cselect_b32 s5, 1, 0
-; GFX11-UNSAFE-GISEL-TRUE16-NEXT: s_sub_i32 s6, 1, s4
-; GFX11-UNSAFE-GISEL-TRUE16-NEXT: s_or_b32 s8, s2, 0x1000
-; GFX11-UNSAFE-GISEL-TRUE16-NEXT: s_max_i32 s6, s6, 0
-; GFX11-UNSAFE-GISEL-TRUE16-NEXT: s_lshl_b32 s7, s4, 12
-; GFX11-UNSAFE-GISEL-TRUE16-NEXT: s_min_i32 s6, s6, 13
-; GFX11-UNSAFE-GISEL-TRUE16-NEXT: s_lshl_b32 s5, s5, 9
-; GFX11-UNSAFE-GISEL-TRUE16-NEXT: s_lshr_b32 s9, s8, s6
-; GFX11-UNSAFE-GISEL-TRUE16-NEXT: s_or_b32 s2, s2, s7
-; GFX11-UNSAFE-GISEL-TRUE16-NEXT: s_lshl_b32 s6, s9, s6
-; GFX11-UNSAFE-GISEL-TRUE16-NEXT: s_or_b32 s5, s5, 0x7c00
-; GFX11-UNSAFE-GISEL-TRUE16-NEXT: s_cmp_lg_u32 s6, s8
-; GFX11-UNSAFE-GISEL-TRUE16-NEXT: s_cselect_b32 s6, 1, 0
-; GFX11-UNSAFE-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX11-UNSAFE-GISEL-TRUE16-NEXT: s_or_b32 s6, s9, s6
-; GFX11-UNSAFE-GISEL-TRUE16-NEXT: s_cmp_lt_i32 s4, 1
-; GFX11-UNSAFE-GISEL-TRUE16-NEXT: s_cselect_b32 s2, s6, s2
-; GFX11-UNSAFE-GISEL-TRUE16-NEXT: s_and_b32 s6, s2, 7
-; GFX11-UNSAFE-GISEL-TRUE16-NEXT: s_lshr_b32 s2, s2, 2
-; GFX11-UNSAFE-GISEL-TRUE16-NEXT: s_cmp_eq_u32 s6, 3
-; GFX11-UNSAFE-GISEL-TRUE16-NEXT: s_cselect_b32 s7, 1, 0
-; GFX11-UNSAFE-GISEL-TRUE16-NEXT: s_cmp_gt_i32 s6, 5
-; GFX11-UNSAFE-GISEL-TRUE16-NEXT: s_cselect_b32 s6, 1, 0
-; GFX11-UNSAFE-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-UNSAFE-GISEL-TRUE16-NEXT: s_or_b32 s6, s7, s6
-; GFX11-UNSAFE-GISEL-TRUE16-NEXT: s_cmp_lg_u32 s6, 0
-; GFX11-UNSAFE-GISEL-TRUE16-NEXT: s_cselect_b32 s6, 1, 0
-; GFX11-UNSAFE-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-UNSAFE-GISEL-TRUE16-NEXT: s_add_i32 s2, s2, s6
-; GFX11-UNSAFE-GISEL-TRUE16-NEXT: s_cmp_gt_i32 s4, 30
-; GFX11-UNSAFE-GISEL-TRUE16-NEXT: s_cselect_b32 s2, 0x7c00, s2
-; GFX11-UNSAFE-GISEL-TRUE16-NEXT: s_cmpk_eq_i32 s4, 0x40f
-; GFX11-UNSAFE-GISEL-TRUE16-NEXT: s_cselect_b32 s2, s5, s2
-; GFX11-UNSAFE-GISEL-TRUE16-NEXT: s_lshr_b32 s3, s3, 16
-; GFX11-UNSAFE-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-UNSAFE-GISEL-TRUE16-NEXT: s_and_b32 s3, s3, 0x8000
-; GFX11-UNSAFE-GISEL-TRUE16-NEXT: s_or_b32 s2, s3, s2
-; GFX11-UNSAFE-GISEL-TRUE16-NEXT: s_mov_b32 s3, 0x31016000
-; GFX11-UNSAFE-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.l, s2
-; GFX11-UNSAFE-GISEL-TRUE16-NEXT: s_mov_b32 s2, -1
-; GFX11-UNSAFE-GISEL-TRUE16-NEXT: buffer_store_b16 v0, off, s[0:3], 0
-; GFX11-UNSAFE-GISEL-TRUE16-NEXT: s_endpgm
-;
-; GFX11-UNSAFE-GISEL-FAKE16-LABEL: fptrunc_f64_to_f16:
-; GFX11-UNSAFE-GISEL-FAKE16: ; %bb.0:
-; GFX11-UNSAFE-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-UNSAFE-GISEL-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-UNSAFE-GISEL-FAKE16-NEXT: s_bfe_u32 s4, s3, 0xb0014
-; GFX11-UNSAFE-GISEL-FAKE16-NEXT: s_lshr_b32 s5, s3, 8
-; GFX11-UNSAFE-GISEL-FAKE16-NEXT: s_and_b32 s6, s3, 0x1ff
-; GFX11-UNSAFE-GISEL-FAKE16-NEXT: s_addk_i32 s4, 0xfc10
-; GFX11-UNSAFE-GISEL-FAKE16-NEXT: s_and_b32 s5, s5, 0xffe
-; GFX11-UNSAFE-GISEL-FAKE16-NEXT: s_or_b32 s2, s6, s2
-; GFX11-UNSAFE-GISEL-FAKE16-NEXT: s_cselect_b32 s2, 1, 0
-; GFX11-UNSAFE-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-UNSAFE-GISEL-FAKE16-NEXT: s_or_b32 s2, s5, s2
-; GFX11-UNSAFE-GISEL-FAKE16-NEXT: s_cselect_b32 s5, 1, 0
-; GFX11-UNSAFE-GISEL-FAKE16-NEXT: s_sub_i32 s6, 1, s4
-; GFX11-UNSAFE-GISEL-FAKE16-NEXT: s_or_b32 s8, s2, 0x1000
-; GFX11-UNSAFE-GISEL-FAKE16-NEXT: s_max_i32 s6, s6, 0
-; GFX11-UNSAFE-GISEL-FAKE16-NEXT: s_lshl_b32 s7, s4, 12
-; GFX11-UNSAFE-GISEL-FAKE16-NEXT: s_min_i32 s6, s6, 13
-; GFX11-UNSAFE-GISEL-FAKE16-NEXT: s_lshl_b32 s5, s5, 9
-; GFX11-UNSAFE-GISEL-FAKE16-NEXT: s_lshr_b32 s9, s8, s6
-; GFX11-UNSAFE-GISEL-FAKE16-NEXT: s_or_b32 s2, s2, s7
-; GFX11-UNSAFE-GISEL-FAKE16-NEXT: s_lshl_b32 s6, s9, s6
-; GFX11-UNSAFE-GISEL-FAKE16-NEXT: s_or_b32 s5, s5, 0x7c00
-; GFX11-UNSAFE-GISEL-FAKE16-NEXT: s_cmp_lg_u32 s6, s8
-; GFX11-UNSAFE-GISEL-FAKE16-NEXT: s_cselect_b32 s6, 1, 0
-; GFX11-UNSAFE-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX11-UNSAFE-GISEL-FAKE16-NEXT: s_or_b32 s6, s9, s6
-; GFX11-UNSAFE-GISEL-FAKE16-NEXT: s_cmp_lt_i32 s4, 1
-; GFX11-UNSAFE-GISEL-FAKE16-NEXT: s_cselect_b32 s2, s6, s2
-; GFX11-UNSAFE-GISEL-FAKE16-NEXT: s_and_b32 s6, s2, 7
-; GFX11-UNSAFE-GISEL-FAKE16-NEXT: s_lshr_b32 s2, s2, 2
-; GFX11-UNSAFE-GISEL-FAKE16-NEXT: s_cmp_eq_u32 s6, 3
-; GFX11-UNSAFE-GISEL-FAKE16-NEXT: s_cselect_b32 s7, 1, 0
-; GFX11-UNSAFE-GISEL-FAKE16-NEXT: s_cmp_gt_i32 s6, 5
-; GFX11-UNSAFE-GISEL-FAKE16-NEXT: s_cselect_b32 s6, 1, 0
-; GFX11-UNSAFE-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-UNSAFE-GISEL-FAKE16-NEXT: s_or_b32 s6, s7, s6
-; GFX11-UNSAFE-GISEL-FAKE16-NEXT: s_cmp_lg_u32 s6, 0
-; GFX11-UNSAFE-GISEL-FAKE16-NEXT: s_cselect_b32 s6, 1, 0
-; GFX11-UNSAFE-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-UNSAFE-GISEL-FAKE16-NEXT: s_add_i32 s2, s2, s6
-; GFX11-UNSAFE-GISEL-FAKE16-NEXT: s_cmp_gt_i32 s4, 30
-; GFX11-UNSAFE-GISEL-FAKE16-NEXT: s_cselect_b32 s2, 0x7c00, s2
-; GFX11-UNSAFE-GISEL-FAKE16-NEXT: s_cmpk_eq_i32 s4, 0x40f
-; GFX11-UNSAFE-GISEL-FAKE16-NEXT: s_cselect_b32 s2, s5, s2
-; GFX11-UNSAFE-GISEL-FAKE16-NEXT: s_lshr_b32 s3, s3, 16
-; GFX11-UNSAFE-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-UNSAFE-GISEL-FAKE16-NEXT: s_and_b32 s3, s3, 0x8000
-; GFX11-UNSAFE-GISEL-FAKE16-NEXT: s_or_b32 s2, s3, s2
-; GFX11-UNSAFE-GISEL-FAKE16-NEXT: s_mov_b32 s3, 0x31016000
-; GFX11-UNSAFE-GISEL-FAKE16-NEXT: v_mov_b32_e32 v0, s2
-; GFX11-UNSAFE-GISEL-FAKE16-NEXT: s_mov_b32 s2, -1
-; GFX11-UNSAFE-GISEL-FAKE16-NEXT: buffer_store_b16 v0, off, s[0:3], 0
-; GFX11-UNSAFE-GISEL-FAKE16-NEXT: s_endpgm
+; GFX11-GISEL-LABEL: fptrunc_f64_to_f16:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: s_bfe_u32 s4, s3, 0xb0014
+; GFX11-GISEL-NEXT: s_lshr_b32 s5, s3, 8
+; GFX11-GISEL-NEXT: s_and_b32 s6, s3, 0x1ff
+; GFX11-GISEL-NEXT: s_addk_i32 s4, 0xfc10
+; GFX11-GISEL-NEXT: s_and_b32 s5, s5, 0xffe
+; GFX11-GISEL-NEXT: s_or_b32 s2, s6, s2
+; GFX11-GISEL-NEXT: s_cselect_b32 s2, 1, 0
+; GFX11-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-NEXT: s_or_b32 s2, s5, s2
+; GFX11-GISEL-NEXT: s_cselect_b32 s5, 1, 0
+; GFX11-GISEL-NEXT: s_sub_i32 s6, 1, s4
+; GFX11-GISEL-NEXT: s_or_b32 s8, s2, 0x1000
+; GFX11-GISEL-NEXT: s_max_i32 s6, s6, 0
+; GFX11-GISEL-NEXT: s_lshl_b32 s7, s4, 12
+; GFX11-GISEL-NEXT: s_min_i32 s6, s6, 13
+; GFX11-GISEL-NEXT: s_lshl_b32 s5, s5, 9
+; GFX11-GISEL-NEXT: s_lshr_b32 s9, s8, s6
+; GFX11-GISEL-NEXT: s_or_b32 s2, s2, s7
+; GFX11-GISEL-NEXT: s_lshl_b32 s6, s9, s6
+; GFX11-GISEL-NEXT: s_or_b32 s5, s5, 0x7c00
+; GFX11-GISEL-NEXT: s_cmp_lg_u32 s6, s8
+; GFX11-GISEL-NEXT: s_cselect_b32 s6, 1, 0
+; GFX11-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX11-GISEL-NEXT: s_or_b32 s6, s9, s6
+; GFX11-GISEL-NEXT: s_cmp_lt_i32 s4, 1
+; GFX11-GISEL-NEXT: s_cselect_b32 s2, s6, s2
+; GFX11-GISEL-NEXT: s_and_b32 s6, s2, 7
+; GFX11-GISEL-NEXT: s_lshr_b32 s2, s2, 2
+; GFX11-GISEL-NEXT: s_cmp_eq_u32 s6, 3
+; GFX11-GISEL-NEXT: s_cselect_b32 s7, 1, 0
+; GFX11-GISEL-NEXT: s_cmp_gt_i32 s6, 5
+; GFX11-GISEL-NEXT: s_cselect_b32 s6, 1, 0
+; GFX11-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-GISEL-NEXT: s_or_b32 s6, s7, s6
+; GFX11-GISEL-NEXT: s_cmp_lg_u32 s6, 0
+; GFX11-GISEL-NEXT: s_cselect_b32 s6, 1, 0
+; GFX11-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-NEXT: s_add_i32 s2, s2, s6
+; GFX11-GISEL-NEXT: s_cmp_gt_i32 s4, 30
+; GFX11-GISEL-NEXT: s_cselect_b32 s2, 0x7c00, s2
+; GFX11-GISEL-NEXT: s_cmpk_eq_i32 s4, 0x40f
+; GFX11-GISEL-NEXT: s_cselect_b32 s2, s5, s2
+; GFX11-GISEL-NEXT: s_lshr_b32 s3, s3, 16
+; GFX11-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-GISEL-NEXT: s_and_b32 s3, s3, 0x8000
+; GFX11-GISEL-NEXT: s_or_b32 s2, s3, s2
+; GFX11-GISEL-NEXT: s_mov_b32 s3, 0x31016000
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX11-GISEL-NEXT: s_mov_b32 s2, -1
+; GFX11-GISEL-NEXT: buffer_store_b16 v0, off, s[0:3], 0
+; GFX11-GISEL-NEXT: s_endpgm
%result = fptrunc double %in to half
%result_i16 = bitcast half %result to i16
store i16 %result_i16, ptr addrspace(1) %out
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.bitop3.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.bitop3.ll
index 2c5d958aa9dbe..eb6fef7eb65a5 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.bitop3.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.bitop3.ll
@@ -307,7 +307,7 @@ define amdgpu_ps half @bitop3_b16_sss(i16 inreg %a, i16 inreg %b, i16 inreg %c)
; GFX1250-GISEL-TRUE16-NEXT: global_prefetch_b8 v0, null scope:SCOPE_SE
; GFX1250-GISEL-TRUE16-NEXT: v_nop
; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.l, s2
+; GFX1250-GISEL-TRUE16-NEXT: v_mov_b32_e32 v0, s2
; GFX1250-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-GISEL-TRUE16-NEXT: v_bitop3_b16 v0.l, s0, s1, v0.l bitop3:0x12
; GFX1250-GISEL-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.f16.fp8.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.f16.fp8.ll
index cdd17ddd10473..cc3bb15b72af3 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.f16.fp8.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.f16.fp8.ll
@@ -263,8 +263,11 @@ define amdgpu_ps float @test_cvt_f16_bf8_byte3_hi(i32 %a) {
; GFX1250-GISEL-REAL16-NEXT: global_prefetch_b8 v0, null scope:SCOPE_SE
; GFX1250-GISEL-REAL16-NEXT: v_nop
; GFX1250-GISEL-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-REAL16-NEXT: v_cvt_f16_bf8_e64 v0.h, v0 byte_sel:3
-; GFX1250-GISEL-REAL16-NEXT: v_mov_b16_e32 v0.l, 0
+; GFX1250-GISEL-REAL16-NEXT: v_cvt_f16_bf8_e64 v0.l, v0 byte_sel:3
+; GFX1250-GISEL-REAL16-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-GISEL-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-REAL16-NEXT: v_mov_b16_e32 v1.h, v0.l
+; GFX1250-GISEL-REAL16-NEXT: v_mov_b32_e32 v0, v1
; GFX1250-GISEL-REAL16-NEXT: ; return to shader part epilog
;
; GFX1250-GISEL-FAKE16-LABEL: test_cvt_f16_bf8_byte3_hi:
@@ -488,8 +491,11 @@ define amdgpu_ps float @test_cvt_f16_fp8_byte3_hi(i32 %a) {
; GFX1250-GISEL-REAL16-NEXT: global_prefetch_b8 v0, null scope:SCOPE_SE
; GFX1250-GISEL-REAL16-NEXT: v_nop
; GFX1250-GISEL-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-REAL16-NEXT: v_cvt_f16_fp8_e64 v0.h, v0 byte_sel:3
-; GFX1250-GISEL-REAL16-NEXT: v_mov_b16_e32 v0.l, 0
+; GFX1250-GISEL-REAL16-NEXT: v_cvt_f16_fp8_e64 v0.l, v0 byte_sel:3
+; GFX1250-GISEL-REAL16-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-GISEL-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-REAL16-NEXT: v_mov_b16_e32 v1.h, v0.l
+; GFX1250-GISEL-REAL16-NEXT: v_mov_b32_e32 v0, v1
; GFX1250-GISEL-REAL16-NEXT: ; return to shader part epilog
;
; GFX1250-GISEL-FAKE16-LABEL: test_cvt_f16_fp8_byte3_hi:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fdot2.bf16.bf16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fdot2.bf16.bf16.ll
index e284f99fa05b6..452b83a7fbd36 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fdot2.bf16.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fdot2.bf16.bf16.ll
@@ -166,7 +166,7 @@ define amdgpu_ps void @test_llvm_amdgcn_fdot2_bf16_bf16_sis(
;
; GISEL-GFX11-TRUE16-LABEL: test_llvm_amdgcn_fdot2_bf16_bf16_sis:
; GISEL-GFX11-TRUE16: ; %bb.0: ; %entry
-; GISEL-GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, s1
+; GISEL-GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, s1
; GISEL-GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GISEL-GFX11-TRUE16-NEXT: v_dot2_bf16_bf16 v2.l, s0, 0x3f803f80, v2.l
; GISEL-GFX11-TRUE16-NEXT: global_store_b16 v[0:1], v2, off
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ptr.s.buffer.load-gfx12.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ptr.s.buffer.load-gfx12.ll
index d1391533271c7..2cba50062527a 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ptr.s.buffer.load-gfx12.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ptr.s.buffer.load-gfx12.ll
@@ -93,7 +93,7 @@ define amdgpu_kernel void @ptr_s_buffer_load_i16(ptr addrspace(1) %out, ptr addr
; GFX1200-GISEL-NEXT: s_wait_kmcnt 0x0
; GFX1200-GISEL-NEXT: s_buffer_load_u16 s0, s[0:3], s6 offset:0x0
; GFX1200-GISEL-NEXT: s_wait_kmcnt 0x0
-; GFX1200-GISEL-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX1200-GISEL-NEXT: v_mov_b32_e32 v0, s0
; GFX1200-GISEL-NEXT: global_store_b16 v1, v0, s[4:5]
; GFX1200-GISEL-NEXT: s_endpgm
;
@@ -124,7 +124,7 @@ define amdgpu_kernel void @ptr_s_buffer_load_i16(ptr addrspace(1) %out, ptr addr
; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0
; GFX1250-GISEL-NEXT: s_buffer_load_u16 s0, s[0:3], s8 offset:0x0 nv
; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0
-; GFX1250-GISEL-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX1250-GISEL-NEXT: v_mov_b32_e32 v0, s0
; GFX1250-GISEL-NEXT: global_store_b16 v1, v0, s[6:7]
; GFX1250-GISEL-NEXT: s_endpgm
%load = call i16 @llvm.amdgcn.ptr.s.buffer.load.i16(ptr addrspace(8) %rsrc, i32 %offset, i32 0), !invariant.load !0
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.buffer.load.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.buffer.load.ll
index bdfe37fb28463..1969ae6a47e7e 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.buffer.load.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.buffer.load.ll
@@ -1350,7 +1350,7 @@ define amdgpu_ps void @raw_buffer_load_f16(<4 x i32> inreg %rsrc, ptr addrspace(
; GFX12-GISEL-TRUE16-NEXT: buffer_load_u16 v1, off, s[0:3], null
; GFX12-GISEL-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-GISEL-TRUE16-NEXT: v_readfirstlane_b32 s0, v1
-; GFX12-GISEL-TRUE16-NEXT: v_mov_b16_e32 v1.l, s0
+; GFX12-GISEL-TRUE16-NEXT: v_mov_b32_e32 v1, s0
; GFX12-GISEL-TRUE16-NEXT: ds_store_b16 v0, v1
; GFX12-GISEL-TRUE16-NEXT: s_endpgm
main_body:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.add.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.add.ll
index 4d367c1eb3c06..c4a8301c05dd5 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.add.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.add.ll
@@ -183,21 +183,21 @@ define amdgpu_kernel void @uniform_value_i16(ptr addrspace(1) %out, i16 %in) {
; GFX1164DAGISEL-NEXT: global_store_b16 v0, v1, s[0:1]
; GFX1164DAGISEL-NEXT: s_endpgm
;
-; GFX1164GISEL-FAKE16-LABEL: uniform_value_i16:
-; GFX1164GISEL-FAKE16: ; %bb.0: ; %entry
-; GFX1164GISEL-FAKE16-NEXT: s_clause 0x1
-; GFX1164GISEL-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x2c
-; GFX1164GISEL-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1164GISEL-FAKE16-NEXT: s_mov_b64 s[2:3], exec
-; GFX1164GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, 0
-; GFX1164GISEL-FAKE16-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX1164GISEL-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164GISEL-FAKE16-NEXT: s_sext_i32_i16 s3, s6
-; GFX1164GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1164GISEL-FAKE16-NEXT: s_mul_i32 s2, s3, s2
-; GFX1164GISEL-FAKE16-NEXT: v_mov_b32_e32 v0, s2
-; GFX1164GISEL-FAKE16-NEXT: global_store_b16 v1, v0, s[0:1]
-; GFX1164GISEL-FAKE16-NEXT: s_endpgm
+; GFX1164GISEL-LABEL: uniform_value_i16:
+; GFX1164GISEL: ; %bb.0: ; %entry
+; GFX1164GISEL-NEXT: s_clause 0x1
+; GFX1164GISEL-NEXT: s_load_b32 s6, s[4:5], 0x2c
+; GFX1164GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1164GISEL-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164GISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX1164GISEL-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
+; GFX1164GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1164GISEL-NEXT: s_sext_i32_i16 s3, s6
+; GFX1164GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1164GISEL-NEXT: s_mul_i32 s2, s3, s2
+; GFX1164GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX1164GISEL-NEXT: global_store_b16 v1, v0, s[0:1]
+; GFX1164GISEL-NEXT: s_endpgm
;
; GFX1132DAGISEL-LABEL: uniform_value_i16:
; GFX1132DAGISEL: ; %bb.0: ; %entry
@@ -215,53 +215,21 @@ define amdgpu_kernel void @uniform_value_i16(ptr addrspace(1) %out, i16 %in) {
; GFX1132DAGISEL-NEXT: global_store_b16 v0, v1, s[0:1]
; GFX1132DAGISEL-NEXT: s_endpgm
;
-; GFX1132GISEL-FAKE16-LABEL: uniform_value_i16:
-; GFX1132GISEL-FAKE16: ; %bb.0: ; %entry
-; GFX1132GISEL-FAKE16-NEXT: s_clause 0x1
-; GFX1132GISEL-FAKE16-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX1132GISEL-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1132GISEL-FAKE16-NEXT: s_mov_b32 s3, exec_lo
-; GFX1132GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, 0
-; GFX1132GISEL-FAKE16-NEXT: s_bcnt1_i32_b32 s3, s3
-; GFX1132GISEL-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132GISEL-FAKE16-NEXT: s_sext_i32_i16 s2, s2
-; GFX1132GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1132GISEL-FAKE16-NEXT: s_mul_i32 s2, s2, s3
-; GFX1132GISEL-FAKE16-NEXT: v_mov_b32_e32 v0, s2
-; GFX1132GISEL-FAKE16-NEXT: global_store_b16 v1, v0, s[0:1]
-; GFX1132GISEL-FAKE16-NEXT: s_endpgm
-;
-; GFX1164GISEL-TRUE16-LABEL: uniform_value_i16:
-; GFX1164GISEL-TRUE16: ; %bb.0: ; %entry
-; GFX1164GISEL-TRUE16-NEXT: s_clause 0x1
-; GFX1164GISEL-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x2c
-; GFX1164GISEL-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1164GISEL-TRUE16-NEXT: s_mov_b64 s[2:3], exec
-; GFX1164GISEL-TRUE16-NEXT: v_mov_b32_e32 v1, 0
-; GFX1164GISEL-TRUE16-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX1164GISEL-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164GISEL-TRUE16-NEXT: s_sext_i32_i16 s3, s6
-; GFX1164GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1164GISEL-TRUE16-NEXT: s_mul_i32 s2, s3, s2
-; GFX1164GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.l, s2
-; GFX1164GISEL-TRUE16-NEXT: global_store_b16 v1, v0, s[0:1]
-; GFX1164GISEL-TRUE16-NEXT: s_endpgm
-;
-; GFX1132GISEL-TRUE16-LABEL: uniform_value_i16:
-; GFX1132GISEL-TRUE16: ; %bb.0: ; %entry
-; GFX1132GISEL-TRUE16-NEXT: s_clause 0x1
-; GFX1132GISEL-TRUE16-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX1132GISEL-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1132GISEL-TRUE16-NEXT: s_mov_b32 s3, exec_lo
-; GFX1132GISEL-TRUE16-NEXT: v_mov_b32_e32 v1, 0
-; GFX1132GISEL-TRUE16-NEXT: s_bcnt1_i32_b32 s3, s3
-; GFX1132GISEL-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132GISEL-TRUE16-NEXT: s_sext_i32_i16 s2, s2
-; GFX1132GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1132GISEL-TRUE16-NEXT: s_mul_i32 s2, s2, s3
-; GFX1132GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.l, s2
-; GFX1132GISEL-TRUE16-NEXT: global_store_b16 v1, v0, s[0:1]
-; GFX1132GISEL-TRUE16-NEXT: s_endpgm
+; GFX1132GISEL-LABEL: uniform_value_i16:
+; GFX1132GISEL: ; %bb.0: ; %entry
+; GFX1132GISEL-NEXT: s_clause 0x1
+; GFX1132GISEL-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GFX1132GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1132GISEL-NEXT: s_mov_b32 s3, exec_lo
+; GFX1132GISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX1132GISEL-NEXT: s_bcnt1_i32_b32 s3, s3
+; GFX1132GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1132GISEL-NEXT: s_sext_i32_i16 s2, s2
+; GFX1132GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1132GISEL-NEXT: s_mul_i32 s2, s2, s3
+; GFX1132GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX1132GISEL-NEXT: global_store_b16 v1, v0, s[0:1]
+; GFX1132GISEL-NEXT: s_endpgm
; GFX12DAGISEL-LABEL: uniform_value_i16:
; GFX12DAGISEL: ; %bb.0: ; %entry
; GFX12DAGISEL-NEXT: s_load_b96 s[0:2], s[4:5], 0x24
@@ -5156,5 +5124,9 @@ attributes #0 = { nounwind }
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
; GFX1132DAGISEL-FAKE16: {{.*}}
; GFX1132DAGISEL-TRUE16: {{.*}}
+; GFX1132GISEL-FAKE16: {{.*}}
+; GFX1132GISEL-TRUE16: {{.*}}
; GFX1164DAGISEL-FAKE16: {{.*}}
; GFX1164DAGISEL-TRUE16: {{.*}}
+; GFX1164GISEL-FAKE16: {{.*}}
+; GFX1164GISEL-TRUE16: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.and.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.and.ll
index c876aa657338d..9d5aa73de8324 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.and.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.and.ll
@@ -126,18 +126,18 @@ define amdgpu_kernel void @uniform_value_i16(ptr addrspace(1) %out, i16 %in) {
; GFX1164DAGISEL-NEXT: global_store_b16 v0, v1, s[0:1]
; GFX1164DAGISEL-NEXT: s_endpgm
;
-; GFX1164GISEL-FAKE16-LABEL: uniform_value_i16:
-; GFX1164GISEL-FAKE16: ; %bb.0: ; %entry
-; GFX1164GISEL-FAKE16-NEXT: s_clause 0x1
-; GFX1164GISEL-FAKE16-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX1164GISEL-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1164GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, 0
-; GFX1164GISEL-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164GISEL-FAKE16-NEXT: s_and_b32 s2, s2, 0xffff
-; GFX1164GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164GISEL-FAKE16-NEXT: v_mov_b32_e32 v0, s2
-; GFX1164GISEL-FAKE16-NEXT: global_store_b16 v1, v0, s[0:1]
-; GFX1164GISEL-FAKE16-NEXT: s_endpgm
+; GFX1164GISEL-LABEL: uniform_value_i16:
+; GFX1164GISEL: ; %bb.0: ; %entry
+; GFX1164GISEL-NEXT: s_clause 0x1
+; GFX1164GISEL-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GFX1164GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1164GISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX1164GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1164GISEL-NEXT: s_and_b32 s2, s2, 0xffff
+; GFX1164GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX1164GISEL-NEXT: global_store_b16 v1, v0, s[0:1]
+; GFX1164GISEL-NEXT: s_endpgm
;
; GFX1132DAGISEL-LABEL: uniform_value_i16:
; GFX1132DAGISEL: ; %bb.0: ; %entry
@@ -151,44 +151,18 @@ define amdgpu_kernel void @uniform_value_i16(ptr addrspace(1) %out, i16 %in) {
; GFX1132DAGISEL-NEXT: global_store_b16 v0, v1, s[0:1]
; GFX1132DAGISEL-NEXT: s_endpgm
;
-; GFX1132GISEL-FAKE16-LABEL: uniform_value_i16:
-; GFX1132GISEL-FAKE16: ; %bb.0: ; %entry
-; GFX1132GISEL-FAKE16-NEXT: s_clause 0x1
-; GFX1132GISEL-FAKE16-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX1132GISEL-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1132GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, 0
-; GFX1132GISEL-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132GISEL-FAKE16-NEXT: s_and_b32 s2, s2, 0xffff
-; GFX1132GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132GISEL-FAKE16-NEXT: v_mov_b32_e32 v0, s2
-; GFX1132GISEL-FAKE16-NEXT: global_store_b16 v1, v0, s[0:1]
-; GFX1132GISEL-FAKE16-NEXT: s_endpgm
-;
-; GFX1164GISEL-TRUE16-LABEL: uniform_value_i16:
-; GFX1164GISEL-TRUE16: ; %bb.0: ; %entry
-; GFX1164GISEL-TRUE16-NEXT: s_clause 0x1
-; GFX1164GISEL-TRUE16-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX1164GISEL-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1164GISEL-TRUE16-NEXT: v_mov_b32_e32 v1, 0
-; GFX1164GISEL-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164GISEL-TRUE16-NEXT: s_and_b32 s2, s2, 0xffff
-; GFX1164GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.l, s2
-; GFX1164GISEL-TRUE16-NEXT: global_store_b16 v1, v0, s[0:1]
-; GFX1164GISEL-TRUE16-NEXT: s_endpgm
-;
-; GFX1132GISEL-TRUE16-LABEL: uniform_value_i16:
-; GFX1132GISEL-TRUE16: ; %bb.0: ; %entry
-; GFX1132GISEL-TRUE16-NEXT: s_clause 0x1
-; GFX1132GISEL-TRUE16-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX1132GISEL-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1132GISEL-TRUE16-NEXT: v_mov_b32_e32 v1, 0
-; GFX1132GISEL-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132GISEL-TRUE16-NEXT: s_and_b32 s2, s2, 0xffff
-; GFX1132GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.l, s2
-; GFX1132GISEL-TRUE16-NEXT: global_store_b16 v1, v0, s[0:1]
-; GFX1132GISEL-TRUE16-NEXT: s_endpgm
+; GFX1132GISEL-LABEL: uniform_value_i16:
+; GFX1132GISEL: ; %bb.0: ; %entry
+; GFX1132GISEL-NEXT: s_clause 0x1
+; GFX1132GISEL-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GFX1132GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1132GISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX1132GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1132GISEL-NEXT: s_and_b32 s2, s2, 0xffff
+; GFX1132GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX1132GISEL-NEXT: global_store_b16 v1, v0, s[0:1]
+; GFX1132GISEL-NEXT: s_endpgm
entry:
%result = call i16 @llvm.amdgcn.wave.reduce.and.i16(i16 %in, i32 1)
store i16 %result, ptr addrspace(1) %out
@@ -4045,3 +4019,8 @@ endif:
}
attributes #0 = { nounwind }
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; GFX1132GISEL-FAKE16: {{.*}}
+; GFX1132GISEL-TRUE16: {{.*}}
+; GFX1164GISEL-FAKE16: {{.*}}
+; GFX1164GISEL-TRUE16: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.max.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.max.ll
index 7bc7f987bb2d4..65af1f6d6596c 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.max.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.max.ll
@@ -126,18 +126,18 @@ define amdgpu_kernel void @uniform_value_i16(ptr addrspace(1) %out, i16 %in) {
; GFX1164DAGISEL-NEXT: global_store_b16 v0, v1, s[0:1]
; GFX1164DAGISEL-NEXT: s_endpgm
;
-; GFX1164GISEL-FAKE16-LABEL: uniform_value_i16:
-; GFX1164GISEL-FAKE16: ; %bb.0: ; %entry
-; GFX1164GISEL-FAKE16-NEXT: s_clause 0x1
-; GFX1164GISEL-FAKE16-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX1164GISEL-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1164GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, 0
-; GFX1164GISEL-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164GISEL-FAKE16-NEXT: s_sext_i32_i16 s2, s2
-; GFX1164GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164GISEL-FAKE16-NEXT: v_mov_b32_e32 v0, s2
-; GFX1164GISEL-FAKE16-NEXT: global_store_b16 v1, v0, s[0:1]
-; GFX1164GISEL-FAKE16-NEXT: s_endpgm
+; GFX1164GISEL-LABEL: uniform_value_i16:
+; GFX1164GISEL: ; %bb.0: ; %entry
+; GFX1164GISEL-NEXT: s_clause 0x1
+; GFX1164GISEL-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GFX1164GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1164GISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX1164GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1164GISEL-NEXT: s_sext_i32_i16 s2, s2
+; GFX1164GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX1164GISEL-NEXT: global_store_b16 v1, v0, s[0:1]
+; GFX1164GISEL-NEXT: s_endpgm
;
; GFX1132DAGISEL-LABEL: uniform_value_i16:
; GFX1132DAGISEL: ; %bb.0: ; %entry
@@ -151,44 +151,18 @@ define amdgpu_kernel void @uniform_value_i16(ptr addrspace(1) %out, i16 %in) {
; GFX1132DAGISEL-NEXT: global_store_b16 v0, v1, s[0:1]
; GFX1132DAGISEL-NEXT: s_endpgm
;
-; GFX1132GISEL-FAKE16-LABEL: uniform_value_i16:
-; GFX1132GISEL-FAKE16: ; %bb.0: ; %entry
-; GFX1132GISEL-FAKE16-NEXT: s_clause 0x1
-; GFX1132GISEL-FAKE16-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX1132GISEL-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1132GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, 0
-; GFX1132GISEL-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132GISEL-FAKE16-NEXT: s_sext_i32_i16 s2, s2
-; GFX1132GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132GISEL-FAKE16-NEXT: v_mov_b32_e32 v0, s2
-; GFX1132GISEL-FAKE16-NEXT: global_store_b16 v1, v0, s[0:1]
-; GFX1132GISEL-FAKE16-NEXT: s_endpgm
-;
-; GFX1164GISEL-TRUE16-LABEL: uniform_value_i16:
-; GFX1164GISEL-TRUE16: ; %bb.0: ; %entry
-; GFX1164GISEL-TRUE16-NEXT: s_clause 0x1
-; GFX1164GISEL-TRUE16-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX1164GISEL-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1164GISEL-TRUE16-NEXT: v_mov_b32_e32 v1, 0
-; GFX1164GISEL-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164GISEL-TRUE16-NEXT: s_sext_i32_i16 s2, s2
-; GFX1164GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.l, s2
-; GFX1164GISEL-TRUE16-NEXT: global_store_b16 v1, v0, s[0:1]
-; GFX1164GISEL-TRUE16-NEXT: s_endpgm
-;
-; GFX1132GISEL-TRUE16-LABEL: uniform_value_i16:
-; GFX1132GISEL-TRUE16: ; %bb.0: ; %entry
-; GFX1132GISEL-TRUE16-NEXT: s_clause 0x1
-; GFX1132GISEL-TRUE16-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX1132GISEL-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1132GISEL-TRUE16-NEXT: v_mov_b32_e32 v1, 0
-; GFX1132GISEL-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132GISEL-TRUE16-NEXT: s_sext_i32_i16 s2, s2
-; GFX1132GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.l, s2
-; GFX1132GISEL-TRUE16-NEXT: global_store_b16 v1, v0, s[0:1]
-; GFX1132GISEL-TRUE16-NEXT: s_endpgm
+; GFX1132GISEL-LABEL: uniform_value_i16:
+; GFX1132GISEL: ; %bb.0: ; %entry
+; GFX1132GISEL-NEXT: s_clause 0x1
+; GFX1132GISEL-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GFX1132GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1132GISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX1132GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1132GISEL-NEXT: s_sext_i32_i16 s2, s2
+; GFX1132GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX1132GISEL-NEXT: global_store_b16 v1, v0, s[0:1]
+; GFX1132GISEL-NEXT: s_endpgm
entry:
%result = call i16 @llvm.amdgcn.wave.reduce.max.i16(i16 %in, i32 1)
store i16 %result, ptr addrspace(1) %out
@@ -4220,5 +4194,9 @@ attributes #0 = { nounwind }
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
; GFX1132DAGISEL-FAKE16: {{.*}}
; GFX1132DAGISEL-TRUE16: {{.*}}
+; GFX1132GISEL-FAKE16: {{.*}}
+; GFX1132GISEL-TRUE16: {{.*}}
; GFX1164DAGISEL-FAKE16: {{.*}}
; GFX1164DAGISEL-TRUE16: {{.*}}
+; GFX1164GISEL-FAKE16: {{.*}}
+; GFX1164GISEL-TRUE16: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.min.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.min.ll
index 6f472a6e18584..f42da8cd7df37 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.min.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.min.ll
@@ -126,18 +126,18 @@ define amdgpu_kernel void @uniform_value_i16(ptr addrspace(1) %out, i16 %in) {
; GFX1164DAGISEL-NEXT: global_store_b16 v0, v1, s[0:1]
; GFX1164DAGISEL-NEXT: s_endpgm
;
-; GFX1164GISEL-FAKE16-LABEL: uniform_value_i16:
-; GFX1164GISEL-FAKE16: ; %bb.0: ; %entry
-; GFX1164GISEL-FAKE16-NEXT: s_clause 0x1
-; GFX1164GISEL-FAKE16-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX1164GISEL-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1164GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, 0
-; GFX1164GISEL-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164GISEL-FAKE16-NEXT: s_sext_i32_i16 s2, s2
-; GFX1164GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164GISEL-FAKE16-NEXT: v_mov_b32_e32 v0, s2
-; GFX1164GISEL-FAKE16-NEXT: global_store_b16 v1, v0, s[0:1]
-; GFX1164GISEL-FAKE16-NEXT: s_endpgm
+; GFX1164GISEL-LABEL: uniform_value_i16:
+; GFX1164GISEL: ; %bb.0: ; %entry
+; GFX1164GISEL-NEXT: s_clause 0x1
+; GFX1164GISEL-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GFX1164GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1164GISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX1164GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1164GISEL-NEXT: s_sext_i32_i16 s2, s2
+; GFX1164GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX1164GISEL-NEXT: global_store_b16 v1, v0, s[0:1]
+; GFX1164GISEL-NEXT: s_endpgm
;
; GFX1132DAGISEL-LABEL: uniform_value_i16:
; GFX1132DAGISEL: ; %bb.0: ; %entry
@@ -151,44 +151,18 @@ define amdgpu_kernel void @uniform_value_i16(ptr addrspace(1) %out, i16 %in) {
; GFX1132DAGISEL-NEXT: global_store_b16 v0, v1, s[0:1]
; GFX1132DAGISEL-NEXT: s_endpgm
;
-; GFX1132GISEL-FAKE16-LABEL: uniform_value_i16:
-; GFX1132GISEL-FAKE16: ; %bb.0: ; %entry
-; GFX1132GISEL-FAKE16-NEXT: s_clause 0x1
-; GFX1132GISEL-FAKE16-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX1132GISEL-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1132GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, 0
-; GFX1132GISEL-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132GISEL-FAKE16-NEXT: s_sext_i32_i16 s2, s2
-; GFX1132GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132GISEL-FAKE16-NEXT: v_mov_b32_e32 v0, s2
-; GFX1132GISEL-FAKE16-NEXT: global_store_b16 v1, v0, s[0:1]
-; GFX1132GISEL-FAKE16-NEXT: s_endpgm
-;
-; GFX1164GISEL-TRUE16-LABEL: uniform_value_i16:
-; GFX1164GISEL-TRUE16: ; %bb.0: ; %entry
-; GFX1164GISEL-TRUE16-NEXT: s_clause 0x1
-; GFX1164GISEL-TRUE16-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX1164GISEL-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1164GISEL-TRUE16-NEXT: v_mov_b32_e32 v1, 0
-; GFX1164GISEL-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164GISEL-TRUE16-NEXT: s_sext_i32_i16 s2, s2
-; GFX1164GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.l, s2
-; GFX1164GISEL-TRUE16-NEXT: global_store_b16 v1, v0, s[0:1]
-; GFX1164GISEL-TRUE16-NEXT: s_endpgm
-;
-; GFX1132GISEL-TRUE16-LABEL: uniform_value_i16:
-; GFX1132GISEL-TRUE16: ; %bb.0: ; %entry
-; GFX1132GISEL-TRUE16-NEXT: s_clause 0x1
-; GFX1132GISEL-TRUE16-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX1132GISEL-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1132GISEL-TRUE16-NEXT: v_mov_b32_e32 v1, 0
-; GFX1132GISEL-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132GISEL-TRUE16-NEXT: s_sext_i32_i16 s2, s2
-; GFX1132GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.l, s2
-; GFX1132GISEL-TRUE16-NEXT: global_store_b16 v1, v0, s[0:1]
-; GFX1132GISEL-TRUE16-NEXT: s_endpgm
+; GFX1132GISEL-LABEL: uniform_value_i16:
+; GFX1132GISEL: ; %bb.0: ; %entry
+; GFX1132GISEL-NEXT: s_clause 0x1
+; GFX1132GISEL-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GFX1132GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1132GISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX1132GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1132GISEL-NEXT: s_sext_i32_i16 s2, s2
+; GFX1132GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX1132GISEL-NEXT: global_store_b16 v1, v0, s[0:1]
+; GFX1132GISEL-NEXT: s_endpgm
entry:
%result = call i16 @llvm.amdgcn.wave.reduce.min.i16(i16 %in, i32 1)
store i16 %result, ptr addrspace(1) %out
@@ -4220,5 +4194,9 @@ attributes #0 = { nounwind }
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
; GFX1132DAGISEL-FAKE16: {{.*}}
; GFX1132DAGISEL-TRUE16: {{.*}}
+; GFX1132GISEL-FAKE16: {{.*}}
+; GFX1132GISEL-TRUE16: {{.*}}
; GFX1164DAGISEL-FAKE16: {{.*}}
; GFX1164DAGISEL-TRUE16: {{.*}}
+; GFX1164GISEL-FAKE16: {{.*}}
+; GFX1164GISEL-TRUE16: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.or.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.or.ll
index 1f3d19668f86b..75a6c7d580573 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.or.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.or.ll
@@ -126,18 +126,18 @@ define amdgpu_kernel void @uniform_value_i16(ptr addrspace(1) %out, i16 %in) {
; GFX1164DAGISEL-NEXT: global_store_b16 v0, v1, s[0:1]
; GFX1164DAGISEL-NEXT: s_endpgm
;
-; GFX1164GISEL-FAKE16-LABEL: uniform_value_i16:
-; GFX1164GISEL-FAKE16: ; %bb.0: ; %entry
-; GFX1164GISEL-FAKE16-NEXT: s_clause 0x1
-; GFX1164GISEL-FAKE16-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX1164GISEL-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1164GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, 0
-; GFX1164GISEL-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164GISEL-FAKE16-NEXT: s_and_b32 s2, s2, 0xffff
-; GFX1164GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164GISEL-FAKE16-NEXT: v_mov_b32_e32 v0, s2
-; GFX1164GISEL-FAKE16-NEXT: global_store_b16 v1, v0, s[0:1]
-; GFX1164GISEL-FAKE16-NEXT: s_endpgm
+; GFX1164GISEL-LABEL: uniform_value_i16:
+; GFX1164GISEL: ; %bb.0: ; %entry
+; GFX1164GISEL-NEXT: s_clause 0x1
+; GFX1164GISEL-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GFX1164GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1164GISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX1164GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1164GISEL-NEXT: s_and_b32 s2, s2, 0xffff
+; GFX1164GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX1164GISEL-NEXT: global_store_b16 v1, v0, s[0:1]
+; GFX1164GISEL-NEXT: s_endpgm
;
; GFX1132DAGISEL-LABEL: uniform_value_i16:
; GFX1132DAGISEL: ; %bb.0: ; %entry
@@ -151,44 +151,18 @@ define amdgpu_kernel void @uniform_value_i16(ptr addrspace(1) %out, i16 %in) {
; GFX1132DAGISEL-NEXT: global_store_b16 v0, v1, s[0:1]
; GFX1132DAGISEL-NEXT: s_endpgm
;
-; GFX1132GISEL-FAKE16-LABEL: uniform_value_i16:
-; GFX1132GISEL-FAKE16: ; %bb.0: ; %entry
-; GFX1132GISEL-FAKE16-NEXT: s_clause 0x1
-; GFX1132GISEL-FAKE16-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX1132GISEL-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1132GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, 0
-; GFX1132GISEL-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132GISEL-FAKE16-NEXT: s_and_b32 s2, s2, 0xffff
-; GFX1132GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132GISEL-FAKE16-NEXT: v_mov_b32_e32 v0, s2
-; GFX1132GISEL-FAKE16-NEXT: global_store_b16 v1, v0, s[0:1]
-; GFX1132GISEL-FAKE16-NEXT: s_endpgm
-;
-; GFX1164GISEL-TRUE16-LABEL: uniform_value_i16:
-; GFX1164GISEL-TRUE16: ; %bb.0: ; %entry
-; GFX1164GISEL-TRUE16-NEXT: s_clause 0x1
-; GFX1164GISEL-TRUE16-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX1164GISEL-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1164GISEL-TRUE16-NEXT: v_mov_b32_e32 v1, 0
-; GFX1164GISEL-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164GISEL-TRUE16-NEXT: s_and_b32 s2, s2, 0xffff
-; GFX1164GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.l, s2
-; GFX1164GISEL-TRUE16-NEXT: global_store_b16 v1, v0, s[0:1]
-; GFX1164GISEL-TRUE16-NEXT: s_endpgm
-;
-; GFX1132GISEL-TRUE16-LABEL: uniform_value_i16:
-; GFX1132GISEL-TRUE16: ; %bb.0: ; %entry
-; GFX1132GISEL-TRUE16-NEXT: s_clause 0x1
-; GFX1132GISEL-TRUE16-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX1132GISEL-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1132GISEL-TRUE16-NEXT: v_mov_b32_e32 v1, 0
-; GFX1132GISEL-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132GISEL-TRUE16-NEXT: s_and_b32 s2, s2, 0xffff
-; GFX1132GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.l, s2
-; GFX1132GISEL-TRUE16-NEXT: global_store_b16 v1, v0, s[0:1]
-; GFX1132GISEL-TRUE16-NEXT: s_endpgm
+; GFX1132GISEL-LABEL: uniform_value_i16:
+; GFX1132GISEL: ; %bb.0: ; %entry
+; GFX1132GISEL-NEXT: s_clause 0x1
+; GFX1132GISEL-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GFX1132GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1132GISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX1132GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1132GISEL-NEXT: s_and_b32 s2, s2, 0xffff
+; GFX1132GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX1132GISEL-NEXT: global_store_b16 v1, v0, s[0:1]
+; GFX1132GISEL-NEXT: s_endpgm
entry:
%result = call i16 @llvm.amdgcn.wave.reduce.or.i16(i16 %in, i32 1)
store i16 %result, ptr addrspace(1) %out
@@ -4046,3 +4020,8 @@ endif:
}
attributes #0 = { nounwind }
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; GFX1132GISEL-FAKE16: {{.*}}
+; GFX1132GISEL-TRUE16: {{.*}}
+; GFX1164GISEL-FAKE16: {{.*}}
+; GFX1164GISEL-TRUE16: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.sub.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.sub.ll
index ae2dcfac5a4d0..58e47554ff89e 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.sub.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.sub.ll
@@ -195,23 +195,23 @@ define amdgpu_kernel void @uniform_value_i16(ptr addrspace(1) %out, i16 %in) {
; GFX1164DAGISEL-NEXT: global_store_b16 v0, v1, s[0:1]
; GFX1164DAGISEL-NEXT: s_endpgm
;
-; GFX1164GISEL-FAKE16-LABEL: uniform_value_i16:
-; GFX1164GISEL-FAKE16: ; %bb.0: ; %entry
-; GFX1164GISEL-FAKE16-NEXT: s_clause 0x1
-; GFX1164GISEL-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x2c
-; GFX1164GISEL-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1164GISEL-FAKE16-NEXT: s_mov_b64 s[2:3], exec
-; GFX1164GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, 0
-; GFX1164GISEL-FAKE16-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX1164GISEL-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164GISEL-FAKE16-NEXT: s_sext_i32_i16 s4, s6
-; GFX1164GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1164GISEL-FAKE16-NEXT: s_sub_i32 s3, 0, s4
-; GFX1164GISEL-FAKE16-NEXT: s_mul_i32 s2, s3, s2
-; GFX1164GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164GISEL-FAKE16-NEXT: v_mov_b32_e32 v0, s2
-; GFX1164GISEL-FAKE16-NEXT: global_store_b16 v1, v0, s[0:1]
-; GFX1164GISEL-FAKE16-NEXT: s_endpgm
+; GFX1164GISEL-LABEL: uniform_value_i16:
+; GFX1164GISEL: ; %bb.0: ; %entry
+; GFX1164GISEL-NEXT: s_clause 0x1
+; GFX1164GISEL-NEXT: s_load_b32 s6, s[4:5], 0x2c
+; GFX1164GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1164GISEL-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164GISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX1164GISEL-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
+; GFX1164GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1164GISEL-NEXT: s_sext_i32_i16 s4, s6
+; GFX1164GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1164GISEL-NEXT: s_sub_i32 s3, 0, s4
+; GFX1164GISEL-NEXT: s_mul_i32 s2, s3, s2
+; GFX1164GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX1164GISEL-NEXT: global_store_b16 v1, v0, s[0:1]
+; GFX1164GISEL-NEXT: s_endpgm
;
; GFX1132DAGISEL-LABEL: uniform_value_i16:
; GFX1132DAGISEL: ; %bb.0: ; %entry
@@ -230,59 +230,23 @@ define amdgpu_kernel void @uniform_value_i16(ptr addrspace(1) %out, i16 %in) {
; GFX1132DAGISEL-NEXT: global_store_b16 v0, v1, s[0:1]
; GFX1132DAGISEL-NEXT: s_endpgm
;
-; GFX1132GISEL-FAKE16-LABEL: uniform_value_i16:
-; GFX1132GISEL-FAKE16: ; %bb.0: ; %entry
-; GFX1132GISEL-FAKE16-NEXT: s_clause 0x1
-; GFX1132GISEL-FAKE16-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX1132GISEL-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1132GISEL-FAKE16-NEXT: s_mov_b32 s3, exec_lo
-; GFX1132GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, 0
-; GFX1132GISEL-FAKE16-NEXT: s_bcnt1_i32_b32 s3, s3
-; GFX1132GISEL-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132GISEL-FAKE16-NEXT: s_sext_i32_i16 s2, s2
-; GFX1132GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1132GISEL-FAKE16-NEXT: s_sub_i32 s2, 0, s2
-; GFX1132GISEL-FAKE16-NEXT: s_mul_i32 s2, s2, s3
-; GFX1132GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132GISEL-FAKE16-NEXT: v_mov_b32_e32 v0, s2
-; GFX1132GISEL-FAKE16-NEXT: global_store_b16 v1, v0, s[0:1]
-; GFX1132GISEL-FAKE16-NEXT: s_endpgm
-;
-; GFX1164GISEL-TRUE16-LABEL: uniform_value_i16:
-; GFX1164GISEL-TRUE16: ; %bb.0: ; %entry
-; GFX1164GISEL-TRUE16-NEXT: s_clause 0x1
-; GFX1164GISEL-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x2c
-; GFX1164GISEL-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1164GISEL-TRUE16-NEXT: s_mov_b64 s[2:3], exec
-; GFX1164GISEL-TRUE16-NEXT: v_mov_b32_e32 v1, 0
-; GFX1164GISEL-TRUE16-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX1164GISEL-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164GISEL-TRUE16-NEXT: s_sext_i32_i16 s4, s6
-; GFX1164GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1164GISEL-TRUE16-NEXT: s_sub_i32 s3, 0, s4
-; GFX1164GISEL-TRUE16-NEXT: s_mul_i32 s2, s3, s2
-; GFX1164GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.l, s2
-; GFX1164GISEL-TRUE16-NEXT: global_store_b16 v1, v0, s[0:1]
-; GFX1164GISEL-TRUE16-NEXT: s_endpgm
-;
-; GFX1132GISEL-TRUE16-LABEL: uniform_value_i16:
-; GFX1132GISEL-TRUE16: ; %bb.0: ; %entry
-; GFX1132GISEL-TRUE16-NEXT: s_clause 0x1
-; GFX1132GISEL-TRUE16-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX1132GISEL-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1132GISEL-TRUE16-NEXT: s_mov_b32 s3, exec_lo
-; GFX1132GISEL-TRUE16-NEXT: v_mov_b32_e32 v1, 0
-; GFX1132GISEL-TRUE16-NEXT: s_bcnt1_i32_b32 s3, s3
-; GFX1132GISEL-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132GISEL-TRUE16-NEXT: s_sext_i32_i16 s2, s2
-; GFX1132GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1132GISEL-TRUE16-NEXT: s_sub_i32 s2, 0, s2
-; GFX1132GISEL-TRUE16-NEXT: s_mul_i32 s2, s2, s3
-; GFX1132GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.l, s2
-; GFX1132GISEL-TRUE16-NEXT: global_store_b16 v1, v0, s[0:1]
-; GFX1132GISEL-TRUE16-NEXT: s_endpgm
+; GFX1132GISEL-LABEL: uniform_value_i16:
+; GFX1132GISEL: ; %bb.0: ; %entry
+; GFX1132GISEL-NEXT: s_clause 0x1
+; GFX1132GISEL-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GFX1132GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1132GISEL-NEXT: s_mov_b32 s3, exec_lo
+; GFX1132GISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX1132GISEL-NEXT: s_bcnt1_i32_b32 s3, s3
+; GFX1132GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1132GISEL-NEXT: s_sext_i32_i16 s2, s2
+; GFX1132GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1132GISEL-NEXT: s_sub_i32 s2, 0, s2
+; GFX1132GISEL-NEXT: s_mul_i32 s2, s2, s3
+; GFX1132GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX1132GISEL-NEXT: global_store_b16 v1, v0, s[0:1]
+; GFX1132GISEL-NEXT: s_endpgm
; GFX12DAGISEL-LABEL: uniform_value_i16:
; GFX12DAGISEL: ; %bb.0: ; %entry
; GFX12DAGISEL-NEXT: s_load_b96 s[0:2], s[4:5], 0x24
@@ -5287,5 +5251,9 @@ attributes #0 = { nounwind }
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
; GFX1132DAGISEL-FAKE16: {{.*}}
; GFX1132DAGISEL-TRUE16: {{.*}}
+; GFX1132GISEL-FAKE16: {{.*}}
+; GFX1132GISEL-TRUE16: {{.*}}
; GFX1164DAGISEL-FAKE16: {{.*}}
; GFX1164DAGISEL-TRUE16: {{.*}}
+; GFX1164GISEL-FAKE16: {{.*}}
+; GFX1164GISEL-TRUE16: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.umax.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.umax.ll
index 69b24c6f1b17a..5f12243bd0cec 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.umax.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.umax.ll
@@ -126,18 +126,18 @@ define amdgpu_kernel void @uniform_value_i16(ptr addrspace(1) %out, i16 %in) {
; GFX1164DAGISEL-NEXT: global_store_b16 v0, v1, s[0:1]
; GFX1164DAGISEL-NEXT: s_endpgm
;
-; GFX1164GISEL-FAKE16-LABEL: uniform_value_i16:
-; GFX1164GISEL-FAKE16: ; %bb.0: ; %entry
-; GFX1164GISEL-FAKE16-NEXT: s_clause 0x1
-; GFX1164GISEL-FAKE16-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX1164GISEL-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1164GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, 0
-; GFX1164GISEL-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164GISEL-FAKE16-NEXT: s_and_b32 s2, s2, 0xffff
-; GFX1164GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164GISEL-FAKE16-NEXT: v_mov_b32_e32 v0, s2
-; GFX1164GISEL-FAKE16-NEXT: global_store_b16 v1, v0, s[0:1]
-; GFX1164GISEL-FAKE16-NEXT: s_endpgm
+; GFX1164GISEL-LABEL: uniform_value_i16:
+; GFX1164GISEL: ; %bb.0: ; %entry
+; GFX1164GISEL-NEXT: s_clause 0x1
+; GFX1164GISEL-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GFX1164GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1164GISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX1164GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1164GISEL-NEXT: s_and_b32 s2, s2, 0xffff
+; GFX1164GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX1164GISEL-NEXT: global_store_b16 v1, v0, s[0:1]
+; GFX1164GISEL-NEXT: s_endpgm
;
; GFX1132DAGISEL-LABEL: uniform_value_i16:
; GFX1132DAGISEL: ; %bb.0: ; %entry
@@ -151,44 +151,18 @@ define amdgpu_kernel void @uniform_value_i16(ptr addrspace(1) %out, i16 %in) {
; GFX1132DAGISEL-NEXT: global_store_b16 v0, v1, s[0:1]
; GFX1132DAGISEL-NEXT: s_endpgm
;
-; GFX1132GISEL-FAKE16-LABEL: uniform_value_i16:
-; GFX1132GISEL-FAKE16: ; %bb.0: ; %entry
-; GFX1132GISEL-FAKE16-NEXT: s_clause 0x1
-; GFX1132GISEL-FAKE16-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX1132GISEL-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1132GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, 0
-; GFX1132GISEL-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132GISEL-FAKE16-NEXT: s_and_b32 s2, s2, 0xffff
-; GFX1132GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132GISEL-FAKE16-NEXT: v_mov_b32_e32 v0, s2
-; GFX1132GISEL-FAKE16-NEXT: global_store_b16 v1, v0, s[0:1]
-; GFX1132GISEL-FAKE16-NEXT: s_endpgm
-;
-; GFX1164GISEL-TRUE16-LABEL: uniform_value_i16:
-; GFX1164GISEL-TRUE16: ; %bb.0: ; %entry
-; GFX1164GISEL-TRUE16-NEXT: s_clause 0x1
-; GFX1164GISEL-TRUE16-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX1164GISEL-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1164GISEL-TRUE16-NEXT: v_mov_b32_e32 v1, 0
-; GFX1164GISEL-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164GISEL-TRUE16-NEXT: s_and_b32 s2, s2, 0xffff
-; GFX1164GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.l, s2
-; GFX1164GISEL-TRUE16-NEXT: global_store_b16 v1, v0, s[0:1]
-; GFX1164GISEL-TRUE16-NEXT: s_endpgm
-;
-; GFX1132GISEL-TRUE16-LABEL: uniform_value_i16:
-; GFX1132GISEL-TRUE16: ; %bb.0: ; %entry
-; GFX1132GISEL-TRUE16-NEXT: s_clause 0x1
-; GFX1132GISEL-TRUE16-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX1132GISEL-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1132GISEL-TRUE16-NEXT: v_mov_b32_e32 v1, 0
-; GFX1132GISEL-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132GISEL-TRUE16-NEXT: s_and_b32 s2, s2, 0xffff
-; GFX1132GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.l, s2
-; GFX1132GISEL-TRUE16-NEXT: global_store_b16 v1, v0, s[0:1]
-; GFX1132GISEL-TRUE16-NEXT: s_endpgm
+; GFX1132GISEL-LABEL: uniform_value_i16:
+; GFX1132GISEL: ; %bb.0: ; %entry
+; GFX1132GISEL-NEXT: s_clause 0x1
+; GFX1132GISEL-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GFX1132GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1132GISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX1132GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1132GISEL-NEXT: s_and_b32 s2, s2, 0xffff
+; GFX1132GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX1132GISEL-NEXT: global_store_b16 v1, v0, s[0:1]
+; GFX1132GISEL-NEXT: s_endpgm
; GFX11GISEL-LABEL: uniform_value_i16:
; GFX11GISEL: ; %bb.0: ; %entry
; GFX11GISEL-NEXT: s_clause 0x1
@@ -4387,3 +4361,8 @@ endif:
}
attributes #0 = { nounwind }
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; GFX1132GISEL-FAKE16: {{.*}}
+; GFX1132GISEL-TRUE16: {{.*}}
+; GFX1164GISEL-FAKE16: {{.*}}
+; GFX1164GISEL-TRUE16: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.umin.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.umin.ll
index 7a3a939891089..12357cc9e8726 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.umin.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.umin.ll
@@ -126,18 +126,18 @@ define amdgpu_kernel void @uniform_value_i16(ptr addrspace(1) %out, i16 %in) {
; GFX1164DAGISEL-NEXT: global_store_b16 v0, v1, s[0:1]
; GFX1164DAGISEL-NEXT: s_endpgm
;
-; GFX1164GISEL-FAKE16-LABEL: uniform_value_i16:
-; GFX1164GISEL-FAKE16: ; %bb.0: ; %entry
-; GFX1164GISEL-FAKE16-NEXT: s_clause 0x1
-; GFX1164GISEL-FAKE16-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX1164GISEL-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1164GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, 0
-; GFX1164GISEL-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164GISEL-FAKE16-NEXT: s_and_b32 s2, s2, 0xffff
-; GFX1164GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164GISEL-FAKE16-NEXT: v_mov_b32_e32 v0, s2
-; GFX1164GISEL-FAKE16-NEXT: global_store_b16 v1, v0, s[0:1]
-; GFX1164GISEL-FAKE16-NEXT: s_endpgm
+; GFX1164GISEL-LABEL: uniform_value_i16:
+; GFX1164GISEL: ; %bb.0: ; %entry
+; GFX1164GISEL-NEXT: s_clause 0x1
+; GFX1164GISEL-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GFX1164GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1164GISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX1164GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1164GISEL-NEXT: s_and_b32 s2, s2, 0xffff
+; GFX1164GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX1164GISEL-NEXT: global_store_b16 v1, v0, s[0:1]
+; GFX1164GISEL-NEXT: s_endpgm
;
; GFX1132DAGISEL-LABEL: uniform_value_i16:
; GFX1132DAGISEL: ; %bb.0: ; %entry
@@ -151,44 +151,18 @@ define amdgpu_kernel void @uniform_value_i16(ptr addrspace(1) %out, i16 %in) {
; GFX1132DAGISEL-NEXT: global_store_b16 v0, v1, s[0:1]
; GFX1132DAGISEL-NEXT: s_endpgm
;
-; GFX1132GISEL-FAKE16-LABEL: uniform_value_i16:
-; GFX1132GISEL-FAKE16: ; %bb.0: ; %entry
-; GFX1132GISEL-FAKE16-NEXT: s_clause 0x1
-; GFX1132GISEL-FAKE16-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX1132GISEL-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1132GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, 0
-; GFX1132GISEL-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132GISEL-FAKE16-NEXT: s_and_b32 s2, s2, 0xffff
-; GFX1132GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132GISEL-FAKE16-NEXT: v_mov_b32_e32 v0, s2
-; GFX1132GISEL-FAKE16-NEXT: global_store_b16 v1, v0, s[0:1]
-; GFX1132GISEL-FAKE16-NEXT: s_endpgm
-;
-; GFX1164GISEL-TRUE16-LABEL: uniform_value_i16:
-; GFX1164GISEL-TRUE16: ; %bb.0: ; %entry
-; GFX1164GISEL-TRUE16-NEXT: s_clause 0x1
-; GFX1164GISEL-TRUE16-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX1164GISEL-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1164GISEL-TRUE16-NEXT: v_mov_b32_e32 v1, 0
-; GFX1164GISEL-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164GISEL-TRUE16-NEXT: s_and_b32 s2, s2, 0xffff
-; GFX1164GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.l, s2
-; GFX1164GISEL-TRUE16-NEXT: global_store_b16 v1, v0, s[0:1]
-; GFX1164GISEL-TRUE16-NEXT: s_endpgm
-;
-; GFX1132GISEL-TRUE16-LABEL: uniform_value_i16:
-; GFX1132GISEL-TRUE16: ; %bb.0: ; %entry
-; GFX1132GISEL-TRUE16-NEXT: s_clause 0x1
-; GFX1132GISEL-TRUE16-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX1132GISEL-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1132GISEL-TRUE16-NEXT: v_mov_b32_e32 v1, 0
-; GFX1132GISEL-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132GISEL-TRUE16-NEXT: s_and_b32 s2, s2, 0xffff
-; GFX1132GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.l, s2
-; GFX1132GISEL-TRUE16-NEXT: global_store_b16 v1, v0, s[0:1]
-; GFX1132GISEL-TRUE16-NEXT: s_endpgm
+; GFX1132GISEL-LABEL: uniform_value_i16:
+; GFX1132GISEL: ; %bb.0: ; %entry
+; GFX1132GISEL-NEXT: s_clause 0x1
+; GFX1132GISEL-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GFX1132GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1132GISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX1132GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1132GISEL-NEXT: s_and_b32 s2, s2, 0xffff
+; GFX1132GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX1132GISEL-NEXT: global_store_b16 v1, v0, s[0:1]
+; GFX1132GISEL-NEXT: s_endpgm
entry:
%result = call i16 @llvm.amdgcn.wave.reduce.umin.i16(i16 %in, i32 1)
store i16 %result, ptr addrspace(1) %out
@@ -4191,3 +4165,8 @@ endif:
}
attributes #0 = { nounwind }
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; GFX1132GISEL-FAKE16: {{.*}}
+; GFX1132GISEL-TRUE16: {{.*}}
+; GFX1164GISEL-FAKE16: {{.*}}
+; GFX1164GISEL-TRUE16: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.xor.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.xor.ll
index 7d6c296515733..8e4d7708091b3 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.xor.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.xor.ll
@@ -195,23 +195,23 @@ define amdgpu_kernel void @uniform_value_i16(ptr addrspace(1) %out, i16 %in) {
; GFX1164DAGISEL-NEXT: global_store_b16 v0, v1, s[0:1]
; GFX1164DAGISEL-NEXT: s_endpgm
;
-; GFX1164GISEL-FAKE16-LABEL: uniform_value_i16:
-; GFX1164GISEL-FAKE16: ; %bb.0: ; %entry
-; GFX1164GISEL-FAKE16-NEXT: s_clause 0x1
-; GFX1164GISEL-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x2c
-; GFX1164GISEL-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1164GISEL-FAKE16-NEXT: s_mov_b64 s[2:3], exec
-; GFX1164GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, 0
-; GFX1164GISEL-FAKE16-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX1164GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX1164GISEL-FAKE16-NEXT: s_and_b32 s2, s2, 1
-; GFX1164GISEL-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164GISEL-FAKE16-NEXT: s_and_b32 s3, s6, 0xffff
-; GFX1164GISEL-FAKE16-NEXT: s_mul_i32 s2, s3, s2
-; GFX1164GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164GISEL-FAKE16-NEXT: v_mov_b32_e32 v0, s2
-; GFX1164GISEL-FAKE16-NEXT: global_store_b16 v1, v0, s[0:1]
-; GFX1164GISEL-FAKE16-NEXT: s_endpgm
+; GFX1164GISEL-LABEL: uniform_value_i16:
+; GFX1164GISEL: ; %bb.0: ; %entry
+; GFX1164GISEL-NEXT: s_clause 0x1
+; GFX1164GISEL-NEXT: s_load_b32 s6, s[4:5], 0x2c
+; GFX1164GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1164GISEL-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164GISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX1164GISEL-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
+; GFX1164GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX1164GISEL-NEXT: s_and_b32 s2, s2, 1
+; GFX1164GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1164GISEL-NEXT: s_and_b32 s3, s6, 0xffff
+; GFX1164GISEL-NEXT: s_mul_i32 s2, s3, s2
+; GFX1164GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX1164GISEL-NEXT: global_store_b16 v1, v0, s[0:1]
+; GFX1164GISEL-NEXT: s_endpgm
;
; GFX1132DAGISEL-LABEL: uniform_value_i16:
; GFX1132DAGISEL: ; %bb.0: ; %entry
@@ -230,59 +230,23 @@ define amdgpu_kernel void @uniform_value_i16(ptr addrspace(1) %out, i16 %in) {
; GFX1132DAGISEL-NEXT: global_store_b16 v0, v1, s[0:1]
; GFX1132DAGISEL-NEXT: s_endpgm
;
-; GFX1132GISEL-FAKE16-LABEL: uniform_value_i16:
-; GFX1132GISEL-FAKE16: ; %bb.0: ; %entry
-; GFX1132GISEL-FAKE16-NEXT: s_clause 0x1
-; GFX1132GISEL-FAKE16-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX1132GISEL-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1132GISEL-FAKE16-NEXT: s_mov_b32 s3, exec_lo
-; GFX1132GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, 0
-; GFX1132GISEL-FAKE16-NEXT: s_bcnt1_i32_b32 s3, s3
-; GFX1132GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX1132GISEL-FAKE16-NEXT: s_and_b32 s3, s3, 1
-; GFX1132GISEL-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132GISEL-FAKE16-NEXT: s_and_b32 s2, s2, 0xffff
-; GFX1132GISEL-FAKE16-NEXT: s_mul_i32 s2, s2, s3
-; GFX1132GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132GISEL-FAKE16-NEXT: v_mov_b32_e32 v0, s2
-; GFX1132GISEL-FAKE16-NEXT: global_store_b16 v1, v0, s[0:1]
-; GFX1132GISEL-FAKE16-NEXT: s_endpgm
-;
-; GFX1164GISEL-TRUE16-LABEL: uniform_value_i16:
-; GFX1164GISEL-TRUE16: ; %bb.0: ; %entry
-; GFX1164GISEL-TRUE16-NEXT: s_clause 0x1
-; GFX1164GISEL-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x2c
-; GFX1164GISEL-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1164GISEL-TRUE16-NEXT: s_mov_b64 s[2:3], exec
-; GFX1164GISEL-TRUE16-NEXT: v_mov_b32_e32 v1, 0
-; GFX1164GISEL-TRUE16-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX1164GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX1164GISEL-TRUE16-NEXT: s_and_b32 s2, s2, 1
-; GFX1164GISEL-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164GISEL-TRUE16-NEXT: s_and_b32 s3, s6, 0xffff
-; GFX1164GISEL-TRUE16-NEXT: s_mul_i32 s2, s3, s2
-; GFX1164GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.l, s2
-; GFX1164GISEL-TRUE16-NEXT: global_store_b16 v1, v0, s[0:1]
-; GFX1164GISEL-TRUE16-NEXT: s_endpgm
-;
-; GFX1132GISEL-TRUE16-LABEL: uniform_value_i16:
-; GFX1132GISEL-TRUE16: ; %bb.0: ; %entry
-; GFX1132GISEL-TRUE16-NEXT: s_clause 0x1
-; GFX1132GISEL-TRUE16-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX1132GISEL-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1132GISEL-TRUE16-NEXT: s_mov_b32 s3, exec_lo
-; GFX1132GISEL-TRUE16-NEXT: v_mov_b32_e32 v1, 0
-; GFX1132GISEL-TRUE16-NEXT: s_bcnt1_i32_b32 s3, s3
-; GFX1132GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX1132GISEL-TRUE16-NEXT: s_and_b32 s3, s3, 1
-; GFX1132GISEL-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132GISEL-TRUE16-NEXT: s_and_b32 s2, s2, 0xffff
-; GFX1132GISEL-TRUE16-NEXT: s_mul_i32 s2, s2, s3
-; GFX1132GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.l, s2
-; GFX1132GISEL-TRUE16-NEXT: global_store_b16 v1, v0, s[0:1]
-; GFX1132GISEL-TRUE16-NEXT: s_endpgm
+; GFX1132GISEL-LABEL: uniform_value_i16:
+; GFX1132GISEL: ; %bb.0: ; %entry
+; GFX1132GISEL-NEXT: s_clause 0x1
+; GFX1132GISEL-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GFX1132GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1132GISEL-NEXT: s_mov_b32 s3, exec_lo
+; GFX1132GISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX1132GISEL-NEXT: s_bcnt1_i32_b32 s3, s3
+; GFX1132GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX1132GISEL-NEXT: s_and_b32 s3, s3, 1
+; GFX1132GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1132GISEL-NEXT: s_and_b32 s2, s2, 0xffff
+; GFX1132GISEL-NEXT: s_mul_i32 s2, s2, s3
+; GFX1132GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX1132GISEL-NEXT: global_store_b16 v1, v0, s[0:1]
+; GFX1132GISEL-NEXT: s_endpgm
entry:
%result = call i16 @llvm.amdgcn.wave.reduce.xor.i16(i16 %in, i32 1)
store i16 %result, ptr addrspace(1) %out
@@ -4841,3 +4805,8 @@ entry:
}
attributes #0 = { nounwind }
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; GFX1132GISEL-FAKE16: {{.*}}
+; GFX1132GISEL-TRUE16: {{.*}}
+; GFX1164GISEL-FAKE16: {{.*}}
+; GFX1164GISEL-TRUE16: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rsq.f16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rsq.f16.ll
index e5979d8c99b60..d2199772c1b25 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rsq.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rsq.f16.ll
@@ -159,7 +159,7 @@ define amdgpu_kernel void @rsq_f16(
; GISEL-GFX12-TRUE16-NEXT: v_s_rsq_f16 s2, s2
; GISEL-GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
; GISEL-GFX12-TRUE16-NEXT: s_delay_alu instid0(TRANS32_DEP_1)
-; GISEL-GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, s2
+; GISEL-GFX12-TRUE16-NEXT: v_mov_b32_e32 v0, s2
; GISEL-GFX12-TRUE16-NEXT: s_mov_b32 s2, -1
; GISEL-GFX12-TRUE16-NEXT: buffer_store_b16 v0, off, s[0:3], null
; GISEL-GFX12-TRUE16-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.fptrunc.round.ll b/llvm/test/CodeGen/AMDGPU/llvm.fptrunc.round.ll
index 2ab76c99095f8..c29a124706d13 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.fptrunc.round.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.fptrunc.round.ll
@@ -693,20 +693,35 @@ define amdgpu_gs void @s_fptrunc_round_f32_to_f16_upward_multiple_calls(float in
; GISEL-NEXT: global_store_short v[0:1], v2, off
; GISEL-NEXT: s_endpgm
;
-; GFX12-LABEL: s_fptrunc_round_f32_to_f16_upward_multiple_calls:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 1), 1
-; GFX12-NEXT: s_cvt_f16_f32 s0, s0
-; GFX12-NEXT: s_cvt_f16_f32 s2, s1
-; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 2
-; GFX12-NEXT: s_cvt_f16_f32 s1, s1
-; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 3, 1), 0
-; GFX12-NEXT: s_add_f16 s0, s0, s2
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_3) | instskip(NEXT) | instid1(SALU_CYCLE_3)
-; GFX12-NEXT: s_add_f16 s0, s1, s0
-; GFX12-NEXT: v_mov_b16_e32 v2.l, s0
-; GFX12-NEXT: global_store_b16 v[0:1], v2, off
-; GFX12-NEXT: s_endpgm
+; GFX12-SDAG-LABEL: s_fptrunc_round_f32_to_f16_upward_multiple_calls:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 1), 1
+; GFX12-SDAG-NEXT: s_cvt_f16_f32 s0, s0
+; GFX12-SDAG-NEXT: s_cvt_f16_f32 s2, s1
+; GFX12-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 2
+; GFX12-SDAG-NEXT: s_cvt_f16_f32 s1, s1
+; GFX12-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 3, 1), 0
+; GFX12-SDAG-NEXT: s_add_f16 s0, s0, s2
+; GFX12-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_3) | instskip(NEXT) | instid1(SALU_CYCLE_3)
+; GFX12-SDAG-NEXT: s_add_f16 s0, s1, s0
+; GFX12-SDAG-NEXT: v_mov_b16_e32 v2.l, s0
+; GFX12-SDAG-NEXT: global_store_b16 v[0:1], v2, off
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: s_fptrunc_round_f32_to_f16_upward_multiple_calls:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 1), 1
+; GFX12-GISEL-NEXT: s_cvt_f16_f32 s0, s0
+; GFX12-GISEL-NEXT: s_cvt_f16_f32 s2, s1
+; GFX12-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 2
+; GFX12-GISEL-NEXT: s_cvt_f16_f32 s1, s1
+; GFX12-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 3, 1), 0
+; GFX12-GISEL-NEXT: s_add_f16 s0, s0, s2
+; GFX12-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_3) | instskip(NEXT) | instid1(SALU_CYCLE_3)
+; GFX12-GISEL-NEXT: s_add_f16 s0, s1, s0
+; GFX12-GISEL-NEXT: v_mov_b32_e32 v2, s0
+; GFX12-GISEL-NEXT: global_store_b16 v[0:1], v2, off
+; GFX12-GISEL-NEXT: s_endpgm
%res1 = call half @llvm.fptrunc.round.f16.f32(float %a, metadata !"round.upward")
%res2 = call half @llvm.fptrunc.round.f16.f32(float %b, metadata !"round.upward")
%res3 = call half @llvm.fptrunc.round.f16.f32(float %b, metadata !"round.downward")
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.sqrt.f16.ll b/llvm/test/CodeGen/AMDGPU/llvm.sqrt.f16.ll
index bcaba060b812f..4e82a8f68366d 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.sqrt.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.sqrt.f16.ll
@@ -146,7 +146,7 @@ define amdgpu_kernel void @sqrt_f16(
; GFX12-TRUE16-GISEL-NEXT: v_s_sqrt_f16 s2, s2
; GFX12-TRUE16-GISEL-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX12-TRUE16-GISEL-NEXT: s_delay_alu instid0(TRANS32_DEP_1)
-; GFX12-TRUE16-GISEL-NEXT: v_mov_b16_e32 v0.l, s2
+; GFX12-TRUE16-GISEL-NEXT: v_mov_b32_e32 v0, s2
; GFX12-TRUE16-GISEL-NEXT: s_mov_b32 s2, -1
; GFX12-TRUE16-GISEL-NEXT: buffer_store_b16 v0, off, s[0:3], null
; GFX12-TRUE16-GISEL-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/minimummaximum.ll b/llvm/test/CodeGen/AMDGPU/minimummaximum.ll
index 10f6984276d69..32091c78120a1 100644
--- a/llvm/test/CodeGen/AMDGPU/minimummaximum.ll
+++ b/llvm/test/CodeGen/AMDGPU/minimummaximum.ll
@@ -228,11 +228,10 @@ define amdgpu_ps void @s_test_minmax_f16(half inreg %a, half inreg %b, half inre
;
; GFX1170-GISEL-TRUE16-LABEL: s_test_minmax_f16:
; GFX1170-GISEL-TRUE16: ; %bb.0:
-; GFX1170-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.l, s2
-; GFX1170-GISEL-TRUE16-NEXT: v_mov_b32_e32 v1, 0
+; GFX1170-GISEL-TRUE16-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, 0
; GFX1170-GISEL-TRUE16-NEXT: s_mov_b32 s6, s3
; GFX1170-GISEL-TRUE16-NEXT: s_mov_b32 s7, s4
-; GFX1170-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1170-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1170-GISEL-TRUE16-NEXT: v_maximumminimum_f16 v0.l, s0, s1, v0.l
; GFX1170-GISEL-TRUE16-NEXT: global_store_b16 v1, v0, s[6:7]
; GFX1170-GISEL-TRUE16-NEXT: s_endpgm
@@ -270,29 +269,17 @@ define amdgpu_ps void @s_test_minmax_f16(half inreg %a, half inreg %b, half inre
; GFX12-SDAG-FAKE16-NEXT: global_store_b16 v0, v1, s[4:5]
; GFX12-SDAG-FAKE16-NEXT: s_endpgm
;
-; GFX12-GISEL-TRUE16-LABEL: s_test_minmax_f16:
-; GFX12-GISEL-TRUE16: ; %bb.0:
-; GFX12-GISEL-TRUE16-NEXT: s_maximum_f16 s0, s0, s1
-; GFX12-GISEL-TRUE16-NEXT: v_mov_b32_e32 v1, 0
-; GFX12-GISEL-TRUE16-NEXT: s_mov_b32 s6, s3
-; GFX12-GISEL-TRUE16-NEXT: s_mov_b32 s7, s4
-; GFX12-GISEL-TRUE16-NEXT: s_minimum_f16 s0, s0, s2
-; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
-; GFX12-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
-; GFX12-GISEL-TRUE16-NEXT: global_store_b16 v1, v0, s[6:7]
-; GFX12-GISEL-TRUE16-NEXT: s_endpgm
-;
-; GFX12-GISEL-FAKE16-LABEL: s_test_minmax_f16:
-; GFX12-GISEL-FAKE16: ; %bb.0:
-; GFX12-GISEL-FAKE16-NEXT: s_maximum_f16 s0, s0, s1
-; GFX12-GISEL-FAKE16-NEXT: s_mov_b32 s6, s3
-; GFX12-GISEL-FAKE16-NEXT: s_mov_b32 s7, s4
-; GFX12-GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, 0
-; GFX12-GISEL-FAKE16-NEXT: s_minimum_f16 s0, s0, s2
-; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
-; GFX12-GISEL-FAKE16-NEXT: v_mov_b32_e32 v0, s0
-; GFX12-GISEL-FAKE16-NEXT: global_store_b16 v1, v0, s[6:7]
-; GFX12-GISEL-FAKE16-NEXT: s_endpgm
+; GFX12-GISEL-LABEL: s_test_minmax_f16:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_maximum_f16 s0, s0, s1
+; GFX12-GISEL-NEXT: s_mov_b32 s6, s3
+; GFX12-GISEL-NEXT: s_mov_b32 s7, s4
+; GFX12-GISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX12-GISEL-NEXT: s_minimum_f16 s0, s0, s2
+; GFX12-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
+; GFX12-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-GISEL-NEXT: global_store_b16 v1, v0, s[6:7]
+; GFX12-GISEL-NEXT: s_endpgm
%smax = call half @llvm.maximum.f16(half %a, half %b)
%sminmax = call half @llvm.minimum.f16(half %smax, half %c)
store half %sminmax, ptr addrspace(1) %out
diff --git a/llvm/test/CodeGen/AMDGPU/minmax.ll b/llvm/test/CodeGen/AMDGPU/minmax.ll
index 68e987b6adf72..e24f50f91bf1b 100644
--- a/llvm/test/CodeGen/AMDGPU/minmax.ll
+++ b/llvm/test/CodeGen/AMDGPU/minmax.ll
@@ -1059,8 +1059,7 @@ define amdgpu_ps void @s_test_minmax_f16_ieee_false(half inreg %a, half inreg %b
;
; GISEL-GFX11-TRUE16-LABEL: s_test_minmax_f16_ieee_false:
; GISEL-GFX11-TRUE16: ; %bb.0:
-; GISEL-GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, s2
-; GISEL-GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, 0
+; GISEL-GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, 0
; GISEL-GFX11-TRUE16-NEXT: s_mov_b32 s6, s3
; GISEL-GFX11-TRUE16-NEXT: s_mov_b32 s7, s4
; GISEL-GFX11-TRUE16-NEXT: v_maxmin_f16 v0.l, s0, s1, v0.l
@@ -1095,27 +1094,16 @@ define amdgpu_ps void @s_test_minmax_f16_ieee_false(half inreg %a, half inreg %b
; SDAG-GFX1170-FAKE16-NEXT: global_store_b16 v1, v0, s[4:5]
; SDAG-GFX1170-FAKE16-NEXT: s_endpgm
;
-; GISEL-GFX1170-TRUE16-LABEL: s_test_minmax_f16_ieee_false:
-; GISEL-GFX1170-TRUE16: ; %bb.0:
-; GISEL-GFX1170-TRUE16-NEXT: s_max_f16 s0, s0, s1
-; GISEL-GFX1170-TRUE16-NEXT: v_mov_b32_e32 v1, 0
-; GISEL-GFX1170-TRUE16-NEXT: s_mov_b32 s6, s3
-; GISEL-GFX1170-TRUE16-NEXT: s_mov_b32 s7, s4
-; GISEL-GFX1170-TRUE16-NEXT: s_min_f16 s0, s0, s2
-; GISEL-GFX1170-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
-; GISEL-GFX1170-TRUE16-NEXT: global_store_b16 v1, v0, s[6:7]
-; GISEL-GFX1170-TRUE16-NEXT: s_endpgm
-;
-; GISEL-GFX1170-FAKE16-LABEL: s_test_minmax_f16_ieee_false:
-; GISEL-GFX1170-FAKE16: ; %bb.0:
-; GISEL-GFX1170-FAKE16-NEXT: s_max_f16 s0, s0, s1
-; GISEL-GFX1170-FAKE16-NEXT: s_mov_b32 s6, s3
-; GISEL-GFX1170-FAKE16-NEXT: s_mov_b32 s7, s4
-; GISEL-GFX1170-FAKE16-NEXT: v_mov_b32_e32 v1, 0
-; GISEL-GFX1170-FAKE16-NEXT: s_min_f16 s0, s0, s2
-; GISEL-GFX1170-FAKE16-NEXT: v_mov_b32_e32 v0, s0
-; GISEL-GFX1170-FAKE16-NEXT: global_store_b16 v1, v0, s[6:7]
-; GISEL-GFX1170-FAKE16-NEXT: s_endpgm
+; GISEL-GFX1170-LABEL: s_test_minmax_f16_ieee_false:
+; GISEL-GFX1170: ; %bb.0:
+; GISEL-GFX1170-NEXT: s_max_f16 s0, s0, s1
+; GISEL-GFX1170-NEXT: s_mov_b32 s6, s3
+; GISEL-GFX1170-NEXT: s_mov_b32 s7, s4
+; GISEL-GFX1170-NEXT: v_mov_b32_e32 v1, 0
+; GISEL-GFX1170-NEXT: s_min_f16 s0, s0, s2
+; GISEL-GFX1170-NEXT: v_mov_b32_e32 v0, s0
+; GISEL-GFX1170-NEXT: global_store_b16 v1, v0, s[6:7]
+; GISEL-GFX1170-NEXT: s_endpgm
;
; SDAG-GFX12-TRUE16-LABEL: s_test_minmax_f16_ieee_false:
; SDAG-GFX12-TRUE16: ; %bb.0:
@@ -1136,27 +1124,16 @@ define amdgpu_ps void @s_test_minmax_f16_ieee_false(half inreg %a, half inreg %b
; SDAG-GFX12-FAKE16-NEXT: global_store_b16 v1, v0, s[4:5]
; SDAG-GFX12-FAKE16-NEXT: s_endpgm
;
-; GISEL-GFX12-TRUE16-LABEL: s_test_minmax_f16_ieee_false:
-; GISEL-GFX12-TRUE16: ; %bb.0:
-; GISEL-GFX12-TRUE16-NEXT: s_max_num_f16 s0, s0, s1
-; GISEL-GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, 0
-; GISEL-GFX12-TRUE16-NEXT: s_mov_b32 s6, s3
-; GISEL-GFX12-TRUE16-NEXT: s_mov_b32 s7, s4
-; GISEL-GFX12-TRUE16-NEXT: s_min_num_f16 s0, s0, s2
-; GISEL-GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
-; GISEL-GFX12-TRUE16-NEXT: global_store_b16 v1, v0, s[6:7]
-; GISEL-GFX12-TRUE16-NEXT: s_endpgm
-;
-; GISEL-GFX12-FAKE16-LABEL: s_test_minmax_f16_ieee_false:
-; GISEL-GFX12-FAKE16: ; %bb.0:
-; GISEL-GFX12-FAKE16-NEXT: s_max_num_f16 s0, s0, s1
-; GISEL-GFX12-FAKE16-NEXT: s_mov_b32 s6, s3
-; GISEL-GFX12-FAKE16-NEXT: s_mov_b32 s7, s4
-; GISEL-GFX12-FAKE16-NEXT: v_mov_b32_e32 v1, 0
-; GISEL-GFX12-FAKE16-NEXT: s_min_num_f16 s0, s0, s2
-; GISEL-GFX12-FAKE16-NEXT: v_mov_b32_e32 v0, s0
-; GISEL-GFX12-FAKE16-NEXT: global_store_b16 v1, v0, s[6:7]
-; GISEL-GFX12-FAKE16-NEXT: s_endpgm
+; GISEL-GFX12-LABEL: s_test_minmax_f16_ieee_false:
+; GISEL-GFX12: ; %bb.0:
+; GISEL-GFX12-NEXT: s_max_num_f16 s0, s0, s1
+; GISEL-GFX12-NEXT: s_mov_b32 s6, s3
+; GISEL-GFX12-NEXT: s_mov_b32 s7, s4
+; GISEL-GFX12-NEXT: v_mov_b32_e32 v1, 0
+; GISEL-GFX12-NEXT: s_min_num_f16 s0, s0, s2
+; GISEL-GFX12-NEXT: v_mov_b32_e32 v0, s0
+; GISEL-GFX12-NEXT: global_store_b16 v1, v0, s[6:7]
+; GISEL-GFX12-NEXT: s_endpgm
;
; SDAG-GFX1250-TRUE16-LABEL: s_test_minmax_f16_ieee_false:
; SDAG-GFX1250-TRUE16: ; %bb.0:
@@ -1183,33 +1160,19 @@ define amdgpu_ps void @s_test_minmax_f16_ieee_false(half inreg %a, half inreg %b
; SDAG-GFX1250-FAKE16-NEXT: global_store_b16 v1, v0, s[4:5]
; SDAG-GFX1250-FAKE16-NEXT: s_endpgm
;
-; GISEL-GFX1250-TRUE16-LABEL: s_test_minmax_f16_ieee_false:
-; GISEL-GFX1250-TRUE16: ; %bb.0:
-; GISEL-GFX1250-TRUE16-NEXT: global_prefetch_b8 v0, null scope:SCOPE_SE
-; GISEL-GFX1250-TRUE16-NEXT: v_nop
-; GISEL-GFX1250-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GISEL-GFX1250-TRUE16-NEXT: s_max_num_f16 s0, s0, s1
-; GISEL-GFX1250-TRUE16-NEXT: v_mov_b32_e32 v1, 0
-; GISEL-GFX1250-TRUE16-NEXT: s_mov_b32 s6, s3
-; GISEL-GFX1250-TRUE16-NEXT: s_mov_b32 s7, s4
-; GISEL-GFX1250-TRUE16-NEXT: s_min_num_f16 s0, s0, s2
-; GISEL-GFX1250-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
-; GISEL-GFX1250-TRUE16-NEXT: global_store_b16 v1, v0, s[6:7]
-; GISEL-GFX1250-TRUE16-NEXT: s_endpgm
-;
-; GISEL-GFX1250-FAKE16-LABEL: s_test_minmax_f16_ieee_false:
-; GISEL-GFX1250-FAKE16: ; %bb.0:
-; GISEL-GFX1250-FAKE16-NEXT: global_prefetch_b8 v0, null scope:SCOPE_SE
-; GISEL-GFX1250-FAKE16-NEXT: v_nop
-; GISEL-GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GISEL-GFX1250-FAKE16-NEXT: s_max_num_f16 s0, s0, s1
-; GISEL-GFX1250-FAKE16-NEXT: s_mov_b32 s6, s3
-; GISEL-GFX1250-FAKE16-NEXT: s_mov_b32 s7, s4
-; GISEL-GFX1250-FAKE16-NEXT: v_mov_b32_e32 v1, 0
-; GISEL-GFX1250-FAKE16-NEXT: s_min_num_f16 s0, s0, s2
-; GISEL-GFX1250-FAKE16-NEXT: v_mov_b32_e32 v0, s0
-; GISEL-GFX1250-FAKE16-NEXT: global_store_b16 v1, v0, s[6:7]
-; GISEL-GFX1250-FAKE16-NEXT: s_endpgm
+; GISEL-GFX1250-LABEL: s_test_minmax_f16_ieee_false:
+; GISEL-GFX1250: ; %bb.0:
+; GISEL-GFX1250-NEXT: global_prefetch_b8 v0, null scope:SCOPE_SE
+; GISEL-GFX1250-NEXT: v_nop
+; GISEL-GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GISEL-GFX1250-NEXT: s_max_num_f16 s0, s0, s1
+; GISEL-GFX1250-NEXT: s_mov_b32 s6, s3
+; GISEL-GFX1250-NEXT: s_mov_b32 s7, s4
+; GISEL-GFX1250-NEXT: v_mov_b32_e32 v1, 0
+; GISEL-GFX1250-NEXT: s_min_num_f16 s0, s0, s2
+; GISEL-GFX1250-NEXT: v_mov_b32_e32 v0, s0
+; GISEL-GFX1250-NEXT: global_store_b16 v1, v0, s[6:7]
+; GISEL-GFX1250-NEXT: s_endpgm
%smax = call half @llvm.maxnum.f16(half %a, half %b)
%sminmax = call half @llvm.minnum.f16(half %smax, half %c)
store half %sminmax, ptr addrspace(1) %out
More information about the llvm-commits
mailing list