[llvm] [ExpandReductions] Add support for non-pow2 vector counts (PR #208738)
Prajwal KP via llvm-commits
llvm-commits at lists.llvm.org
Fri Jul 10 09:05:10 PDT 2026
https://github.com/Prajwal-kp-18 updated https://github.com/llvm/llvm-project/pull/208738
>From 8713dc2d72e1edbfccbabd54235b82488fea5a5a Mon Sep 17 00:00:00 2001
From: Prajwal <prajwal.kp.1817 at gmail.com>
Date: Fri, 10 Jul 2026 19:52:57 +0530
Subject: [PATCH] [CodeGen] Expand non-pow2 vector reductions by scalarizing
---
llvm/lib/CodeGen/ExpandReductions.cpp | 46 ++++--
llvm/test/CodeGen/AMDGPU/vector-reduce-add.ll | 128 ++++++---------
llvm/test/CodeGen/AMDGPU/vector-reduce-and.ll | 96 ++++--------
llvm/test/CodeGen/AMDGPU/vector-reduce-mul.ll | 122 +++++++--------
llvm/test/CodeGen/AMDGPU/vector-reduce-or.ll | 63 +++-----
.../test/CodeGen/AMDGPU/vector-reduce-smax.ll | 146 ++++++------------
.../test/CodeGen/AMDGPU/vector-reduce-smin.ll | 146 ++++++------------
.../test/CodeGen/AMDGPU/vector-reduce-umax.ll | 102 +++++-------
.../test/CodeGen/AMDGPU/vector-reduce-umin.ll | 104 +++++--------
llvm/test/CodeGen/AMDGPU/vector-reduce-xor.ll | 80 ++++------
.../Generic/expand-experimental-reductions.ll | 84 +++++++++-
.../WebAssembly/simd-vecreduce-bool.ll | 4 +-
.../CodeGen/X86/vector-reduce-fmax-nnan.ll | 24 +--
llvm/test/CodeGen/X86/vector-reduce-or-cmp.ll | 127 ++++++++++-----
14 files changed, 558 insertions(+), 714 deletions(-)
diff --git a/llvm/lib/CodeGen/ExpandReductions.cpp b/llvm/lib/CodeGen/ExpandReductions.cpp
index 30c1a8f9fa0e5..ca0862cb97e0a 100644
--- a/llvm/lib/CodeGen/ExpandReductions.cpp
+++ b/llvm/lib/CodeGen/ExpandReductions.cpp
@@ -28,6 +28,21 @@ using namespace llvm;
namespace {
+Value *expandScalarizedReduction(IRBuilderBase &Builder, Value *Vec,
+ unsigned RdxOpcode, RecurKind RK) {
+ auto *VecTy = cast<FixedVectorType>(Vec->getType());
+ Value *Res = Builder.CreateExtractElement(Vec, uint64_t(0));
+ for (unsigned I = 1, E = VecTy->getNumElements(); I != E; ++I) {
+ Value *Ext = Builder.CreateExtractElement(Vec, I);
+ if (RdxOpcode == Instruction::ICmp || RdxOpcode == Instruction::FCmp)
+ Res = createMinMaxOp(Builder, RK, Res, Ext);
+ else
+ Res = Builder.CreateBinOp((Instruction::BinaryOps)RdxOpcode, Res, Ext,
+ "bin.rdx");
+ }
+ return Res;
+}
+
bool expandReductions(Function &F, const TargetTransformInfo *TTI,
DominatorTree *DT, LoopInfo *LI) {
bool Changed = false;
@@ -81,12 +96,11 @@ bool expandReductions(Function &F, const TargetTransformInfo *TTI,
Rdx = expandReductionViaLoop(Builder, Vec, RdxOpcode, Acc, DT, LI);
break;
}
- if (!FMF.allowReassoc())
+ if (!FMF.allowReassoc() ||
+ !isPowerOf2_32(
+ cast<FixedVectorType>(Vec->getType())->getNumElements()))
Rdx = getOrderedReduction(Builder, Acc, Vec, RdxOpcode, RK);
else {
- if (!isPowerOf2_32(
- cast<FixedVectorType>(Vec->getType())->getNumElements()))
- continue;
Rdx = getShuffleReduction(Builder, Vec, RdxOpcode, RS, RK);
Rdx = Builder.CreateBinOp((Instruction::BinaryOps)RdxOpcode, Acc, Rdx,
"bin.rdx");
@@ -105,10 +119,9 @@ bool expandReductions(Function &F, const TargetTransformInfo *TTI,
Value *Vec = II->getArgOperand(0);
auto *FTy = cast<FixedVectorType>(Vec->getType());
unsigned NumElts = FTy->getNumElements();
- if (!isPowerOf2_32(NumElts))
- continue;
- if (FTy->getElementType() == Builder.getInt1Ty()) {
+ if (FTy->getElementType() == Builder.getInt1Ty() &&
+ isPowerOf2_32(NumElts)) {
Rdx = Builder.CreateBitCast(Vec, Builder.getIntNTy(NumElts));
if (ID == Intrinsic::vector_reduce_and) {
Rdx = Builder.CreateICmpEQ(
@@ -120,6 +133,10 @@ bool expandReductions(Function &F, const TargetTransformInfo *TTI,
break;
}
unsigned RdxOpcode = getArithmeticReductionInstruction(ID);
+ if (!isPowerOf2_32(NumElts)) {
+ Rdx = expandScalarizedReduction(Builder, Vec, RdxOpcode, RK);
+ break;
+ }
Rdx = getShuffleReduction(Builder, Vec, RdxOpcode, RS, RK);
break;
}
@@ -139,8 +156,10 @@ bool expandReductions(Function &F, const TargetTransformInfo *TTI,
break;
}
if (!isPowerOf2_32(
- cast<FixedVectorType>(Vec->getType())->getNumElements()))
- continue;
+ cast<FixedVectorType>(Vec->getType())->getNumElements())) {
+ Rdx = expandScalarizedReduction(Builder, Vec, RdxOpcode, RK);
+ break;
+ }
Rdx = getShuffleReduction(Builder, Vec, RdxOpcode, RS, RK);
break;
}
@@ -149,11 +168,14 @@ bool expandReductions(Function &F, const TargetTransformInfo *TTI,
// We require "nnan" to use a shuffle reduction; "nsz" is implied by the
// semantics of the reduction.
Value *Vec = II->getArgOperand(0);
- if (!isPowerOf2_32(
- cast<FixedVectorType>(Vec->getType())->getNumElements()) ||
- !FMF.noNaNs())
+ if (!FMF.noNaNs())
continue;
unsigned RdxOpcode = getArithmeticReductionInstruction(ID);
+ if (!isPowerOf2_32(
+ cast<FixedVectorType>(Vec->getType())->getNumElements())) {
+ Rdx = expandScalarizedReduction(Builder, Vec, RdxOpcode, RK);
+ break;
+ }
Rdx = getShuffleReduction(Builder, Vec, RdxOpcode, RS, RK);
break;
}
diff --git a/llvm/test/CodeGen/AMDGPU/vector-reduce-add.ll b/llvm/test/CodeGen/AMDGPU/vector-reduce-add.ll
index 52b3398c5fa81..0d20d88cdc610 100644
--- a/llvm/test/CodeGen/AMDGPU/vector-reduce-add.ll
+++ b/llvm/test/CodeGen/AMDGPU/vector-reduce-add.ll
@@ -110,76 +110,48 @@ entry:
}
define i8 @test_vector_reduce_add_v3i8(<3 x i8> %v) {
-; GFX7-SDAG-LABEL: test_vector_reduce_add_v3i8:
-; GFX7-SDAG: ; %bb.0: ; %entry
-; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-SDAG-NEXT: v_add_i32_e32 v0, vcc, v0, v2
-; GFX7-SDAG-NEXT: v_add_i32_e32 v0, vcc, v0, v1
-; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX7-GISEL-LABEL: test_vector_reduce_add_v3i8:
-; GFX7-GISEL: ; %bb.0: ; %entry
-; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v1
-; GFX7-GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v2
-; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX8-SDAG-LABEL: test_vector_reduce_add_v3i8:
-; GFX8-SDAG: ; %bb.0: ; %entry
-; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-SDAG-NEXT: v_add_u16_e32 v0, v0, v2
-; GFX8-SDAG-NEXT: v_add_u16_e32 v0, v0, v1
-; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX8-GISEL-LABEL: test_vector_reduce_add_v3i8:
-; GFX8-GISEL: ; %bb.0: ; %entry
-; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-GISEL-NEXT: v_add_u16_e32 v0, v0, v1
-; GFX8-GISEL-NEXT: v_add_u16_e32 v0, v0, v2
-; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX9-SDAG-LABEL: test_vector_reduce_add_v3i8:
-; GFX9-SDAG: ; %bb.0: ; %entry
-; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-SDAG-NEXT: v_add_u16_e32 v0, v0, v2
-; GFX9-SDAG-NEXT: v_add_u16_e32 v0, v0, v1
-; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
+; GFX7-LABEL: test_vector_reduce_add_v3i8:
+; GFX7: ; %bb.0: ; %entry
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_add_i32_e32 v0, vcc, v0, v1
+; GFX7-NEXT: v_add_i32_e32 v0, vcc, v0, v2
+; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9-GISEL-LABEL: test_vector_reduce_add_v3i8:
-; GFX9-GISEL: ; %bb.0: ; %entry
-; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-GISEL-NEXT: v_add_u16_e32 v0, v0, v1
-; GFX9-GISEL-NEXT: v_add_u16_e32 v0, v0, v2
-; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+; GFX8-LABEL: test_vector_reduce_add_v3i8:
+; GFX8: ; %bb.0: ; %entry
+; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT: v_add_u16_e32 v0, v0, v1
+; GFX8-NEXT: v_add_u16_e32 v0, v0, v2
+; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-SDAG-LABEL: test_vector_reduce_add_v3i8:
-; GFX10-SDAG: ; %bb.0: ; %entry
-; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_add_nc_u16 v0, v0, v2
-; GFX10-SDAG-NEXT: v_add_nc_u16 v0, v0, v1
-; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
+; GFX9-LABEL: test_vector_reduce_add_v3i8:
+; GFX9: ; %bb.0: ; %entry
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_add_u16_e32 v0, v0, v1
+; GFX9-NEXT: v_add_u16_e32 v0, v0, v2
+; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-GISEL-LABEL: test_vector_reduce_add_v3i8:
-; GFX10-GISEL: ; %bb.0: ; %entry
-; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_add_nc_u16 v0, v0, v1
-; GFX10-GISEL-NEXT: v_add_nc_u16 v0, v0, v2
-; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
+; GFX10-LABEL: test_vector_reduce_add_v3i8:
+; GFX10: ; %bb.0: ; %entry
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_add_nc_u16 v0, v0, v1
+; GFX10-NEXT: v_add_nc_u16 v0, v0, v2
+; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_add_v3i8:
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_add_nc_u16 v0.l, v0.l, v2.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_add_nc_u16 v0.l, v0.l, v1.l
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_add_nc_u16 v0.l, v0.l, v2.l
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-FAKE16-LABEL: test_vector_reduce_add_v3i8:
; GFX11-SDAG-FAKE16: ; %bb.0: ; %entry
; GFX11-SDAG-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-FAKE16-NEXT: v_add_nc_u16 v0, v0, v2
-; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-SDAG-FAKE16-NEXT: v_add_nc_u16 v0, v0, v1
+; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-FAKE16-NEXT: v_add_nc_u16 v0, v0, v2
; GFX11-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-GISEL-TRUE16-LABEL: test_vector_reduce_add_v3i8:
@@ -205,9 +177,9 @@ define i8 @test_vector_reduce_add_v3i8(<3 x i8> %v) {
; GFX12-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: v_add_nc_u16 v0.l, v0.l, v2.l
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-SDAG-TRUE16-NEXT: v_add_nc_u16 v0.l, v0.l, v1.l
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-TRUE16-NEXT: v_add_nc_u16 v0.l, v0.l, v2.l
; GFX12-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-SDAG-FAKE16-LABEL: test_vector_reduce_add_v3i8:
@@ -217,9 +189,9 @@ define i8 @test_vector_reduce_add_v3i8(<3 x i8> %v) {
; GFX12-SDAG-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-SDAG-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-FAKE16-NEXT: v_add_nc_u16 v0, v0, v2
-; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-SDAG-FAKE16-NEXT: v_add_nc_u16 v0, v0, v1
+; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-FAKE16-NEXT: v_add_nc_u16 v0, v0, v2
; GFX12-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GISEL-TRUE16-LABEL: test_vector_reduce_add_v3i8:
@@ -1277,10 +1249,9 @@ define i16 @test_vector_reduce_add_v3i16(<3 x i16> %v) {
; GFX7-SDAG-LABEL: test_vector_reduce_add_v3i16:
; GFX7-SDAG: ; %bb.0: ; %entry
; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-SDAG-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX7-SDAG-NEXT: v_lshrrev_b32_e32 v2, 16, v0
-; GFX7-SDAG-NEXT: v_add_i32_e32 v0, vcc, v0, v1
+; GFX7-SDAG-NEXT: v_alignbit_b32 v2, v1, v0, 16
; GFX7-SDAG-NEXT: v_add_i32_e32 v0, vcc, v0, v2
+; GFX7-SDAG-NEXT: v_add_i32_e32 v0, vcc, v0, v1
; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX7-GISEL-LABEL: test_vector_reduce_add_v3i16:
@@ -1301,34 +1272,33 @@ define i16 @test_vector_reduce_add_v3i16(<3 x i16> %v) {
; GFX9-LABEL: test_vector_reduce_add_v3i16:
; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_pk_add_u16 v1, v0, v1
-; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: v_add_u16_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_add_u16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_add_u16_e32 v0, v0, v1
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: test_vector_reduce_add_v3i16:
; GFX10: ; %bb.0: ; %entry
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_lshrrev_b32_e32 v2, 16, v0
-; GFX10-NEXT: v_pk_add_u16 v0, v0, v1
; GFX10-NEXT: v_add_nc_u16 v0, v0, v2
+; GFX10-NEXT: v_add_nc_u16 v0, v0, v1
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_add_v3i16:
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_pk_add_u16 v1, v0, v1
+; GFX11-SDAG-TRUE16-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_add_nc_u16 v0.l, v1.l, v0.h
+; GFX11-SDAG-TRUE16-NEXT: v_add_nc_u16 v0.l, v0.l, v1.l
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-FAKE16-LABEL: test_vector_reduce_add_v3i16:
; GFX11-SDAG-FAKE16: ; %bb.0: ; %entry
; GFX11-SDAG-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
-; GFX11-SDAG-FAKE16-NEXT: v_pk_add_u16 v0, v0, v1
-; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-FAKE16-NEXT: v_add_nc_u16 v0, v0, v2
+; GFX11-SDAG-FAKE16-NEXT: v_add_nc_u16 v0, v0, v1
; GFX11-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-GISEL-TRUE16-LABEL: test_vector_reduce_add_v3i16:
@@ -1343,9 +1313,9 @@ define i16 @test_vector_reduce_add_v3i16(<3 x i16> %v) {
; GFX11-GISEL-FAKE16: ; %bb.0: ; %entry
; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
-; GFX11-GISEL-FAKE16-NEXT: v_pk_add_u16 v0, v0, v1
-; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-GISEL-FAKE16-NEXT: v_add_nc_u16 v0, v0, v2
+; GFX11-GISEL-FAKE16-NEXT: v_add_nc_u16 v0, v0, v1
; GFX11-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-SDAG-TRUE16-LABEL: test_vector_reduce_add_v3i16:
@@ -1355,9 +1325,9 @@ define i16 @test_vector_reduce_add_v3i16(<3 x i16> %v) {
; GFX12-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: v_pk_add_u16 v1, v0, v1
+; GFX12-SDAG-TRUE16-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_add_nc_u16 v0.l, v1.l, v0.h
+; GFX12-SDAG-TRUE16-NEXT: v_add_nc_u16 v0.l, v0.l, v1.l
; GFX12-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-SDAG-FAKE16-LABEL: test_vector_reduce_add_v3i16:
@@ -1368,9 +1338,9 @@ define i16 @test_vector_reduce_add_v3i16(<3 x i16> %v) {
; GFX12-SDAG-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX12-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
-; GFX12-SDAG-FAKE16-NEXT: v_pk_add_u16 v0, v0, v1
-; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-SDAG-FAKE16-NEXT: v_add_nc_u16 v0, v0, v2
+; GFX12-SDAG-FAKE16-NEXT: v_add_nc_u16 v0, v0, v1
; GFX12-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GISEL-TRUE16-LABEL: test_vector_reduce_add_v3i16:
@@ -1393,9 +1363,9 @@ define i16 @test_vector_reduce_add_v3i16(<3 x i16> %v) {
; GFX12-GISEL-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX12-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
-; GFX12-GISEL-FAKE16-NEXT: v_pk_add_u16 v0, v0, v1
-; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-GISEL-FAKE16-NEXT: v_add_nc_u16 v0, v0, v2
+; GFX12-GISEL-FAKE16-NEXT: v_add_nc_u16 v0, v0, v1
; GFX12-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
entry:
%res = call i16 @llvm.vector.reduce.add.v3i16(<3 x i16> %v)
diff --git a/llvm/test/CodeGen/AMDGPU/vector-reduce-and.ll b/llvm/test/CodeGen/AMDGPU/vector-reduce-and.ll
index a4e6f1e8b6a2c..012dc2a803f2a 100644
--- a/llvm/test/CodeGen/AMDGPU/vector-reduce-and.ll
+++ b/llvm/test/CodeGen/AMDGPU/vector-reduce-and.ll
@@ -161,76 +161,48 @@ entry:
}
define i8 @test_vector_reduce_and_v3i8(<3 x i8> %v) {
-; GFX7-SDAG-LABEL: test_vector_reduce_and_v3i8:
-; GFX7-SDAG: ; %bb.0: ; %entry
-; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-SDAG-NEXT: v_and_b32_e32 v0, v0, v2
-; GFX7-SDAG-NEXT: v_and_b32_e32 v0, v0, v1
-; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX7-GISEL-LABEL: test_vector_reduce_and_v3i8:
-; GFX7-GISEL: ; %bb.0: ; %entry
-; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-GISEL-NEXT: v_and_b32_e32 v0, v0, v1
-; GFX7-GISEL-NEXT: v_and_b32_e32 v0, v0, v2
-; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX8-SDAG-LABEL: test_vector_reduce_and_v3i8:
-; GFX8-SDAG: ; %bb.0: ; %entry
-; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-SDAG-NEXT: v_and_b32_e32 v0, v0, v2
-; GFX8-SDAG-NEXT: v_and_b32_e32 v0, v0, v1
-; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX8-GISEL-LABEL: test_vector_reduce_and_v3i8:
-; GFX8-GISEL: ; %bb.0: ; %entry
-; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-GISEL-NEXT: v_and_b32_e32 v0, v0, v1
-; GFX8-GISEL-NEXT: v_and_b32_e32 v0, v0, v2
-; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX9-SDAG-LABEL: test_vector_reduce_and_v3i8:
-; GFX9-SDAG: ; %bb.0: ; %entry
-; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-SDAG-NEXT: v_and_b32_e32 v0, v0, v2
-; GFX9-SDAG-NEXT: v_and_b32_e32 v0, v0, v1
-; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
+; GFX7-LABEL: test_vector_reduce_and_v3i8:
+; GFX7: ; %bb.0: ; %entry
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_and_b32_e32 v0, v0, v1
+; GFX7-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9-GISEL-LABEL: test_vector_reduce_and_v3i8:
-; GFX9-GISEL: ; %bb.0: ; %entry
-; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-GISEL-NEXT: v_and_b32_e32 v0, v0, v1
-; GFX9-GISEL-NEXT: v_and_b32_e32 v0, v0, v2
-; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+; GFX8-LABEL: test_vector_reduce_and_v3i8:
+; GFX8: ; %bb.0: ; %entry
+; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT: v_and_b32_e32 v0, v0, v1
+; GFX8-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-SDAG-LABEL: test_vector_reduce_and_v3i8:
-; GFX10-SDAG: ; %bb.0: ; %entry
-; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_and_b32_e32 v0, v0, v2
-; GFX10-SDAG-NEXT: v_and_b32_e32 v0, v0, v1
-; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
+; GFX9-LABEL: test_vector_reduce_and_v3i8:
+; GFX9: ; %bb.0: ; %entry
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_and_b32_e32 v0, v0, v1
+; GFX9-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-GISEL-LABEL: test_vector_reduce_and_v3i8:
-; GFX10-GISEL: ; %bb.0: ; %entry
-; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_and_b32_e32 v0, v0, v1
-; GFX10-GISEL-NEXT: v_and_b32_e32 v0, v0, v2
-; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
+; GFX10-LABEL: test_vector_reduce_and_v3i8:
+; GFX10: ; %bb.0: ; %entry
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_and_b32_e32 v0, v0, v1
+; GFX10-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_and_v3i8:
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v0.l, v0.l, v2.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v0.l, v0.l, v1.l
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v0.l, v0.l, v2.l
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-FAKE16-LABEL: test_vector_reduce_and_v3i8:
; GFX11-SDAG-FAKE16: ; %bb.0: ; %entry
; GFX11-SDAG-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, v0, v2
-; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, v0, v1
+; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, v0, v2
; GFX11-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-GISEL-LABEL: test_vector_reduce_and_v3i8:
@@ -248,9 +220,9 @@ define i8 @test_vector_reduce_and_v3i8(<3 x i8> %v) {
; GFX12-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v0.l, v0.l, v2.l
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v0.l, v0.l, v1.l
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v0.l, v0.l, v2.l
; GFX12-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-SDAG-FAKE16-LABEL: test_vector_reduce_and_v3i8:
@@ -260,9 +232,9 @@ define i8 @test_vector_reduce_and_v3i8(<3 x i8> %v) {
; GFX12-SDAG-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-SDAG-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, v0, v2
-; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, v0, v1
+; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, v0, v2
; GFX12-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GISEL-LABEL: test_vector_reduce_and_v3i8:
@@ -1068,9 +1040,9 @@ define i16 @test_vector_reduce_and_v3i16(<3 x i16> %v) {
; GFX7-SDAG-LABEL: test_vector_reduce_and_v3i16:
; GFX7-SDAG: ; %bb.0: ; %entry
; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-SDAG-NEXT: v_lshrrev_b32_e32 v2, 16, v0
-; GFX7-SDAG-NEXT: v_and_b32_e32 v0, v0, v1
+; GFX7-SDAG-NEXT: v_alignbit_b32 v2, v1, v0, 16
; GFX7-SDAG-NEXT: v_and_b32_e32 v0, v0, v2
+; GFX7-SDAG-NEXT: v_and_b32_e32 v0, v0, v1
; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX7-GISEL-LABEL: test_vector_reduce_and_v3i16:
diff --git a/llvm/test/CodeGen/AMDGPU/vector-reduce-mul.ll b/llvm/test/CodeGen/AMDGPU/vector-reduce-mul.ll
index a7a10aacf8bed..712a7688b139b 100644
--- a/llvm/test/CodeGen/AMDGPU/vector-reduce-mul.ll
+++ b/llvm/test/CodeGen/AMDGPU/vector-reduce-mul.ll
@@ -160,8 +160,12 @@ define i8 @test_vector_reduce_mul_v3i8(<3 x i8> %v) {
; GFX7-SDAG-LABEL: test_vector_reduce_mul_v3i8:
; GFX7-SDAG: ; %bb.0: ; %entry
; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-SDAG-NEXT: v_mul_lo_u32 v0, v0, v2
-; GFX7-SDAG-NEXT: v_mul_lo_u32 v0, v1, v0
+; GFX7-SDAG-NEXT: v_and_b32_e32 v1, 0xff, v1
+; GFX7-SDAG-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX7-SDAG-NEXT: v_mul_u32_u24_e32 v0, v0, v1
+; GFX7-SDAG-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX7-SDAG-NEXT: v_and_b32_e32 v1, 0xff, v2
+; GFX7-SDAG-NEXT: v_mul_u32_u24_e32 v0, v0, v1
; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX7-GISEL-LABEL: test_vector_reduce_mul_v3i8:
@@ -171,62 +175,41 @@ define i8 @test_vector_reduce_mul_v3i8(<3 x i8> %v) {
; GFX7-GISEL-NEXT: v_mul_lo_u32 v0, v0, v2
; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-SDAG-LABEL: test_vector_reduce_mul_v3i8:
-; GFX8-SDAG: ; %bb.0: ; %entry
-; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-SDAG-NEXT: v_mul_lo_u16_e32 v0, v0, v2
-; GFX8-SDAG-NEXT: v_mul_lo_u16_e32 v0, v1, v0
-; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX8-GISEL-LABEL: test_vector_reduce_mul_v3i8:
-; GFX8-GISEL: ; %bb.0: ; %entry
-; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v0, v0, v1
-; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v0, v0, v2
-; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX9-SDAG-LABEL: test_vector_reduce_mul_v3i8:
-; GFX9-SDAG: ; %bb.0: ; %entry
-; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-SDAG-NEXT: v_mul_lo_u16_e32 v0, v0, v2
-; GFX9-SDAG-NEXT: v_mul_lo_u16_e32 v0, v1, v0
-; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX9-GISEL-LABEL: test_vector_reduce_mul_v3i8:
-; GFX9-GISEL: ; %bb.0: ; %entry
-; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-GISEL-NEXT: v_mul_lo_u16_e32 v0, v0, v1
-; GFX9-GISEL-NEXT: v_mul_lo_u16_e32 v0, v0, v2
-; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+; GFX8-LABEL: test_vector_reduce_mul_v3i8:
+; GFX8: ; %bb.0: ; %entry
+; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT: v_mul_lo_u16_e32 v0, v0, v1
+; GFX8-NEXT: v_mul_lo_u16_e32 v0, v0, v2
+; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-SDAG-LABEL: test_vector_reduce_mul_v3i8:
-; GFX10-SDAG: ; %bb.0: ; %entry
-; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_mul_lo_u16 v0, v0, v2
-; GFX10-SDAG-NEXT: v_mul_lo_u16 v0, v1, v0
-; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
+; GFX9-LABEL: test_vector_reduce_mul_v3i8:
+; GFX9: ; %bb.0: ; %entry
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_mul_lo_u16_e32 v0, v0, v1
+; GFX9-NEXT: v_mul_lo_u16_e32 v0, v0, v2
+; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-GISEL-LABEL: test_vector_reduce_mul_v3i8:
-; GFX10-GISEL: ; %bb.0: ; %entry
-; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_mul_lo_u16 v0, v0, v1
-; GFX10-GISEL-NEXT: v_mul_lo_u16 v0, v0, v2
-; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
+; GFX10-LABEL: test_vector_reduce_mul_v3i8:
+; GFX10: ; %bb.0: ; %entry
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_mul_lo_u16 v0, v0, v1
+; GFX10-NEXT: v_mul_lo_u16 v0, v0, v2
+; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_mul_v3i8:
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v2.l
+; GFX11-SDAG-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v1.l
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_mul_lo_u16 v0.l, v1.l, v0.l
+; GFX11-SDAG-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v2.l
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-FAKE16-LABEL: test_vector_reduce_mul_v3i8:
; GFX11-SDAG-FAKE16: ; %bb.0: ; %entry
; GFX11-SDAG-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-FAKE16-NEXT: v_mul_lo_u16 v0, v0, v2
+; GFX11-SDAG-FAKE16-NEXT: v_mul_lo_u16 v0, v0, v1
; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-SDAG-FAKE16-NEXT: v_mul_lo_u16 v0, v1, v0
+; GFX11-SDAG-FAKE16-NEXT: v_mul_lo_u16 v0, v0, v2
; GFX11-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-GISEL-LABEL: test_vector_reduce_mul_v3i8:
@@ -244,9 +227,9 @@ define i8 @test_vector_reduce_mul_v3i8(<3 x i8> %v) {
; GFX12-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v2.l
+; GFX12-SDAG-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v1.l
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_mul_lo_u16 v0.l, v1.l, v0.l
+; GFX12-SDAG-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v2.l
; GFX12-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-SDAG-FAKE16-LABEL: test_vector_reduce_mul_v3i8:
@@ -256,9 +239,9 @@ define i8 @test_vector_reduce_mul_v3i8(<3 x i8> %v) {
; GFX12-SDAG-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-SDAG-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-FAKE16-NEXT: v_mul_lo_u16 v0, v0, v2
+; GFX12-SDAG-FAKE16-NEXT: v_mul_lo_u16 v0, v0, v1
; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-SDAG-FAKE16-NEXT: v_mul_lo_u16 v0, v1, v0
+; GFX12-SDAG-FAKE16-NEXT: v_mul_lo_u16 v0, v0, v2
; GFX12-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GISEL-LABEL: test_vector_reduce_mul_v3i8:
@@ -1065,12 +1048,12 @@ define i16 @test_vector_reduce_mul_v3i16(<3 x i16> %v) {
; GFX7-SDAG-LABEL: test_vector_reduce_mul_v3i16:
; GFX7-SDAG: ; %bb.0: ; %entry
; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-SDAG-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX7-SDAG-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7-SDAG-NEXT: v_mul_u32_u24_e32 v0, v0, v2
+; GFX7-SDAG-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX7-SDAG-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX7-SDAG-NEXT: v_or_b32_e32 v1, 0x10000, v1
-; GFX7-SDAG-NEXT: v_mul_lo_u32 v1, v0, v1
-; GFX7-SDAG-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX7-SDAG-NEXT: v_mul_u32_u24_e32 v0, 1, v0
-; GFX7-SDAG-NEXT: v_mul_lo_u32 v0, v1, v0
+; GFX7-SDAG-NEXT: v_mul_u32_u24_e32 v0, v0, v1
; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX7-GISEL-LABEL: test_vector_reduce_mul_v3i16:
@@ -1091,43 +1074,42 @@ define i16 @test_vector_reduce_mul_v3i16(<3 x i16> %v) {
; GFX9-LABEL: test_vector_reduce_mul_v3i16:
; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_pk_mul_lo_u16 v1, v0, v1
-; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: v_mul_lo_u16_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_mul_lo_u16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_mul_lo_u16_e32 v0, v0, v1
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: test_vector_reduce_mul_v3i16:
; GFX10: ; %bb.0: ; %entry
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_lshrrev_b32_e32 v2, 16, v0
-; GFX10-NEXT: v_pk_mul_lo_u16 v0, v0, v1
; GFX10-NEXT: v_mul_lo_u16 v0, v0, v2
+; GFX10-NEXT: v_mul_lo_u16 v0, v0, v1
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_mul_v3i16:
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_pk_mul_lo_u16 v1, v0, v1
+; GFX11-SDAG-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v0.h
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_mul_lo_u16 v0.l, v1.l, v0.h
+; GFX11-SDAG-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v1.l
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-FAKE16-LABEL: test_vector_reduce_mul_v3i16:
; GFX11-SDAG-FAKE16: ; %bb.0: ; %entry
; GFX11-SDAG-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
-; GFX11-SDAG-FAKE16-NEXT: v_pk_mul_lo_u16 v0, v0, v1
-; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-FAKE16-NEXT: v_mul_lo_u16 v0, v0, v2
+; GFX11-SDAG-FAKE16-NEXT: v_mul_lo_u16 v0, v0, v1
; GFX11-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-GISEL-LABEL: test_vector_reduce_mul_v3i16:
; GFX11-GISEL: ; %bb.0: ; %entry
; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
-; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v1
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-GISEL-NEXT: v_mul_lo_u16 v0, v0, v2
+; GFX11-GISEL-NEXT: v_mul_lo_u16 v0, v0, v1
; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-SDAG-TRUE16-LABEL: test_vector_reduce_mul_v3i16:
@@ -1137,9 +1119,9 @@ define i16 @test_vector_reduce_mul_v3i16(<3 x i16> %v) {
; GFX12-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: v_pk_mul_lo_u16 v1, v0, v1
+; GFX12-SDAG-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v0.h
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_mul_lo_u16 v0.l, v1.l, v0.h
+; GFX12-SDAG-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v1.l
; GFX12-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-SDAG-FAKE16-LABEL: test_vector_reduce_mul_v3i16:
@@ -1150,9 +1132,9 @@ define i16 @test_vector_reduce_mul_v3i16(<3 x i16> %v) {
; GFX12-SDAG-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX12-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
-; GFX12-SDAG-FAKE16-NEXT: v_pk_mul_lo_u16 v0, v0, v1
-; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-SDAG-FAKE16-NEXT: v_mul_lo_u16 v0, v0, v2
+; GFX12-SDAG-FAKE16-NEXT: v_mul_lo_u16 v0, v0, v1
; GFX12-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GISEL-LABEL: test_vector_reduce_mul_v3i16:
@@ -1163,9 +1145,9 @@ define i16 @test_vector_reduce_mul_v3i16(<3 x i16> %v) {
; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
-; GFX12-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v1
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-GISEL-NEXT: v_mul_lo_u16 v0, v0, v2
+; GFX12-GISEL-NEXT: v_mul_lo_u16 v0, v0, v1
; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
entry:
%res = call i16 @llvm.vector.reduce.mul.v3i16(<3 x i16> %v)
diff --git a/llvm/test/CodeGen/AMDGPU/vector-reduce-or.ll b/llvm/test/CodeGen/AMDGPU/vector-reduce-or.ll
index c3aa0d752e4cb..5a68c8b7625a7 100644
--- a/llvm/test/CodeGen/AMDGPU/vector-reduce-or.ll
+++ b/llvm/test/CodeGen/AMDGPU/vector-reduce-or.ll
@@ -161,39 +161,25 @@ entry:
}
define i8 @test_vector_reduce_or_v3i8(<3 x i8> %v) {
-; GFX7-SDAG-LABEL: test_vector_reduce_or_v3i8:
-; GFX7-SDAG: ; %bb.0: ; %entry
-; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-SDAG-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX7-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX7-GISEL-LABEL: test_vector_reduce_or_v3i8:
-; GFX7-GISEL: ; %bb.0: ; %entry
-; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX7-GISEL-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX8-SDAG-LABEL: test_vector_reduce_or_v3i8:
-; GFX8-SDAG: ; %bb.0: ; %entry
-; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-SDAG-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX8-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
+; GFX7-LABEL: test_vector_reduce_or_v3i8:
+; GFX7: ; %bb.0: ; %entry
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX7-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-GISEL-LABEL: test_vector_reduce_or_v3i8:
-; GFX8-GISEL: ; %bb.0: ; %entry
-; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX8-GISEL-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
+; GFX8-LABEL: test_vector_reduce_or_v3i8:
+; GFX8: ; %bb.0: ; %entry
+; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-SDAG-LABEL: test_vector_reduce_or_v3i8:
; GFX9-SDAG: ; %bb.0: ; %entry
; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-SDAG-NEXT: v_or_b32_e32 v0, v0, v2
; GFX9-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX9-SDAG-NEXT: v_or_b32_e32 v0, v0, v2
; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-GISEL-LABEL: test_vector_reduce_or_v3i8:
@@ -205,8 +191,8 @@ define i8 @test_vector_reduce_or_v3i8(<3 x i8> %v) {
; GFX10-SDAG-LABEL: test_vector_reduce_or_v3i8:
; GFX10-SDAG: ; %bb.0: ; %entry
; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_or_b32_e32 v0, v0, v2
; GFX10-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX10-SDAG-NEXT: v_or_b32_e32 v0, v0, v2
; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-GISEL-LABEL: test_vector_reduce_or_v3i8:
@@ -218,17 +204,17 @@ define i8 @test_vector_reduce_or_v3i8(<3 x i8> %v) {
; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_or_v3i8:
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v2.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v1.l
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v2.l
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-FAKE16-LABEL: test_vector_reduce_or_v3i8:
; GFX11-SDAG-FAKE16: ; %bb.0: ; %entry
; GFX11-SDAG-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-FAKE16-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-SDAG-FAKE16-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-FAKE16-NEXT: v_or_b32_e32 v0, v0, v2
; GFX11-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-GISEL-LABEL: test_vector_reduce_or_v3i8:
@@ -244,9 +230,9 @@ define i8 @test_vector_reduce_or_v3i8(<3 x i8> %v) {
; GFX12-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v2.l
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-SDAG-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v1.l
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v2.l
; GFX12-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-SDAG-FAKE16-LABEL: test_vector_reduce_or_v3i8:
@@ -256,9 +242,9 @@ define i8 @test_vector_reduce_or_v3i8(<3 x i8> %v) {
; GFX12-SDAG-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-SDAG-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-FAKE16-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-SDAG-FAKE16-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-FAKE16-NEXT: v_or_b32_e32 v0, v0, v2
; GFX12-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GISEL-LABEL: test_vector_reduce_or_v3i8:
@@ -896,10 +882,9 @@ define i16 @test_vector_reduce_or_v3i16(<3 x i16> %v) {
; GFX7-SDAG-LABEL: test_vector_reduce_or_v3i16:
; GFX7-SDAG: ; %bb.0: ; %entry
; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-SDAG-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX7-SDAG-NEXT: v_lshrrev_b32_e32 v2, 16, v0
-; GFX7-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX7-SDAG-NEXT: v_alignbit_b32 v2, v1, v0, 16
; GFX7-SDAG-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX7-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX7-GISEL-LABEL: test_vector_reduce_or_v3i16:
diff --git a/llvm/test/CodeGen/AMDGPU/vector-reduce-smax.ll b/llvm/test/CodeGen/AMDGPU/vector-reduce-smax.ll
index 5f13fece9dd73..bb7c35326ab1c 100644
--- a/llvm/test/CodeGen/AMDGPU/vector-reduce-smax.ll
+++ b/llvm/test/CodeGen/AMDGPU/vector-reduce-smax.ll
@@ -137,11 +137,9 @@ define i8 @test_vector_reduce_smax_v3i8(<3 x i8> %v) {
; GFX7-SDAG: ; %bb.0: ; %entry
; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-SDAG-NEXT: v_bfe_i32 v2, v2, 0, 8
-; GFX7-SDAG-NEXT: v_bfe_i32 v0, v0, 0, 8
; GFX7-SDAG-NEXT: v_bfe_i32 v1, v1, 0, 8
-; GFX7-SDAG-NEXT: v_max_i32_e32 v0, v0, v2
-; GFX7-SDAG-NEXT: s_movk_i32 s4, 0xff80
-; GFX7-SDAG-NEXT: v_max3_i32 v0, v1, v0, s4
+; GFX7-SDAG-NEXT: v_bfe_i32 v0, v0, 0, 8
+; GFX7-SDAG-NEXT: v_max3_i32 v0, v0, v1, v2
; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX7-GISEL-LABEL: test_vector_reduce_smax_v3i8:
@@ -153,28 +151,20 @@ define i8 @test_vector_reduce_smax_v3i8(<3 x i8> %v) {
; GFX7-GISEL-NEXT: v_max3_i32 v0, v0, v1, v2
; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-SDAG-LABEL: test_vector_reduce_smax_v3i8:
-; GFX8-SDAG: ; %bb.0: ; %entry
-; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-SDAG-NEXT: v_max_i16_sdwa v0, sext(v0), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX8-SDAG-NEXT: v_max_i16_sdwa v0, sext(v1), v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX8-SDAG-NEXT: v_max_i16_e32 v0, 0xff80, v0
-; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX8-GISEL-LABEL: test_vector_reduce_smax_v3i8:
-; GFX8-GISEL: ; %bb.0: ; %entry
-; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-GISEL-NEXT: v_max_i16_sdwa v0, sext(v0), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX8-GISEL-NEXT: v_max_i16_sdwa v0, v0, sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
+; GFX8-LABEL: test_vector_reduce_smax_v3i8:
+; GFX8: ; %bb.0: ; %entry
+; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT: v_max_i16_sdwa v0, sext(v0), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX8-NEXT: v_max_i16_sdwa v0, v0, sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-SDAG-LABEL: test_vector_reduce_smax_v3i8:
; GFX9-SDAG: ; %bb.0: ; %entry
; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_bfe_i32 v2, v2, 0, 8
; GFX9-SDAG-NEXT: v_bfe_i32 v1, v1, 0, 8
-; GFX9-SDAG-NEXT: v_max_i16_sdwa v0, sext(v0), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX9-SDAG-NEXT: s_movk_i32 s0, 0xff80
-; GFX9-SDAG-NEXT: v_max3_i16 v0, v1, v0, s0
+; GFX9-SDAG-NEXT: v_bfe_i32 v0, v0, 0, 8
+; GFX9-SDAG-NEXT: v_max3_i16 v0, v0, v1, v2
; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-GISEL-LABEL: test_vector_reduce_smax_v3i8:
@@ -190,10 +180,9 @@ define i8 @test_vector_reduce_smax_v3i8(<3 x i8> %v) {
; GFX10-SDAG: ; %bb.0: ; %entry
; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-SDAG-NEXT: v_bfe_i32 v2, v2, 0, 8
-; GFX10-SDAG-NEXT: v_bfe_i32 v0, v0, 0, 8
; GFX10-SDAG-NEXT: v_bfe_i32 v1, v1, 0, 8
-; GFX10-SDAG-NEXT: v_max_i16 v0, v0, v2
-; GFX10-SDAG-NEXT: v_max3_i16 v0, v1, v0, 0xff80
+; GFX10-SDAG-NEXT: v_bfe_i32 v0, v0, 0, 8
+; GFX10-SDAG-NEXT: v_max3_i16 v0, v0, v1, v2
; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-GISEL-LABEL: test_vector_reduce_smax_v3i8:
@@ -208,28 +197,21 @@ define i8 @test_vector_reduce_smax_v3i8(<3 x i8> %v) {
; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_smax_v3i8:
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v2, v2, 0, 8
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l
; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v0, v0, 0, 8
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
-; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v2, v3, 0, 8
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT: v_max_i16 v0.l, v0.l, v1.l
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
+; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v1, v1, 0, 8
+; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v2, v2, 0, 8
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_max3_i16 v0.l, v1.l, v0.l, 0xff80
+; GFX11-SDAG-TRUE16-NEXT: v_max3_i16 v0.l, v0.l, v1.l, v2.l
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-FAKE16-LABEL: test_vector_reduce_smax_v3i8:
; GFX11-SDAG-FAKE16: ; %bb.0: ; %entry
; GFX11-SDAG-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-SDAG-FAKE16-NEXT: v_bfe_i32 v2, v2, 0, 8
-; GFX11-SDAG-FAKE16-NEXT: v_bfe_i32 v0, v0, 0, 8
; GFX11-SDAG-FAKE16-NEXT: v_bfe_i32 v1, v1, 0, 8
-; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-FAKE16-NEXT: v_max_i16 v0, v0, v2
-; GFX11-SDAG-FAKE16-NEXT: v_max3_i16 v0, v1, v0, 0xff80
+; GFX11-SDAG-FAKE16-NEXT: v_bfe_i32 v0, v0, 0, 8
+; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-FAKE16-NEXT: v_max3_i16 v0, v0, v1, v2
; GFX11-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-GISEL-LABEL: test_vector_reduce_smax_v3i8:
@@ -249,17 +231,11 @@ define i8 @test_vector_reduce_smax_v3i8(<3 x i8> %v) {
; GFX12-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v2, v2, 0, 8
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l
; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v0, v0, 0, 8
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
-; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v2, v3, 0, 8
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-SDAG-TRUE16-NEXT: v_max_i16 v0.l, v0.l, v1.l
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
+; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v1, v1, 0, 8
+; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v2, v2, 0, 8
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_max3_i16 v0.l, v1.l, v0.l, 0xff80
+; GFX12-SDAG-TRUE16-NEXT: v_max3_i16 v0.l, v0.l, v1.l, v2.l
; GFX12-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-SDAG-FAKE16-LABEL: test_vector_reduce_smax_v3i8:
@@ -270,11 +246,10 @@ define i8 @test_vector_reduce_smax_v3i8(<3 x i8> %v) {
; GFX12-SDAG-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX12-SDAG-FAKE16-NEXT: v_bfe_i32 v2, v2, 0, 8
-; GFX12-SDAG-FAKE16-NEXT: v_bfe_i32 v0, v0, 0, 8
; GFX12-SDAG-FAKE16-NEXT: v_bfe_i32 v1, v1, 0, 8
-; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-FAKE16-NEXT: v_max_i16 v0, v0, v2
-; GFX12-SDAG-FAKE16-NEXT: v_max3_i16 v0, v1, v0, 0xff80
+; GFX12-SDAG-FAKE16-NEXT: v_bfe_i32 v0, v0, 0, 8
+; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-FAKE16-NEXT: v_max3_i16 v0, v0, v1, v2
; GFX12-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GISEL-LABEL: test_vector_reduce_smax_v3i8:
@@ -1236,12 +1211,10 @@ define i16 @test_vector_reduce_smax_v3i16(<3 x i16> %v) {
; GFX7-SDAG-LABEL: test_vector_reduce_smax_v3i16:
; GFX7-SDAG: ; %bb.0: ; %entry
; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-SDAG-NEXT: v_ashrrev_i32_e32 v2, 16, v0
+; GFX7-SDAG-NEXT: v_bfe_i32 v2, v0, 0, 16
+; GFX7-SDAG-NEXT: v_ashrrev_i32_e32 v0, 16, v0
; GFX7-SDAG-NEXT: v_bfe_i32 v1, v1, 0, 16
-; GFX7-SDAG-NEXT: v_bfe_i32 v0, v0, 0, 16
-; GFX7-SDAG-NEXT: v_max_i32_e32 v0, v0, v1
-; GFX7-SDAG-NEXT: s_movk_i32 s4, 0x8000
-; GFX7-SDAG-NEXT: v_max3_i32 v0, v0, v2, s4
+; GFX7-SDAG-NEXT: v_max3_i32 v0, v2, v0, v1
; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX7-GISEL-LABEL: test_vector_reduce_smax_v3i16:
@@ -1263,55 +1236,37 @@ define i16 @test_vector_reduce_smax_v3i16(<3 x i16> %v) {
; GFX9-LABEL: test_vector_reduce_smax_v3i16:
; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: s_movk_i32 s0, 0x8000
-; GFX9-NEXT: v_mov_b32_e32 v2, 0x5040100
-; GFX9-NEXT: v_perm_b32 v1, s0, v1, v2
-; GFX9-NEXT: v_pk_max_i16 v0, v0, v1
-; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: v_max_i16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX9-NEXT: v_max3_i16 v0, v0, v2, v1
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: test_vector_reduce_smax_v3i16:
; GFX10: ; %bb.0: ; %entry
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_movk_i32 s4, 0x8000
-; GFX10-NEXT: v_perm_b32 v1, s4, v1, 0x5040100
-; GFX10-NEXT: v_pk_max_i16 v0, v0, v1
-; GFX10-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX10-NEXT: v_max_i16 v0, v0, v1
+; GFX10-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX10-NEXT: v_max3_i16 v0, v0, v2, v1
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_smax_v3i16:
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.h, 0x8000
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_pk_max_i16 v0, v0, v1
-; GFX11-SDAG-TRUE16-NEXT: v_max_i16 v0.l, v0.l, v0.h
+; GFX11-SDAG-TRUE16-NEXT: v_max3_i16 v0.l, v0.l, v0.h, v1.l
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-FAKE16-LABEL: test_vector_reduce_smax_v3i16:
; GFX11-SDAG-FAKE16: ; %bb.0: ; %entry
; GFX11-SDAG-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-FAKE16-NEXT: s_movk_i32 s0, 0x8000
-; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-FAKE16-NEXT: v_perm_b32 v1, s0, v1, 0x5040100
-; GFX11-SDAG-FAKE16-NEXT: v_pk_max_i16 v0, v0, v1
-; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX11-SDAG-FAKE16-NEXT: v_max_i16 v0, v0, v1
+; GFX11-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-FAKE16-NEXT: v_max3_i16 v0, v0, v2, v1
; GFX11-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-GISEL-LABEL: test_vector_reduce_smax_v3i16:
; GFX11-GISEL: ; %bb.0: ; %entry
; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-NEXT: s_movk_i32 s0, 0x8000
-; GFX11-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-NEXT: v_perm_b32 v1, s0, v1, 0x5040100
-; GFX11-GISEL-NEXT: v_pk_max_i16 v0, v0, v1
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX11-GISEL-NEXT: v_max_i16 v0, v0, v1
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_max3_i16 v0, v0, v2, v1
; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-SDAG-TRUE16-LABEL: test_vector_reduce_smax_v3i16:
@@ -1321,10 +1276,7 @@ define i16 @test_vector_reduce_smax_v3i16(<3 x i16> %v) {
; GFX12-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.h, 0x8000
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_pk_max_i16 v0, v0, v1
-; GFX12-SDAG-TRUE16-NEXT: v_max_i16 v0.l, v0.l, v0.h
+; GFX12-SDAG-TRUE16-NEXT: v_max3_i16 v0.l, v0.l, v0.h, v1.l
; GFX12-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-SDAG-FAKE16-LABEL: test_vector_reduce_smax_v3i16:
@@ -1334,14 +1286,9 @@ define i16 @test_vector_reduce_smax_v3i16(<3 x i16> %v) {
; GFX12-SDAG-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-SDAG-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-FAKE16-NEXT: s_movk_i32 s0, 0x8000
-; GFX12-SDAG-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-SDAG-FAKE16-NEXT: v_perm_b32 v1, s0, v1, 0x5040100
-; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-FAKE16-NEXT: v_pk_max_i16 v0, v0, v1
-; GFX12-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX12-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-SDAG-FAKE16-NEXT: v_max_i16 v0, v0, v1
+; GFX12-SDAG-FAKE16-NEXT: v_max3_i16 v0, v0, v2, v1
; GFX12-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GISEL-LABEL: test_vector_reduce_smax_v3i16:
@@ -1351,14 +1298,9 @@ define i16 @test_vector_reduce_smax_v3i16(<3 x i16> %v) {
; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
-; GFX12-GISEL-NEXT: s_movk_i32 s0, 0x8000
-; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GISEL-NEXT: v_perm_b32 v1, s0, v1, 0x5040100
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-GISEL-NEXT: v_pk_max_i16 v0, v0, v1
-; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-GISEL-NEXT: v_max_i16 v0, v0, v1
+; GFX12-GISEL-NEXT: v_max3_i16 v0, v0, v2, v1
; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
entry:
%res = call i16 @llvm.vector.reduce.smax.v3i16(<3 x i16> %v)
diff --git a/llvm/test/CodeGen/AMDGPU/vector-reduce-smin.ll b/llvm/test/CodeGen/AMDGPU/vector-reduce-smin.ll
index 94ac809189bd5..eacc885752acf 100644
--- a/llvm/test/CodeGen/AMDGPU/vector-reduce-smin.ll
+++ b/llvm/test/CodeGen/AMDGPU/vector-reduce-smin.ll
@@ -137,11 +137,9 @@ define i8 @test_vector_reduce_smin_v3i8(<3 x i8> %v) {
; GFX7-SDAG: ; %bb.0: ; %entry
; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-SDAG-NEXT: v_bfe_i32 v2, v2, 0, 8
-; GFX7-SDAG-NEXT: v_bfe_i32 v0, v0, 0, 8
; GFX7-SDAG-NEXT: v_bfe_i32 v1, v1, 0, 8
-; GFX7-SDAG-NEXT: v_min_i32_e32 v0, v0, v2
-; GFX7-SDAG-NEXT: s_movk_i32 s4, 0x7f
-; GFX7-SDAG-NEXT: v_min3_i32 v0, v1, v0, s4
+; GFX7-SDAG-NEXT: v_bfe_i32 v0, v0, 0, 8
+; GFX7-SDAG-NEXT: v_min3_i32 v0, v0, v1, v2
; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX7-GISEL-LABEL: test_vector_reduce_smin_v3i8:
@@ -153,28 +151,20 @@ define i8 @test_vector_reduce_smin_v3i8(<3 x i8> %v) {
; GFX7-GISEL-NEXT: v_min3_i32 v0, v0, v1, v2
; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-SDAG-LABEL: test_vector_reduce_smin_v3i8:
-; GFX8-SDAG: ; %bb.0: ; %entry
-; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-SDAG-NEXT: v_min_i16_sdwa v0, sext(v0), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX8-SDAG-NEXT: v_min_i16_sdwa v0, sext(v1), v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX8-SDAG-NEXT: v_min_i16_e32 v0, 0x7f, v0
-; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX8-GISEL-LABEL: test_vector_reduce_smin_v3i8:
-; GFX8-GISEL: ; %bb.0: ; %entry
-; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-GISEL-NEXT: v_min_i16_sdwa v0, sext(v0), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX8-GISEL-NEXT: v_min_i16_sdwa v0, v0, sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
+; GFX8-LABEL: test_vector_reduce_smin_v3i8:
+; GFX8: ; %bb.0: ; %entry
+; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT: v_min_i16_sdwa v0, sext(v0), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX8-NEXT: v_min_i16_sdwa v0, v0, sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-SDAG-LABEL: test_vector_reduce_smin_v3i8:
; GFX9-SDAG: ; %bb.0: ; %entry
; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_bfe_i32 v2, v2, 0, 8
; GFX9-SDAG-NEXT: v_bfe_i32 v1, v1, 0, 8
-; GFX9-SDAG-NEXT: v_min_i16_sdwa v0, sext(v0), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX9-SDAG-NEXT: s_movk_i32 s0, 0x7f
-; GFX9-SDAG-NEXT: v_min3_i16 v0, v1, v0, s0
+; GFX9-SDAG-NEXT: v_bfe_i32 v0, v0, 0, 8
+; GFX9-SDAG-NEXT: v_min3_i16 v0, v0, v1, v2
; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-GISEL-LABEL: test_vector_reduce_smin_v3i8:
@@ -190,10 +180,9 @@ define i8 @test_vector_reduce_smin_v3i8(<3 x i8> %v) {
; GFX10-SDAG: ; %bb.0: ; %entry
; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-SDAG-NEXT: v_bfe_i32 v2, v2, 0, 8
-; GFX10-SDAG-NEXT: v_bfe_i32 v0, v0, 0, 8
; GFX10-SDAG-NEXT: v_bfe_i32 v1, v1, 0, 8
-; GFX10-SDAG-NEXT: v_min_i16 v0, v0, v2
-; GFX10-SDAG-NEXT: v_min3_i16 v0, v1, v0, 0x7f
+; GFX10-SDAG-NEXT: v_bfe_i32 v0, v0, 0, 8
+; GFX10-SDAG-NEXT: v_min3_i16 v0, v0, v1, v2
; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-GISEL-LABEL: test_vector_reduce_smin_v3i8:
@@ -208,28 +197,21 @@ define i8 @test_vector_reduce_smin_v3i8(<3 x i8> %v) {
; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_smin_v3i8:
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v2, v2, 0, 8
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l
; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v0, v0, 0, 8
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
-; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v2, v3, 0, 8
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT: v_min_i16 v0.l, v0.l, v1.l
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
+; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v1, v1, 0, 8
+; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v2, v2, 0, 8
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_min3_i16 v0.l, v1.l, v0.l, 0x7f
+; GFX11-SDAG-TRUE16-NEXT: v_min3_i16 v0.l, v0.l, v1.l, v2.l
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-FAKE16-LABEL: test_vector_reduce_smin_v3i8:
; GFX11-SDAG-FAKE16: ; %bb.0: ; %entry
; GFX11-SDAG-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-SDAG-FAKE16-NEXT: v_bfe_i32 v2, v2, 0, 8
-; GFX11-SDAG-FAKE16-NEXT: v_bfe_i32 v0, v0, 0, 8
; GFX11-SDAG-FAKE16-NEXT: v_bfe_i32 v1, v1, 0, 8
-; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-FAKE16-NEXT: v_min_i16 v0, v0, v2
-; GFX11-SDAG-FAKE16-NEXT: v_min3_i16 v0, v1, v0, 0x7f
+; GFX11-SDAG-FAKE16-NEXT: v_bfe_i32 v0, v0, 0, 8
+; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-FAKE16-NEXT: v_min3_i16 v0, v0, v1, v2
; GFX11-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-GISEL-LABEL: test_vector_reduce_smin_v3i8:
@@ -249,17 +231,11 @@ define i8 @test_vector_reduce_smin_v3i8(<3 x i8> %v) {
; GFX12-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v2, v2, 0, 8
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l
; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v0, v0, 0, 8
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
-; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v2, v3, 0, 8
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-SDAG-TRUE16-NEXT: v_min_i16 v0.l, v0.l, v1.l
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
+; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v1, v1, 0, 8
+; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v2, v2, 0, 8
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_min3_i16 v0.l, v1.l, v0.l, 0x7f
+; GFX12-SDAG-TRUE16-NEXT: v_min3_i16 v0.l, v0.l, v1.l, v2.l
; GFX12-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-SDAG-FAKE16-LABEL: test_vector_reduce_smin_v3i8:
@@ -270,11 +246,10 @@ define i8 @test_vector_reduce_smin_v3i8(<3 x i8> %v) {
; GFX12-SDAG-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX12-SDAG-FAKE16-NEXT: v_bfe_i32 v2, v2, 0, 8
-; GFX12-SDAG-FAKE16-NEXT: v_bfe_i32 v0, v0, 0, 8
; GFX12-SDAG-FAKE16-NEXT: v_bfe_i32 v1, v1, 0, 8
-; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-FAKE16-NEXT: v_min_i16 v0, v0, v2
-; GFX12-SDAG-FAKE16-NEXT: v_min3_i16 v0, v1, v0, 0x7f
+; GFX12-SDAG-FAKE16-NEXT: v_bfe_i32 v0, v0, 0, 8
+; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-FAKE16-NEXT: v_min3_i16 v0, v0, v1, v2
; GFX12-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GISEL-LABEL: test_vector_reduce_smin_v3i8:
@@ -1236,12 +1211,10 @@ define i16 @test_vector_reduce_smin_v3i16(<3 x i16> %v) {
; GFX7-SDAG-LABEL: test_vector_reduce_smin_v3i16:
; GFX7-SDAG: ; %bb.0: ; %entry
; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-SDAG-NEXT: v_ashrrev_i32_e32 v2, 16, v0
+; GFX7-SDAG-NEXT: v_bfe_i32 v2, v0, 0, 16
+; GFX7-SDAG-NEXT: v_ashrrev_i32_e32 v0, 16, v0
; GFX7-SDAG-NEXT: v_bfe_i32 v1, v1, 0, 16
-; GFX7-SDAG-NEXT: v_bfe_i32 v0, v0, 0, 16
-; GFX7-SDAG-NEXT: v_min_i32_e32 v0, v0, v1
-; GFX7-SDAG-NEXT: s_movk_i32 s4, 0x7fff
-; GFX7-SDAG-NEXT: v_min3_i32 v0, v0, v2, s4
+; GFX7-SDAG-NEXT: v_min3_i32 v0, v2, v0, v1
; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX7-GISEL-LABEL: test_vector_reduce_smin_v3i16:
@@ -1263,55 +1236,37 @@ define i16 @test_vector_reduce_smin_v3i16(<3 x i16> %v) {
; GFX9-LABEL: test_vector_reduce_smin_v3i16:
; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: s_movk_i32 s0, 0x7fff
-; GFX9-NEXT: v_mov_b32_e32 v2, 0x5040100
-; GFX9-NEXT: v_perm_b32 v1, s0, v1, v2
-; GFX9-NEXT: v_pk_min_i16 v0, v0, v1
-; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: v_min_i16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX9-NEXT: v_min3_i16 v0, v0, v2, v1
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: test_vector_reduce_smin_v3i16:
; GFX10: ; %bb.0: ; %entry
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_movk_i32 s4, 0x7fff
-; GFX10-NEXT: v_perm_b32 v1, s4, v1, 0x5040100
-; GFX10-NEXT: v_pk_min_i16 v0, v0, v1
-; GFX10-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX10-NEXT: v_min_i16 v0, v0, v1
+; GFX10-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX10-NEXT: v_min3_i16 v0, v0, v2, v1
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_smin_v3i16:
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.h, 0x7fff
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_pk_min_i16 v0, v0, v1
-; GFX11-SDAG-TRUE16-NEXT: v_min_i16 v0.l, v0.l, v0.h
+; GFX11-SDAG-TRUE16-NEXT: v_min3_i16 v0.l, v0.l, v0.h, v1.l
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-FAKE16-LABEL: test_vector_reduce_smin_v3i16:
; GFX11-SDAG-FAKE16: ; %bb.0: ; %entry
; GFX11-SDAG-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-FAKE16-NEXT: s_movk_i32 s0, 0x7fff
-; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-FAKE16-NEXT: v_perm_b32 v1, s0, v1, 0x5040100
-; GFX11-SDAG-FAKE16-NEXT: v_pk_min_i16 v0, v0, v1
-; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX11-SDAG-FAKE16-NEXT: v_min_i16 v0, v0, v1
+; GFX11-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-FAKE16-NEXT: v_min3_i16 v0, v0, v2, v1
; GFX11-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-GISEL-LABEL: test_vector_reduce_smin_v3i16:
; GFX11-GISEL: ; %bb.0: ; %entry
; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-NEXT: s_movk_i32 s0, 0x7fff
-; GFX11-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-NEXT: v_perm_b32 v1, s0, v1, 0x5040100
-; GFX11-GISEL-NEXT: v_pk_min_i16 v0, v0, v1
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX11-GISEL-NEXT: v_min_i16 v0, v0, v1
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_min3_i16 v0, v0, v2, v1
; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-SDAG-TRUE16-LABEL: test_vector_reduce_smin_v3i16:
@@ -1321,10 +1276,7 @@ define i16 @test_vector_reduce_smin_v3i16(<3 x i16> %v) {
; GFX12-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.h, 0x7fff
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_pk_min_i16 v0, v0, v1
-; GFX12-SDAG-TRUE16-NEXT: v_min_i16 v0.l, v0.l, v0.h
+; GFX12-SDAG-TRUE16-NEXT: v_min3_i16 v0.l, v0.l, v0.h, v1.l
; GFX12-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-SDAG-FAKE16-LABEL: test_vector_reduce_smin_v3i16:
@@ -1334,14 +1286,9 @@ define i16 @test_vector_reduce_smin_v3i16(<3 x i16> %v) {
; GFX12-SDAG-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-SDAG-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-FAKE16-NEXT: s_movk_i32 s0, 0x7fff
-; GFX12-SDAG-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-SDAG-FAKE16-NEXT: v_perm_b32 v1, s0, v1, 0x5040100
-; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-FAKE16-NEXT: v_pk_min_i16 v0, v0, v1
-; GFX12-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX12-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-SDAG-FAKE16-NEXT: v_min_i16 v0, v0, v1
+; GFX12-SDAG-FAKE16-NEXT: v_min3_i16 v0, v0, v2, v1
; GFX12-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GISEL-LABEL: test_vector_reduce_smin_v3i16:
@@ -1351,14 +1298,9 @@ define i16 @test_vector_reduce_smin_v3i16(<3 x i16> %v) {
; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
-; GFX12-GISEL-NEXT: s_movk_i32 s0, 0x7fff
-; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GISEL-NEXT: v_perm_b32 v1, s0, v1, 0x5040100
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-GISEL-NEXT: v_pk_min_i16 v0, v0, v1
-; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-GISEL-NEXT: v_min_i16 v0, v0, v1
+; GFX12-GISEL-NEXT: v_min3_i16 v0, v0, v2, v1
; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
entry:
%res = call i16 @llvm.vector.reduce.smin.v3i16(<3 x i16> %v)
diff --git a/llvm/test/CodeGen/AMDGPU/vector-reduce-umax.ll b/llvm/test/CodeGen/AMDGPU/vector-reduce-umax.ll
index 2eb06b2f2c1f5..dfea02c8c4b18 100644
--- a/llvm/test/CodeGen/AMDGPU/vector-reduce-umax.ll
+++ b/llvm/test/CodeGen/AMDGPU/vector-reduce-umax.ll
@@ -137,9 +137,9 @@ define i8 @test_vector_reduce_umax_v3i8(<3 x i8> %v) {
; GFX7-SDAG: ; %bb.0: ; %entry
; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-SDAG-NEXT: v_and_b32_e32 v2, 0xff, v2
-; GFX7-SDAG-NEXT: v_and_b32_e32 v0, 0xff, v0
; GFX7-SDAG-NEXT: v_and_b32_e32 v1, 0xff, v1
-; GFX7-SDAG-NEXT: v_max3_u32 v0, v1, v0, v2
+; GFX7-SDAG-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX7-SDAG-NEXT: v_max3_u32 v0, v0, v1, v2
; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX7-GISEL-LABEL: test_vector_reduce_umax_v3i8:
@@ -151,27 +151,20 @@ define i8 @test_vector_reduce_umax_v3i8(<3 x i8> %v) {
; GFX7-GISEL-NEXT: v_max3_u32 v0, v0, v1, v2
; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-SDAG-LABEL: test_vector_reduce_umax_v3i8:
-; GFX8-SDAG: ; %bb.0: ; %entry
-; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-SDAG-NEXT: v_max_u16_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX8-SDAG-NEXT: v_max_u16_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX8-GISEL-LABEL: test_vector_reduce_umax_v3i8:
-; GFX8-GISEL: ; %bb.0: ; %entry
-; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-GISEL-NEXT: v_max_u16_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX8-GISEL-NEXT: v_max_u16_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
+; GFX8-LABEL: test_vector_reduce_umax_v3i8:
+; GFX8: ; %bb.0: ; %entry
+; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT: v_max_u16_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX8-NEXT: v_max_u16_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-SDAG-LABEL: test_vector_reduce_umax_v3i8:
; GFX9-SDAG: ; %bb.0: ; %entry
; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-SDAG-NEXT: v_and_b32_e32 v2, 0xff, v2
-; GFX9-SDAG-NEXT: v_and_b32_e32 v0, 0xff, v0
; GFX9-SDAG-NEXT: v_and_b32_e32 v1, 0xff, v1
-; GFX9-SDAG-NEXT: v_max3_u16 v0, v1, v0, v2
+; GFX9-SDAG-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX9-SDAG-NEXT: v_max3_u16 v0, v0, v1, v2
; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-GISEL-LABEL: test_vector_reduce_umax_v3i8:
@@ -187,9 +180,9 @@ define i8 @test_vector_reduce_umax_v3i8(<3 x i8> %v) {
; GFX10-SDAG: ; %bb.0: ; %entry
; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-SDAG-NEXT: v_and_b32_e32 v2, 0xff, v2
-; GFX10-SDAG-NEXT: v_and_b32_e32 v0, 0xff, v0
; GFX10-SDAG-NEXT: v_and_b32_e32 v1, 0xff, v1
-; GFX10-SDAG-NEXT: v_max3_u16 v0, v1, v0, v2
+; GFX10-SDAG-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX10-SDAG-NEXT: v_max3_u16 v0, v0, v1, v2
; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-GISEL-LABEL: test_vector_reduce_umax_v3i8:
@@ -205,20 +198,20 @@ define i8 @test_vector_reduce_umax_v3i8(<3 x i8> %v) {
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v2.l
-; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l
+; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_max3_u16 v0.l, v1.l, v0.l, v0.h
+; GFX11-SDAG-TRUE16-NEXT: v_max3_u16 v0.l, v0.l, v1.l, v0.h
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-FAKE16-LABEL: test_vector_reduce_umax_v3i8:
; GFX11-SDAG-FAKE16: ; %bb.0: ; %entry
; GFX11-SDAG-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-SDAG-FAKE16-NEXT: v_and_b32_e32 v2, 0xff, v2
-; GFX11-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0xff, v0
; GFX11-SDAG-FAKE16-NEXT: v_and_b32_e32 v1, 0xff, v1
+; GFX11-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0xff, v0
; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-SDAG-FAKE16-NEXT: v_max3_u16 v0, v1, v0, v2
+; GFX11-SDAG-FAKE16-NEXT: v_max3_u16 v0, v0, v1, v2
; GFX11-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-GISEL-LABEL: test_vector_reduce_umax_v3i8:
@@ -239,10 +232,10 @@ define i8 @test_vector_reduce_umax_v3i8(<3 x i8> %v) {
; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v2.l
-; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l
+; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_max3_u16 v0.l, v1.l, v0.l, v0.h
+; GFX12-SDAG-TRUE16-NEXT: v_max3_u16 v0.l, v0.l, v1.l, v0.h
; GFX12-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-SDAG-FAKE16-LABEL: test_vector_reduce_umax_v3i8:
@@ -253,10 +246,10 @@ define i8 @test_vector_reduce_umax_v3i8(<3 x i8> %v) {
; GFX12-SDAG-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX12-SDAG-FAKE16-NEXT: v_and_b32_e32 v2, 0xff, v2
-; GFX12-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0xff, v0
; GFX12-SDAG-FAKE16-NEXT: v_and_b32_e32 v1, 0xff, v1
+; GFX12-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0xff, v0
; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-SDAG-FAKE16-NEXT: v_max3_u16 v0, v1, v0, v2
+; GFX12-SDAG-FAKE16-NEXT: v_max3_u16 v0, v0, v1, v2
; GFX12-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GISEL-LABEL: test_vector_reduce_umax_v3i8:
@@ -1161,11 +1154,10 @@ define i16 @test_vector_reduce_umax_v3i16(<3 x i16> %v) {
; GFX7-SDAG-LABEL: test_vector_reduce_umax_v3i16:
; GFX7-SDAG: ; %bb.0: ; %entry
; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-SDAG-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX7-SDAG-NEXT: v_and_b32_e32 v2, 0xffff, v0
-; GFX7-SDAG-NEXT: v_max_u32_e32 v1, v2, v1
; GFX7-SDAG-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX7-SDAG-NEXT: v_max3_u32 v0, v1, v0, 0
+; GFX7-SDAG-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX7-SDAG-NEXT: v_max3_u32 v0, v2, v0, v1
; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX7-GISEL-LABEL: test_vector_reduce_umax_v3i16:
@@ -1187,50 +1179,37 @@ define i16 @test_vector_reduce_umax_v3i16(<3 x i16> %v) {
; GFX9-LABEL: test_vector_reduce_umax_v3i16:
; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_pk_max_u16 v0, v0, v1
-; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: v_max_u16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX9-NEXT: v_max3_u16 v0, v0, v2, v1
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: test_vector_reduce_umax_v3i16:
; GFX10: ; %bb.0: ; %entry
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10-NEXT: v_pk_max_u16 v0, v0, v1
-; GFX10-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX10-NEXT: v_max_u16 v0, v0, v1
+; GFX10-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX10-NEXT: v_max3_u16 v0, v0, v2, v1
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_umax_v3i16:
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_cvt_u32_u16_e32 v1, v1.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_pk_max_u16 v0, v0, v1
-; GFX11-SDAG-TRUE16-NEXT: v_max_u16 v0.l, v0.l, v0.h
+; GFX11-SDAG-TRUE16-NEXT: v_max3_u16 v0.l, v0.l, v0.h, v1.l
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-FAKE16-LABEL: test_vector_reduce_umax_v3i16:
; GFX11-SDAG-FAKE16: ; %bb.0: ; %entry
; GFX11-SDAG-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-FAKE16-NEXT: v_pk_max_u16 v0, v0, v1
-; GFX11-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX11-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-SDAG-FAKE16-NEXT: v_max_u16 v0, v0, v1
+; GFX11-SDAG-FAKE16-NEXT: v_max3_u16 v0, v0, v2, v1
; GFX11-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-GISEL-LABEL: test_vector_reduce_umax_v3i16:
; GFX11-GISEL: ; %bb.0: ; %entry
; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-NEXT: v_pk_max_u16 v0, v0, v1
-; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-GISEL-NEXT: v_max_u16 v0, v0, v1
+; GFX11-GISEL-NEXT: v_max3_u16 v0, v0, v2, v1
; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-SDAG-TRUE16-LABEL: test_vector_reduce_umax_v3i16:
@@ -1240,10 +1219,7 @@ define i16 @test_vector_reduce_umax_v3i16(<3 x i16> %v) {
; GFX12-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: v_cvt_u32_u16_e32 v1, v1.l
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_pk_max_u16 v0, v0, v1
-; GFX12-SDAG-TRUE16-NEXT: v_max_u16 v0.l, v0.l, v0.h
+; GFX12-SDAG-TRUE16-NEXT: v_max3_u16 v0.l, v0.l, v0.h, v1.l
; GFX12-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-SDAG-FAKE16-LABEL: test_vector_reduce_umax_v3i16:
@@ -1253,12 +1229,9 @@ define i16 @test_vector_reduce_umax_v3i16(<3 x i16> %v) {
; GFX12-SDAG-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-SDAG-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-FAKE16-NEXT: v_pk_max_u16 v0, v0, v1
-; GFX12-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX12-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-SDAG-FAKE16-NEXT: v_max_u16 v0, v0, v1
+; GFX12-SDAG-FAKE16-NEXT: v_max3_u16 v0, v0, v2, v1
; GFX12-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GISEL-LABEL: test_vector_reduce_umax_v3i16:
@@ -1268,12 +1241,9 @@ define i16 @test_vector_reduce_umax_v3i16(<3 x i16> %v) {
; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
-; GFX12-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-GISEL-NEXT: v_pk_max_u16 v0, v0, v1
-; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-GISEL-NEXT: v_max_u16 v0, v0, v1
+; GFX12-GISEL-NEXT: v_max3_u16 v0, v0, v2, v1
; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
entry:
%res = call i16 @llvm.vector.reduce.umax.v3i16(<3 x i16> %v)
diff --git a/llvm/test/CodeGen/AMDGPU/vector-reduce-umin.ll b/llvm/test/CodeGen/AMDGPU/vector-reduce-umin.ll
index 7c3e52e778986..c0b721e000458 100644
--- a/llvm/test/CodeGen/AMDGPU/vector-reduce-umin.ll
+++ b/llvm/test/CodeGen/AMDGPU/vector-reduce-umin.ll
@@ -137,9 +137,9 @@ define i8 @test_vector_reduce_umin_v3i8(<3 x i8> %v) {
; GFX7-SDAG: ; %bb.0: ; %entry
; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-SDAG-NEXT: v_and_b32_e32 v2, 0xff, v2
-; GFX7-SDAG-NEXT: v_and_b32_e32 v0, 0xff, v0
; GFX7-SDAG-NEXT: v_and_b32_e32 v1, 0xff, v1
-; GFX7-SDAG-NEXT: v_min3_u32 v0, v1, v0, v2
+; GFX7-SDAG-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX7-SDAG-NEXT: v_min3_u32 v0, v0, v1, v2
; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX7-GISEL-LABEL: test_vector_reduce_umin_v3i8:
@@ -151,27 +151,20 @@ define i8 @test_vector_reduce_umin_v3i8(<3 x i8> %v) {
; GFX7-GISEL-NEXT: v_min3_u32 v0, v0, v1, v2
; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-SDAG-LABEL: test_vector_reduce_umin_v3i8:
-; GFX8-SDAG: ; %bb.0: ; %entry
-; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-SDAG-NEXT: v_min_u16_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX8-SDAG-NEXT: v_min_u16_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX8-GISEL-LABEL: test_vector_reduce_umin_v3i8:
-; GFX8-GISEL: ; %bb.0: ; %entry
-; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-GISEL-NEXT: v_min_u16_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX8-GISEL-NEXT: v_min_u16_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
+; GFX8-LABEL: test_vector_reduce_umin_v3i8:
+; GFX8: ; %bb.0: ; %entry
+; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT: v_min_u16_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX8-NEXT: v_min_u16_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-SDAG-LABEL: test_vector_reduce_umin_v3i8:
; GFX9-SDAG: ; %bb.0: ; %entry
; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-SDAG-NEXT: v_and_b32_e32 v2, 0xff, v2
-; GFX9-SDAG-NEXT: v_and_b32_e32 v0, 0xff, v0
; GFX9-SDAG-NEXT: v_and_b32_e32 v1, 0xff, v1
-; GFX9-SDAG-NEXT: v_min3_u16 v0, v1, v0, v2
+; GFX9-SDAG-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX9-SDAG-NEXT: v_min3_u16 v0, v0, v1, v2
; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-GISEL-LABEL: test_vector_reduce_umin_v3i8:
@@ -187,9 +180,9 @@ define i8 @test_vector_reduce_umin_v3i8(<3 x i8> %v) {
; GFX10-SDAG: ; %bb.0: ; %entry
; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-SDAG-NEXT: v_and_b32_e32 v2, 0xff, v2
-; GFX10-SDAG-NEXT: v_and_b32_e32 v0, 0xff, v0
; GFX10-SDAG-NEXT: v_and_b32_e32 v1, 0xff, v1
-; GFX10-SDAG-NEXT: v_min3_u16 v0, v1, v0, v2
+; GFX10-SDAG-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX10-SDAG-NEXT: v_min3_u16 v0, v0, v1, v2
; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-GISEL-LABEL: test_vector_reduce_umin_v3i8:
@@ -205,20 +198,20 @@ define i8 @test_vector_reduce_umin_v3i8(<3 x i8> %v) {
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v2.l
-; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l
+; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_min3_u16 v0.l, v1.l, v0.l, v0.h
+; GFX11-SDAG-TRUE16-NEXT: v_min3_u16 v0.l, v0.l, v1.l, v0.h
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-FAKE16-LABEL: test_vector_reduce_umin_v3i8:
; GFX11-SDAG-FAKE16: ; %bb.0: ; %entry
; GFX11-SDAG-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-SDAG-FAKE16-NEXT: v_and_b32_e32 v2, 0xff, v2
-; GFX11-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0xff, v0
; GFX11-SDAG-FAKE16-NEXT: v_and_b32_e32 v1, 0xff, v1
+; GFX11-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0xff, v0
; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-SDAG-FAKE16-NEXT: v_min3_u16 v0, v1, v0, v2
+; GFX11-SDAG-FAKE16-NEXT: v_min3_u16 v0, v0, v1, v2
; GFX11-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-GISEL-LABEL: test_vector_reduce_umin_v3i8:
@@ -239,10 +232,10 @@ define i8 @test_vector_reduce_umin_v3i8(<3 x i8> %v) {
; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v2.l
-; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l
+; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_min3_u16 v0.l, v1.l, v0.l, v0.h
+; GFX12-SDAG-TRUE16-NEXT: v_min3_u16 v0.l, v0.l, v1.l, v0.h
; GFX12-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-SDAG-FAKE16-LABEL: test_vector_reduce_umin_v3i8:
@@ -253,10 +246,10 @@ define i8 @test_vector_reduce_umin_v3i8(<3 x i8> %v) {
; GFX12-SDAG-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX12-SDAG-FAKE16-NEXT: v_and_b32_e32 v2, 0xff, v2
-; GFX12-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0xff, v0
; GFX12-SDAG-FAKE16-NEXT: v_and_b32_e32 v1, 0xff, v1
+; GFX12-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0xff, v0
; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-SDAG-FAKE16-NEXT: v_min3_u16 v0, v1, v0, v2
+; GFX12-SDAG-FAKE16-NEXT: v_min3_u16 v0, v0, v1, v2
; GFX12-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GISEL-LABEL: test_vector_reduce_umin_v3i8:
@@ -1290,12 +1283,10 @@ define i16 @test_vector_reduce_umin_v3i16(<3 x i16> %v) {
; GFX7-SDAG-LABEL: test_vector_reduce_umin_v3i16:
; GFX7-SDAG: ; %bb.0: ; %entry
; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-SDAG-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX7-SDAG-NEXT: v_and_b32_e32 v2, 0xffff, v0
-; GFX7-SDAG-NEXT: s_mov_b32 s4, 0xffff
-; GFX7-SDAG-NEXT: v_min_u32_e32 v1, v2, v1
; GFX7-SDAG-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX7-SDAG-NEXT: v_min3_u32 v0, v1, v0, s4
+; GFX7-SDAG-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX7-SDAG-NEXT: v_min3_u32 v0, v2, v0, v1
; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX7-GISEL-LABEL: test_vector_reduce_umin_v3i16:
@@ -1317,51 +1308,37 @@ define i16 @test_vector_reduce_umin_v3i16(<3 x i16> %v) {
; GFX9-LABEL: test_vector_reduce_umin_v3i16:
; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v2, 0x5040100
-; GFX9-NEXT: v_perm_b32 v1, -1, v1, v2
-; GFX9-NEXT: v_pk_min_u16 v0, v0, v1
-; GFX9-NEXT: s_nop 0
-; GFX9-NEXT: v_min_u16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX9-NEXT: v_min3_u16 v0, v0, v2, v1
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: test_vector_reduce_umin_v3i16:
; GFX10: ; %bb.0: ; %entry
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_perm_b32 v1, -1, v1, 0x5040100
-; GFX10-NEXT: v_pk_min_u16 v0, v0, v1
-; GFX10-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX10-NEXT: v_min_u16 v0, v0, v1
+; GFX10-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX10-NEXT: v_min3_u16 v0, v0, v2, v1
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_umin_v3i16:
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.h, -1
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_pk_min_u16 v0, v0, v1
-; GFX11-SDAG-TRUE16-NEXT: v_min_u16 v0.l, v0.l, v0.h
+; GFX11-SDAG-TRUE16-NEXT: v_min3_u16 v0.l, v0.l, v0.h, v1.l
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-FAKE16-LABEL: test_vector_reduce_umin_v3i16:
; GFX11-SDAG-FAKE16: ; %bb.0: ; %entry
; GFX11-SDAG-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-FAKE16-NEXT: v_perm_b32 v1, -1, v1, 0x5040100
-; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-FAKE16-NEXT: v_pk_min_u16 v0, v0, v1
-; GFX11-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX11-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-SDAG-FAKE16-NEXT: v_min_u16 v0, v0, v1
+; GFX11-SDAG-FAKE16-NEXT: v_min3_u16 v0, v0, v2, v1
; GFX11-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-GISEL-LABEL: test_vector_reduce_umin_v3i16:
; GFX11-GISEL: ; %bb.0: ; %entry
; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-NEXT: v_perm_b32 v1, -1, v1, 0x5040100
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-NEXT: v_pk_min_u16 v0, v0, v1
-; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-GISEL-NEXT: v_min_u16 v0, v0, v1
+; GFX11-GISEL-NEXT: v_min3_u16 v0, v0, v2, v1
; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-SDAG-TRUE16-LABEL: test_vector_reduce_umin_v3i16:
@@ -1371,10 +1348,7 @@ define i16 @test_vector_reduce_umin_v3i16(<3 x i16> %v) {
; GFX12-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.h, -1
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_pk_min_u16 v0, v0, v1
-; GFX12-SDAG-TRUE16-NEXT: v_min_u16 v0.l, v0.l, v0.h
+; GFX12-SDAG-TRUE16-NEXT: v_min3_u16 v0.l, v0.l, v0.h, v1.l
; GFX12-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-SDAG-FAKE16-LABEL: test_vector_reduce_umin_v3i16:
@@ -1384,12 +1358,9 @@ define i16 @test_vector_reduce_umin_v3i16(<3 x i16> %v) {
; GFX12-SDAG-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-SDAG-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-FAKE16-NEXT: v_perm_b32 v1, -1, v1, 0x5040100
-; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-FAKE16-NEXT: v_pk_min_u16 v0, v0, v1
-; GFX12-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX12-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-SDAG-FAKE16-NEXT: v_min_u16 v0, v0, v1
+; GFX12-SDAG-FAKE16-NEXT: v_min3_u16 v0, v0, v2, v1
; GFX12-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GISEL-LABEL: test_vector_reduce_umin_v3i16:
@@ -1399,12 +1370,9 @@ define i16 @test_vector_reduce_umin_v3i16(<3 x i16> %v) {
; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
-; GFX12-GISEL-NEXT: v_perm_b32 v1, -1, v1, 0x5040100
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-GISEL-NEXT: v_pk_min_u16 v0, v0, v1
-; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-GISEL-NEXT: v_min_u16 v0, v0, v1
+; GFX12-GISEL-NEXT: v_min3_u16 v0, v0, v2, v1
; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
entry:
%res = call i16 @llvm.vector.reduce.umin.v3i16(<3 x i16> %v)
diff --git a/llvm/test/CodeGen/AMDGPU/vector-reduce-xor.ll b/llvm/test/CodeGen/AMDGPU/vector-reduce-xor.ll
index 9ae6a15399270..19851c50e1cd7 100644
--- a/llvm/test/CodeGen/AMDGPU/vector-reduce-xor.ll
+++ b/llvm/test/CodeGen/AMDGPU/vector-reduce-xor.ll
@@ -161,53 +161,32 @@ entry:
}
define i8 @test_vector_reduce_xor_v3i8(<3 x i8> %v) {
-; GFX7-SDAG-LABEL: test_vector_reduce_xor_v3i8:
-; GFX7-SDAG: ; %bb.0: ; %entry
-; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-SDAG-NEXT: v_xor_b32_e32 v0, v0, v2
-; GFX7-SDAG-NEXT: v_xor_b32_e32 v0, v0, v1
-; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX7-GISEL-LABEL: test_vector_reduce_xor_v3i8:
-; GFX7-GISEL: ; %bb.0: ; %entry
-; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-GISEL-NEXT: v_xor_b32_e32 v0, v0, v1
-; GFX7-GISEL-NEXT: v_xor_b32_e32 v0, v0, v2
-; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX8-SDAG-LABEL: test_vector_reduce_xor_v3i8:
-; GFX8-SDAG: ; %bb.0: ; %entry
-; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-SDAG-NEXT: v_xor_b32_e32 v0, v0, v2
-; GFX8-SDAG-NEXT: v_xor_b32_e32 v0, v0, v1
-; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX8-GISEL-LABEL: test_vector_reduce_xor_v3i8:
-; GFX8-GISEL: ; %bb.0: ; %entry
-; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-GISEL-NEXT: v_xor_b32_e32 v0, v0, v1
-; GFX8-GISEL-NEXT: v_xor_b32_e32 v0, v0, v2
-; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
+; GFX7-LABEL: test_vector_reduce_xor_v3i8:
+; GFX7: ; %bb.0: ; %entry
+; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_xor_b32_e32 v0, v0, v1
+; GFX7-NEXT: v_xor_b32_e32 v0, v0, v2
+; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9-SDAG-LABEL: test_vector_reduce_xor_v3i8:
-; GFX9-SDAG: ; %bb.0: ; %entry
-; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-SDAG-NEXT: v_xor_b32_e32 v0, v0, v2
-; GFX9-SDAG-NEXT: v_xor_b32_e32 v0, v0, v1
-; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
+; GFX8-LABEL: test_vector_reduce_xor_v3i8:
+; GFX8: ; %bb.0: ; %entry
+; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT: v_xor_b32_e32 v0, v0, v1
+; GFX8-NEXT: v_xor_b32_e32 v0, v0, v2
+; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9-GISEL-LABEL: test_vector_reduce_xor_v3i8:
-; GFX9-GISEL: ; %bb.0: ; %entry
-; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-GISEL-NEXT: v_xor_b32_e32 v0, v0, v1
-; GFX9-GISEL-NEXT: v_xor_b32_e32 v0, v0, v2
-; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+; GFX9-LABEL: test_vector_reduce_xor_v3i8:
+; GFX9: ; %bb.0: ; %entry
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_xor_b32_e32 v0, v0, v1
+; GFX9-NEXT: v_xor_b32_e32 v0, v0, v2
+; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-SDAG-LABEL: test_vector_reduce_xor_v3i8:
; GFX10-SDAG: ; %bb.0: ; %entry
; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_xor_b32_e32 v0, v0, v2
; GFX10-SDAG-NEXT: v_xor_b32_e32 v0, v0, v1
+; GFX10-SDAG-NEXT: v_xor_b32_e32 v0, v0, v2
; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-GISEL-LABEL: test_vector_reduce_xor_v3i8:
@@ -219,17 +198,17 @@ define i8 @test_vector_reduce_xor_v3i8(<3 x i8> %v) {
; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_xor_v3i8:
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_xor_b16 v0.l, v0.l, v2.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_xor_b16 v0.l, v0.l, v1.l
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_xor_b16 v0.l, v0.l, v2.l
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-FAKE16-LABEL: test_vector_reduce_xor_v3i8:
; GFX11-SDAG-FAKE16: ; %bb.0: ; %entry
; GFX11-SDAG-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-FAKE16-NEXT: v_xor_b32_e32 v0, v0, v2
-; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-SDAG-FAKE16-NEXT: v_xor_b32_e32 v0, v0, v1
+; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-FAKE16-NEXT: v_xor_b32_e32 v0, v0, v2
; GFX11-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-GISEL-LABEL: test_vector_reduce_xor_v3i8:
@@ -245,9 +224,9 @@ define i8 @test_vector_reduce_xor_v3i8(<3 x i8> %v) {
; GFX12-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: v_xor_b16 v0.l, v0.l, v2.l
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-SDAG-TRUE16-NEXT: v_xor_b16 v0.l, v0.l, v1.l
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-TRUE16-NEXT: v_xor_b16 v0.l, v0.l, v2.l
; GFX12-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-SDAG-FAKE16-LABEL: test_vector_reduce_xor_v3i8:
@@ -257,9 +236,9 @@ define i8 @test_vector_reduce_xor_v3i8(<3 x i8> %v) {
; GFX12-SDAG-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-SDAG-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-FAKE16-NEXT: v_xor_b32_e32 v0, v0, v2
-; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-SDAG-FAKE16-NEXT: v_xor_b32_e32 v0, v0, v1
+; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-FAKE16-NEXT: v_xor_b32_e32 v0, v0, v2
; GFX12-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GISEL-LABEL: test_vector_reduce_xor_v3i8:
@@ -1045,10 +1024,9 @@ define i16 @test_vector_reduce_xor_v3i16(<3 x i16> %v) {
; GFX7-SDAG-LABEL: test_vector_reduce_xor_v3i16:
; GFX7-SDAG: ; %bb.0: ; %entry
; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-SDAG-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX7-SDAG-NEXT: v_lshrrev_b32_e32 v2, 16, v0
-; GFX7-SDAG-NEXT: v_xor_b32_e32 v0, v0, v1
+; GFX7-SDAG-NEXT: v_alignbit_b32 v2, v1, v0, 16
; GFX7-SDAG-NEXT: v_xor_b32_e32 v0, v0, v2
+; GFX7-SDAG-NEXT: v_xor_b32_e32 v0, v0, v1
; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX7-GISEL-LABEL: test_vector_reduce_xor_v3i16:
diff --git a/llvm/test/CodeGen/Generic/expand-experimental-reductions.ll b/llvm/test/CodeGen/Generic/expand-experimental-reductions.ll
index b8e473cbaa4c8..bb4f03ff089af 100644
--- a/llvm/test/CodeGen/Generic/expand-experimental-reductions.ll
+++ b/llvm/test/CodeGen/Generic/expand-experimental-reductions.ll
@@ -299,16 +299,92 @@ entry:
ret double %r
}
-; FIXME: Why is this not expanded?
-
; Test when the vector size is not power of two.
define i8 @test_v3i8(<3 x i8> %a) nounwind {
; CHECK-LABEL: @test_v3i8(
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[B:%.*]] = call i8 @llvm.vector.reduce.and.v3i8(<3 x i8> [[A:%.*]])
-; CHECK-NEXT: ret i8 [[B]]
+; CHECK-NEXT: [[TMP0:%.*]] = extractelement <3 x i8> [[A:%.*]], i64 0
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <3 x i8> [[A]], i64 1
+; CHECK-NEXT: [[BIN_RDX:%.*]] = and i8 [[TMP0]], [[TMP1]]
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <3 x i8> [[A]], i64 2
+; CHECK-NEXT: [[BIN_RDX1:%.*]] = and i8 [[BIN_RDX]], [[TMP2]]
+; CHECK-NEXT: ret i8 [[BIN_RDX1]]
;
entry:
%b = call i8 @llvm.vector.reduce.and.i8.v3i8(<3 x i8> %a)
ret i8 %b
}
+
+define i32 @smax_v3i32(<3 x i32> %vec) {
+; CHECK-LABEL: @smax_v3i32(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP0:%.*]] = extractelement <3 x i32> [[VEC:%.*]], i64 0
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <3 x i32> [[VEC]], i64 1
+; CHECK-NEXT: [[RDX_MINMAX:%.*]] = call i32 @llvm.smax.i32(i32 [[TMP0]], i32 [[TMP1]])
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <3 x i32> [[VEC]], i64 2
+; CHECK-NEXT: [[RDX_MINMAX1:%.*]] = call i32 @llvm.smax.i32(i32 [[RDX_MINMAX]], i32 [[TMP2]])
+; CHECK-NEXT: ret i32 [[RDX_MINMAX1]]
+;
+entry:
+ %r = call i32 @llvm.vector.reduce.smax.v3i32(<3 x i32> %vec)
+ ret i32 %r
+}
+
+define i1 @or_v3i1(<3 x i1> %vec) {
+; CHECK-LABEL: @or_v3i1(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP0:%.*]] = extractelement <3 x i1> [[VEC:%.*]], i64 0
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <3 x i1> [[VEC]], i64 1
+; CHECK-NEXT: [[BIN_RDX:%.*]] = or i1 [[TMP0]], [[TMP1]]
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <3 x i1> [[VEC]], i64 2
+; CHECK-NEXT: [[BIN_RDX1:%.*]] = or i1 [[BIN_RDX]], [[TMP2]]
+; CHECK-NEXT: ret i1 [[BIN_RDX1]]
+;
+entry:
+ %r = call i1 @llvm.vector.reduce.or.v3i1(<3 x i1> %vec)
+ ret i1 %r
+}
+
+define float @fadd_v3f32(float %accum, <3 x float> %vec) {
+; CHECK-LABEL: @fadd_v3f32(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP0:%.*]] = extractelement <3 x float> [[VEC:%.*]], i32 0
+; CHECK-NEXT: [[BIN_RDX:%.*]] = fadd reassoc float [[ACCUM:%.*]], [[TMP0]]
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <3 x float> [[VEC]], i32 1
+; CHECK-NEXT: [[BIN_RDX1:%.*]] = fadd reassoc float [[BIN_RDX]], [[TMP1]]
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <3 x float> [[VEC]], i32 2
+; CHECK-NEXT: [[BIN_RDX2:%.*]] = fadd reassoc float [[BIN_RDX1]], [[TMP2]]
+; CHECK-NEXT: ret float [[BIN_RDX2]]
+;
+entry:
+ %r = call reassoc float @llvm.vector.reduce.fadd.f32.v3f32(float %accum, <3 x float> %vec)
+ ret float %r
+}
+
+define float @fmax_v3f32(<3 x float> %vec) {
+; CHECK-LABEL: @fmax_v3f32(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP0:%.*]] = extractelement <3 x float> [[VEC:%.*]], i64 0
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <3 x float> [[VEC]], i64 1
+; CHECK-NEXT: [[RDX_MINMAX_CMP:%.*]] = fcmp nnan ogt float [[TMP0]], [[TMP1]]
+; CHECK-NEXT: [[RDX_MINMAX_SELECT:%.*]] = select nnan i1 [[RDX_MINMAX_CMP]], float [[TMP0]], float [[TMP1]]
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <3 x float> [[VEC]], i64 2
+; CHECK-NEXT: [[RDX_MINMAX_CMP1:%.*]] = fcmp nnan ogt float [[RDX_MINMAX_SELECT]], [[TMP2]]
+; CHECK-NEXT: [[RDX_MINMAX_SELECT2:%.*]] = select nnan i1 [[RDX_MINMAX_CMP1]], float [[RDX_MINMAX_SELECT]], float [[TMP2]]
+; CHECK-NEXT: ret float [[RDX_MINMAX_SELECT2]]
+;
+entry:
+ %r = call nnan float @llvm.vector.reduce.fmax.v3f32(<3 x float> %vec)
+ ret float %r
+}
+
+define float @fmax_v3f32_maynan(<3 x float> %vec) {
+; CHECK-LABEL: @fmax_v3f32_maynan(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[R:%.*]] = call float @llvm.vector.reduce.fmax.v3f32(<3 x float> [[VEC:%.*]])
+; CHECK-NEXT: ret float [[R]]
+;
+entry:
+ %r = call float @llvm.vector.reduce.fmax.v3f32(<3 x float> %vec)
+ ret float %r
+}
diff --git a/llvm/test/CodeGen/WebAssembly/simd-vecreduce-bool.ll b/llvm/test/CodeGen/WebAssembly/simd-vecreduce-bool.ll
index a6100dd08d7c6..b30c3ca188d5f 100644
--- a/llvm/test/CodeGen/WebAssembly/simd-vecreduce-bool.ll
+++ b/llvm/test/CodeGen/WebAssembly/simd-vecreduce-bool.ll
@@ -265,9 +265,7 @@ define i1 @test_all_v7i1(<7 x i1> %x) {
; CHECK-NEXT: i32.and $push3=, $pop2, $4
; CHECK-NEXT: i32.and $push4=, $pop3, $5
; CHECK-NEXT: i32.and $push5=, $pop4, $6
-; CHECK-NEXT: i32.const $push6=, 1
-; CHECK-NEXT: i32.and $push7=, $pop5, $pop6
-; CHECK-NEXT: return $pop7
+; CHECK-NEXT: return $pop5
%ret = call i1 @llvm.vector.reduce.and.v7i1(<7 x i1> %x)
ret i1 %ret
}
diff --git a/llvm/test/CodeGen/X86/vector-reduce-fmax-nnan.ll b/llvm/test/CodeGen/X86/vector-reduce-fmax-nnan.ll
index 179790c46f33c..eee61b20da9fd 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-fmax-nnan.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-fmax-nnan.ll
@@ -216,25 +216,11 @@ define double @test_v2f64(<2 x double> %a0) {
}
define double @test_v3f64(<3 x double> %a0) {
-; SSE2-LABEL: test_v3f64:
-; SSE2: # %bb.0:
-; SSE2-NEXT: unpcklpd {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; SSE2-NEXT: shufpd {{.*#+}} xmm2 = xmm2[0],mem[1]
-; SSE2-NEXT: maxpd %xmm2, %xmm0
-; SSE2-NEXT: movapd %xmm0, %xmm1
-; SSE2-NEXT: unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm0[1]
-; SSE2-NEXT: maxsd %xmm1, %xmm0
-; SSE2-NEXT: retq
-;
-; SSE41-LABEL: test_v3f64:
-; SSE41: # %bb.0:
-; SSE41-NEXT: unpcklpd {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; SSE41-NEXT: blendpd {{.*#+}} xmm2 = xmm2[0],mem[1]
-; SSE41-NEXT: maxpd %xmm2, %xmm0
-; SSE41-NEXT: movapd %xmm0, %xmm1
-; SSE41-NEXT: unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm0[1]
-; SSE41-NEXT: maxsd %xmm1, %xmm0
-; SSE41-NEXT: retq
+; SSE-LABEL: test_v3f64:
+; SSE: # %bb.0:
+; SSE-NEXT: maxsd %xmm1, %xmm0
+; SSE-NEXT: maxsd %xmm2, %xmm0
+; SSE-NEXT: retq
;
; AVX-LABEL: test_v3f64:
; AVX: # %bb.0:
diff --git a/llvm/test/CodeGen/X86/vector-reduce-or-cmp.ll b/llvm/test/CodeGen/X86/vector-reduce-or-cmp.ll
index acc90d2b4488d..15a349827dfd8 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-or-cmp.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-or-cmp.ll
@@ -2072,35 +2072,82 @@ define zeroext i1 @PR44781(ptr %0) {
}
define i32 @mask_v3i1(<3 x i32> %a, <3 x i32> %b) {
-; SSE-LABEL: mask_v3i1:
-; SSE: # %bb.0:
-; SSE-NEXT: pcmpeqd %xmm1, %xmm0
-; SSE-NEXT: pcmpeqd %xmm1, %xmm1
-; SSE-NEXT: pxor %xmm0, %xmm1
-; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
-; SSE-NEXT: por %xmm1, %xmm0
-; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
-; SSE-NEXT: por %xmm0, %xmm1
-; SSE-NEXT: movd %xmm1, %eax
-; SSE-NEXT: testb $1, %al
-; SSE-NEXT: je .LBB30_2
-; SSE-NEXT: # %bb.1:
-; SSE-NEXT: xorl %eax, %eax
-; SSE-NEXT: ret{{[l|q]}}
-; SSE-NEXT: .LBB30_2:
-; SSE-NEXT: movl $1, %eax
-; SSE-NEXT: ret{{[l|q]}}
+; X86-SSE2-LABEL: mask_v3i1:
+; X86-SSE2: # %bb.0:
+; X86-SSE2-NEXT: pushl %ebp
+; X86-SSE2-NEXT: .cfi_def_cfa_offset 8
+; X86-SSE2-NEXT: .cfi_offset %ebp, -8
+; X86-SSE2-NEXT: movl %esp, %ebp
+; X86-SSE2-NEXT: .cfi_def_cfa_register %ebp
+; X86-SSE2-NEXT: andl $-16, %esp
+; X86-SSE2-NEXT: subl $32, %esp
+; X86-SSE2-NEXT: pcmpeqd %xmm1, %xmm0
+; X86-SSE2-NEXT: pcmpeqd %xmm1, %xmm1
+; X86-SSE2-NEXT: pxor %xmm0, %xmm1
+; X86-SSE2-NEXT: movdqa %xmm1, (%esp)
+; X86-SSE2-NEXT: movzbl (%esp), %eax
+; X86-SSE2-NEXT: orb {{[0-9]+}}(%esp), %al
+; X86-SSE2-NEXT: orb {{[0-9]+}}(%esp), %al
+; X86-SSE2-NEXT: testb $1, %al
+; X86-SSE2-NEXT: je .LBB30_3
+; X86-SSE2-NEXT: # %bb.1:
+; X86-SSE2-NEXT: xorl %eax, %eax
+; X86-SSE2-NEXT: jmp .LBB30_2
+; X86-SSE2-NEXT: .LBB30_3:
+; X86-SSE2-NEXT: movl $1, %eax
+; X86-SSE2-NEXT: .LBB30_2:
+; X86-SSE2-NEXT: movl %ebp, %esp
+; X86-SSE2-NEXT: popl %ebp
+; X86-SSE2-NEXT: .cfi_def_cfa %esp, 4
+; X86-SSE2-NEXT: retl
+;
+; X64-SSE2-LABEL: mask_v3i1:
+; X64-SSE2: # %bb.0:
+; X64-SSE2-NEXT: pcmpeqd %xmm1, %xmm0
+; X64-SSE2-NEXT: pcmpeqd %xmm1, %xmm1
+; X64-SSE2-NEXT: pxor %xmm0, %xmm1
+; X64-SSE2-NEXT: movdqa %xmm1, -{{[0-9]+}}(%rsp)
+; X64-SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax
+; X64-SSE2-NEXT: orb -{{[0-9]+}}(%rsp), %al
+; X64-SSE2-NEXT: orb -{{[0-9]+}}(%rsp), %al
+; X64-SSE2-NEXT: testb $1, %al
+; X64-SSE2-NEXT: je .LBB30_2
+; X64-SSE2-NEXT: # %bb.1:
+; X64-SSE2-NEXT: xorl %eax, %eax
+; X64-SSE2-NEXT: retq
+; X64-SSE2-NEXT: .LBB30_2:
+; X64-SSE2-NEXT: movl $1, %eax
+; X64-SSE2-NEXT: retq
+;
+; SSE4-LABEL: mask_v3i1:
+; SSE4: # %bb.0:
+; SSE4-NEXT: pcmpeqd %xmm1, %xmm0
+; SSE4-NEXT: pcmpeqd %xmm1, %xmm1
+; SSE4-NEXT: pxor %xmm0, %xmm1
+; SSE4-NEXT: movd %xmm1, %eax
+; SSE4-NEXT: pextrb $4, %xmm1, %ecx
+; SSE4-NEXT: orl %eax, %ecx
+; SSE4-NEXT: pextrb $8, %xmm1, %eax
+; SSE4-NEXT: orl %ecx, %eax
+; SSE4-NEXT: testb $1, %al
+; SSE4-NEXT: je .LBB30_2
+; SSE4-NEXT: # %bb.1:
+; SSE4-NEXT: xorl %eax, %eax
+; SSE4-NEXT: ret{{[l|q]}}
+; SSE4-NEXT: .LBB30_2:
+; SSE4-NEXT: movl $1, %eax
+; SSE4-NEXT: ret{{[l|q]}}
;
; AVX1OR2-LABEL: mask_v3i1:
; AVX1OR2: # %bb.0:
; AVX1OR2-NEXT: vpcmpeqd %xmm1, %xmm0, %xmm0
; AVX1OR2-NEXT: vpcmpeqd %xmm1, %xmm1, %xmm1
; AVX1OR2-NEXT: vpxor %xmm1, %xmm0, %xmm0
-; AVX1OR2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX1OR2-NEXT: vpor %xmm1, %xmm0, %xmm1
-; AVX1OR2-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
-; AVX1OR2-NEXT: vpor %xmm0, %xmm1, %xmm0
; AVX1OR2-NEXT: vmovd %xmm0, %eax
+; AVX1OR2-NEXT: vpextrb $4, %xmm0, %ecx
+; AVX1OR2-NEXT: orl %eax, %ecx
+; AVX1OR2-NEXT: vpextrb $8, %xmm0, %eax
+; AVX1OR2-NEXT: orl %ecx, %eax
; AVX1OR2-NEXT: testb $1, %al
; AVX1OR2-NEXT: je .LBB30_2
; AVX1OR2-NEXT: # %bb.1:
@@ -2116,11 +2163,13 @@ define i32 @mask_v3i1(<3 x i32> %a, <3 x i32> %b) {
; AVX512F-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm0
; AVX512F-NEXT: vpcmpneqd %zmm1, %zmm0, %k0
; AVX512F-NEXT: kshiftrw $2, %k0, %k1
-; AVX512F-NEXT: korw %k1, %k0, %k1
-; AVX512F-NEXT: kshiftrw $1, %k0, %k0
-; AVX512F-NEXT: korw %k0, %k1, %k0
-; AVX512F-NEXT: kmovw %k0, %eax
-; AVX512F-NEXT: testb $1, %al
+; AVX512F-NEXT: kmovw %k1, %eax
+; AVX512F-NEXT: kshiftrw $1, %k0, %k1
+; AVX512F-NEXT: kmovw %k1, %ecx
+; AVX512F-NEXT: kmovw %k0, %edx
+; AVX512F-NEXT: orb %cl, %dl
+; AVX512F-NEXT: orb %al, %dl
+; AVX512F-NEXT: testb $1, %dl
; AVX512F-NEXT: je .LBB30_2
; AVX512F-NEXT: # %bb.1:
; AVX512F-NEXT: xorl %eax, %eax
@@ -2137,11 +2186,13 @@ define i32 @mask_v3i1(<3 x i32> %a, <3 x i32> %b) {
; AVX512BW-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm0
; AVX512BW-NEXT: vpcmpneqd %zmm1, %zmm0, %k0
; AVX512BW-NEXT: kshiftrw $2, %k0, %k1
-; AVX512BW-NEXT: korw %k1, %k0, %k1
-; AVX512BW-NEXT: kshiftrw $1, %k0, %k0
-; AVX512BW-NEXT: korw %k0, %k1, %k0
-; AVX512BW-NEXT: kmovd %k0, %eax
-; AVX512BW-NEXT: testb $1, %al
+; AVX512BW-NEXT: kmovd %k1, %eax
+; AVX512BW-NEXT: kshiftrw $1, %k0, %k1
+; AVX512BW-NEXT: kmovd %k1, %ecx
+; AVX512BW-NEXT: kmovd %k0, %edx
+; AVX512BW-NEXT: orb %cl, %dl
+; AVX512BW-NEXT: orb %al, %dl
+; AVX512BW-NEXT: testb $1, %dl
; AVX512BW-NEXT: je .LBB30_2
; AVX512BW-NEXT: # %bb.1:
; AVX512BW-NEXT: xorl %eax, %eax
@@ -2156,11 +2207,13 @@ define i32 @mask_v3i1(<3 x i32> %a, <3 x i32> %b) {
; AVX512VL: # %bb.0:
; AVX512VL-NEXT: vpcmpneqd %xmm1, %xmm0, %k0
; AVX512VL-NEXT: kshiftrb $2, %k0, %k1
-; AVX512VL-NEXT: korw %k1, %k0, %k1
-; AVX512VL-NEXT: kshiftrb $1, %k0, %k0
-; AVX512VL-NEXT: korw %k0, %k1, %k0
-; AVX512VL-NEXT: kmovd %k0, %eax
-; AVX512VL-NEXT: testb $1, %al
+; AVX512VL-NEXT: kmovd %k1, %eax
+; AVX512VL-NEXT: kshiftrb $1, %k0, %k1
+; AVX512VL-NEXT: kmovd %k1, %ecx
+; AVX512VL-NEXT: kmovd %k0, %edx
+; AVX512VL-NEXT: orb %cl, %dl
+; AVX512VL-NEXT: orb %al, %dl
+; AVX512VL-NEXT: testb $1, %dl
; AVX512VL-NEXT: je .LBB30_2
; AVX512VL-NEXT: # %bb.1:
; AVX512VL-NEXT: xorl %eax, %eax
More information about the llvm-commits
mailing list