[llvm] [SelectionDAG][x86] Ensure vector reduction optimization (PR #144231)
Suhajda Tamás via llvm-commits
llvm-commits at lists.llvm.org
Sun Feb 15 10:59:59 PST 2026
https://github.com/sutajo updated https://github.com/llvm/llvm-project/pull/144231
>From 5a6b5173136ddb1d8b53658c7ea1bd1601eea7fc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Suhajda=20Tam=C3=A1s?= <sutajo at gmail.com>
Date: Sat, 14 Jun 2025 19:04:49 +0200
Subject: [PATCH 1/3] [x86] Add test for reduction
---
llvm/test/CodeGen/X86/optimize-reduction.ll | 140 ++++++++++++++++++++
1 file changed, 140 insertions(+)
create mode 100644 llvm/test/CodeGen/X86/optimize-reduction.ll
diff --git a/llvm/test/CodeGen/X86/optimize-reduction.ll b/llvm/test/CodeGen/X86/optimize-reduction.ll
new file mode 100644
index 0000000000000..003c41612b8bf
--- /dev/null
+++ b/llvm/test/CodeGen/X86/optimize-reduction.ll
@@ -0,0 +1,140 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-- -mattr=+sse4.1,+fast-hops | FileCheck %s --check-prefixes=SSE41
+; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2,+fast-hops | FileCheck %s --check-prefixes=AVX2
+
+define { i16, i16 } @test_reduce_v16i16_with_umin(<16 x i16> %x, <16 x i16> %y) {
+; SSE41-LABEL: test_reduce_v16i16_with_umin:
+; SSE41: # %bb.0:
+; SSE41-NEXT: movdqa %xmm0, %xmm4
+; SSE41-NEXT: pminuw %xmm1, %xmm4
+; SSE41-NEXT: pshufd {{.*#+}} xmm5 = xmm4[2,3,2,3]
+; SSE41-NEXT: pminuw %xmm4, %xmm5
+; SSE41-NEXT: pshufd {{.*#+}} xmm6 = xmm5[1,1,1,1]
+; SSE41-NEXT: pminuw %xmm5, %xmm6
+; SSE41-NEXT: movdqa %xmm6, %xmm5
+; SSE41-NEXT: psrld $16, %xmm5
+; SSE41-NEXT: pminuw %xmm6, %xmm5
+; SSE41-NEXT: phminposuw %xmm4, %xmm4
+; SSE41-NEXT: movd %xmm4, %eax
+; SSE41-NEXT: pshuflw {{.*#+}} xmm4 = xmm5[0,0,0,0,4,5,6,7]
+; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,1,0,1]
+; SSE41-NEXT: pcmpeqw %xmm4, %xmm1
+; SSE41-NEXT: pcmpeqd %xmm5, %xmm5
+; SSE41-NEXT: pxor %xmm5, %xmm1
+; SSE41-NEXT: por %xmm3, %xmm1
+; SSE41-NEXT: pcmpeqw %xmm4, %xmm0
+; SSE41-NEXT: pxor %xmm5, %xmm0
+; SSE41-NEXT: por %xmm2, %xmm0
+; SSE41-NEXT: pminuw %xmm1, %xmm0
+; SSE41-NEXT: phminposuw %xmm0, %xmm0
+; SSE41-NEXT: movd %xmm0, %edx
+; SSE41-NEXT: # kill: def $ax killed $ax killed $eax
+; SSE41-NEXT: # kill: def $dx killed $dx killed $edx
+; SSE41-NEXT: retq
+;
+; AVX2-LABEL: test_reduce_v16i16_with_umin:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm2
+; AVX2-NEXT: vpminuw %xmm2, %xmm0, %xmm2
+; AVX2-NEXT: vpshufd {{.*#+}} xmm3 = xmm2[2,3,2,3]
+; AVX2-NEXT: vpminuw %xmm3, %xmm2, %xmm3
+; AVX2-NEXT: vpshufd {{.*#+}} xmm4 = xmm3[1,1,1,1]
+; AVX2-NEXT: vpminuw %xmm4, %xmm3, %xmm3
+; AVX2-NEXT: vpsrld $16, %xmm3, %xmm4
+; AVX2-NEXT: vphminposuw %xmm2, %xmm2
+; AVX2-NEXT: vmovd %xmm2, %eax
+; AVX2-NEXT: vpminuw %xmm4, %xmm3, %xmm2
+; AVX2-NEXT: vpbroadcastw %xmm2, %ymm2
+; AVX2-NEXT: vpcmpeqw %ymm2, %ymm0, %ymm0
+; AVX2-NEXT: vpcmpeqd %ymm2, %ymm2, %ymm2
+; AVX2-NEXT: vpxor %ymm2, %ymm0, %ymm0
+; AVX2-NEXT: vpor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT: vpminuw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vphminposuw %xmm0, %xmm0
+; AVX2-NEXT: vmovd %xmm0, %edx
+; AVX2-NEXT: # kill: def $ax killed $ax killed $eax
+; AVX2-NEXT: # kill: def $dx killed $dx killed $edx
+; AVX2-NEXT: vzeroupper
+; AVX2-NEXT: retq
+ %min_x = tail call i16 @llvm.vector.reduce.umin.v16i16(<16 x i16> %x)
+ %min_x_vec = insertelement <1 x i16> poison, i16 %min_x, i64 0
+ %min_x_splat = shufflevector <1 x i16> %min_x_vec, <1 x i16> poison, <16 x i32> zeroinitializer
+ %cmp = icmp eq <16 x i16> %x, %min_x_splat
+ %select = select <16 x i1> %cmp, <16 x i16> %y, <16 x i16> splat (i16 -1)
+ %select_min = tail call i16 @llvm.vector.reduce.umin.v16i16(<16 x i16> %select)
+ %ret_0 = insertvalue { i16, i16 } poison, i16 %min_x, 0
+ %ret = insertvalue { i16, i16 } %ret_0, i16 %select_min, 1
+ ret { i16, i16 } %ret
+}
+
+define { i16, i16 } @test_reduce_v16i16_with_add(<16 x i16> %x, <16 x i16> %y) {
+; SSE41-LABEL: test_reduce_v16i16_with_add:
+; SSE41: # %bb.0: # %start
+; SSE41-NEXT: movdqa %xmm0, %xmm4
+; SSE41-NEXT: paddw %xmm1, %xmm4
+; SSE41-NEXT: pshufd {{.*#+}} xmm5 = xmm4[2,3,2,3]
+; SSE41-NEXT: paddw %xmm4, %xmm5
+; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm5[1,1,1,1]
+; SSE41-NEXT: paddw %xmm5, %xmm4
+; SSE41-NEXT: phaddw %xmm4, %xmm4
+; SSE41-NEXT: movdqa %xmm1, %xmm5
+; SSE41-NEXT: phaddw %xmm0, %xmm5
+; SSE41-NEXT: phaddw %xmm5, %xmm5
+; SSE41-NEXT: phaddw %xmm5, %xmm5
+; SSE41-NEXT: phaddw %xmm5, %xmm5
+; SSE41-NEXT: movd %xmm5, %eax
+; SSE41-NEXT: pshuflw {{.*#+}} xmm4 = xmm4[0,0,0,0,4,5,6,7]
+; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,1,0,1]
+; SSE41-NEXT: pcmpeqw %xmm4, %xmm1
+; SSE41-NEXT: pcmpeqd %xmm5, %xmm5
+; SSE41-NEXT: pxor %xmm5, %xmm1
+; SSE41-NEXT: por %xmm3, %xmm1
+; SSE41-NEXT: pcmpeqw %xmm4, %xmm0
+; SSE41-NEXT: pxor %xmm5, %xmm0
+; SSE41-NEXT: por %xmm2, %xmm0
+; SSE41-NEXT: pminuw %xmm1, %xmm0
+; SSE41-NEXT: phminposuw %xmm0, %xmm0
+; SSE41-NEXT: movd %xmm0, %edx
+; SSE41-NEXT: # kill: def $ax killed $ax killed $eax
+; SSE41-NEXT: # kill: def $dx killed $dx killed $edx
+; SSE41-NEXT: retq
+;
+; AVX2-LABEL: test_reduce_v16i16_with_add:
+; AVX2: # %bb.0: # %start
+; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm2
+; AVX2-NEXT: vpaddw %xmm2, %xmm0, %xmm3
+; AVX2-NEXT: vpshufd {{.*#+}} xmm4 = xmm3[2,3,2,3]
+; AVX2-NEXT: vpaddw %xmm4, %xmm3, %xmm3
+; AVX2-NEXT: vpshufd {{.*#+}} xmm4 = xmm3[1,1,1,1]
+; AVX2-NEXT: vpaddw %xmm4, %xmm3, %xmm3
+; AVX2-NEXT: vphaddw %xmm3, %xmm3, %xmm3
+; AVX2-NEXT: vphaddw %xmm0, %xmm2, %xmm2
+; AVX2-NEXT: vphaddw %xmm2, %xmm2, %xmm2
+; AVX2-NEXT: vphaddw %xmm2, %xmm2, %xmm2
+; AVX2-NEXT: vphaddw %xmm2, %xmm2, %xmm2
+; AVX2-NEXT: vmovd %xmm2, %eax
+; AVX2-NEXT: vpbroadcastw %xmm3, %ymm2
+; AVX2-NEXT: vpcmpeqw %ymm2, %ymm0, %ymm0
+; AVX2-NEXT: vpcmpeqd %ymm2, %ymm2, %ymm2
+; AVX2-NEXT: vpxor %ymm2, %ymm0, %ymm0
+; AVX2-NEXT: vpor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT: vpminuw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vphminposuw %xmm0, %xmm0
+; AVX2-NEXT: vmovd %xmm0, %edx
+; AVX2-NEXT: # kill: def $ax killed $ax killed $eax
+; AVX2-NEXT: # kill: def $dx killed $dx killed $edx
+; AVX2-NEXT: vzeroupper
+; AVX2-NEXT: retq
+start:
+ %sum_x = tail call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> %x)
+ %sum_x_vec = insertelement <1 x i16> poison, i16 %sum_x, i64 0
+ %sum_x_splat = shufflevector <1 x i16> %sum_x_vec, <1 x i16> poison, <16 x i32> zeroinitializer
+ %cmp = icmp eq <16 x i16> %x, %sum_x_splat
+ %select = select <16 x i1> %cmp, <16 x i16> %y, <16 x i16> splat (i16 -1)
+ %select_min = tail call i16 @llvm.vector.reduce.umin.v16i16(<16 x i16> %select)
+ %ret_0 = insertvalue { i16, i16 } poison, i16 %sum_x, 0
+ %ret = insertvalue { i16, i16 } %ret_0, i16 %select_min, 1
+ ret { i16, i16 } %ret
+}
>From 503a7b03c2eb3b9ab5810039c890d75cbe192b83 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Suhajda=20Tam=C3=A1s?= <sutajo at gmail.com>
Date: Sat, 14 Jun 2025 19:09:44 +0200
Subject: [PATCH 2/3] [x86] Implement optimization and update tests
---
llvm/lib/Target/X86/X86ISelLowering.cpp | 55 ++++++++++++++++++---
llvm/test/CodeGen/X86/optimize-reduction.ll | 40 +++------------
2 files changed, 55 insertions(+), 40 deletions(-)
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 88616e0f2a0c9..e6769142fab71 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -47892,7 +47892,8 @@ static SDValue combineArithReduction(SDNode *ExtElt, SelectionDAG &DAG,
/// scalars back, while for x64 we should use 64-bit extracts and shifts.
static SDValue combineExtractVectorElt(SDNode *N, SelectionDAG &DAG,
TargetLowering::DAGCombinerInfo &DCI,
- const X86Subtarget &Subtarget) {
+ const X86Subtarget &Subtarget,
+ bool& TransformedBinOpReduction) {
if (SDValue NewOp = combineExtractWithShuffle(N, DAG, DCI, Subtarget))
return NewOp;
@@ -47980,23 +47981,33 @@ static SDValue combineExtractVectorElt(SDNode *N, SelectionDAG &DAG,
// Check whether this extract is the root of a sum of absolute differences
// pattern. This has to be done here because we really want it to happen
// pre-legalization,
- if (SDValue SAD = combineBasicSADPattern(N, DAG, Subtarget))
+ if (SDValue SAD = combineBasicSADPattern(N, DAG, Subtarget)) {
+ TransformedBinOpReduction = true;
return SAD;
+ }
- if (SDValue VPDPBUSD = combineVPDPBUSDPattern(N, DAG, Subtarget))
+ if (SDValue VPDPBUSD = combineVPDPBUSDPattern(N, DAG, Subtarget)) {
+ TransformedBinOpReduction = true;
return VPDPBUSD;
+ }
// Attempt to replace an all_of/any_of horizontal reduction with a MOVMSK.
- if (SDValue Cmp = combinePredicateReduction(N, DAG, Subtarget))
+ if (SDValue Cmp = combinePredicateReduction(N, DAG, Subtarget)) {
+ TransformedBinOpReduction = true;
return Cmp;
+ }
// Attempt to replace min/max v8i16/v16i8 reductions with PHMINPOSUW.
- if (SDValue MinMax = combineMinMaxReduction(N, DAG, Subtarget))
+ if (SDValue MinMax = combineMinMaxReduction(N, DAG, Subtarget)) {
+ TransformedBinOpReduction = true;
return MinMax;
+ }
// Attempt to optimize ADD/FADD/MUL reductions with HADD, promotion etc..
- if (SDValue V = combineArithReduction(N, DAG, Subtarget))
+ if (SDValue V = combineArithReduction(N, DAG, Subtarget)) {
+ TransformedBinOpReduction = true;
return V;
+ }
if (SDValue V = scalarizeExtEltFP(N, DAG, Subtarget, DCI))
return V;
@@ -48066,6 +48077,36 @@ static SDValue combineExtractVectorElt(SDNode *N, SelectionDAG &DAG,
return SDValue();
}
+static SDValue combineExtractVectorEltAndOperand(SDNode* N, SelectionDAG& DAG,
+ TargetLowering::DAGCombinerInfo& DCI,
+ const X86Subtarget& Subtarget)
+{
+ bool TransformedBinOpReduction = false;
+ auto Op = combineExtractVectorElt(N, DAG, DCI, Subtarget, TransformedBinOpReduction);
+
+ if (TransformedBinOpReduction)
+ {
+ // In case we simplified N = extract_vector_element(V, 0) with Op and V
+ // resulted from a reduction, then we need to replace all uses of V with
+ // scalar_to_vector(Op) to make sure that we eliminated the binop + shuffle
+ // pyramid. This is safe to do, because the elements of V are undefined except
+ // for the zeroth element and Op does not depend on V.
+
+ auto OldV = N->getOperand(0);
+ assert(!Op.getNode()->hasPredecessor(OldV.getNode()) &&
+ "Op must not depend on the converted reduction");
+
+ auto NewV = DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), OldV->getValueType(0), Op);
+
+ auto NV = DCI.CombineTo(N, Op);
+ DCI.CombineTo(OldV.getNode(), NewV);
+
+ Op = NV; // Return N so it doesn't get rechecked!
+ }
+
+ return Op;
+}
+
// Convert (vXiY *ext(vXi1 bitcast(iX))) to extend_in_reg(broadcast(iX)).
// This is more or less the reverse of combineBitcastvxi1.
static SDValue combineToExtendBoolVectorInReg(
@@ -62202,7 +62243,7 @@ SDValue X86TargetLowering::PerformDAGCombine(SDNode *N,
case ISD::EXTRACT_VECTOR_ELT:
case X86ISD::PEXTRW:
case X86ISD::PEXTRB:
- return combineExtractVectorElt(N, DAG, DCI, Subtarget);
+ return combineExtractVectorEltAndOperand(N, DAG, DCI, Subtarget);
case ISD::CONCAT_VECTORS:
return combineCONCAT_VECTORS(N, DAG, DCI, Subtarget);
case ISD::INSERT_SUBVECTOR:
diff --git a/llvm/test/CodeGen/X86/optimize-reduction.ll b/llvm/test/CodeGen/X86/optimize-reduction.ll
index 003c41612b8bf..e51ac1bd3c13c 100644
--- a/llvm/test/CodeGen/X86/optimize-reduction.ll
+++ b/llvm/test/CodeGen/X86/optimize-reduction.ll
@@ -7,16 +7,9 @@ define { i16, i16 } @test_reduce_v16i16_with_umin(<16 x i16> %x, <16 x i16> %y)
; SSE41: # %bb.0:
; SSE41-NEXT: movdqa %xmm0, %xmm4
; SSE41-NEXT: pminuw %xmm1, %xmm4
-; SSE41-NEXT: pshufd {{.*#+}} xmm5 = xmm4[2,3,2,3]
-; SSE41-NEXT: pminuw %xmm4, %xmm5
-; SSE41-NEXT: pshufd {{.*#+}} xmm6 = xmm5[1,1,1,1]
-; SSE41-NEXT: pminuw %xmm5, %xmm6
-; SSE41-NEXT: movdqa %xmm6, %xmm5
-; SSE41-NEXT: psrld $16, %xmm5
-; SSE41-NEXT: pminuw %xmm6, %xmm5
; SSE41-NEXT: phminposuw %xmm4, %xmm4
; SSE41-NEXT: movd %xmm4, %eax
-; SSE41-NEXT: pshuflw {{.*#+}} xmm4 = xmm5[0,0,0,0,4,5,6,7]
+; SSE41-NEXT: pshuflw {{.*#+}} xmm4 = xmm4[0,0,0,0,4,5,6,7]
; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,1,0,1]
; SSE41-NEXT: pcmpeqw %xmm4, %xmm1
; SSE41-NEXT: pcmpeqd %xmm5, %xmm5
@@ -36,14 +29,8 @@ define { i16, i16 } @test_reduce_v16i16_with_umin(<16 x i16> %x, <16 x i16> %y)
; AVX2: # %bb.0:
; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm2
; AVX2-NEXT: vpminuw %xmm2, %xmm0, %xmm2
-; AVX2-NEXT: vpshufd {{.*#+}} xmm3 = xmm2[2,3,2,3]
-; AVX2-NEXT: vpminuw %xmm3, %xmm2, %xmm3
-; AVX2-NEXT: vpshufd {{.*#+}} xmm4 = xmm3[1,1,1,1]
-; AVX2-NEXT: vpminuw %xmm4, %xmm3, %xmm3
-; AVX2-NEXT: vpsrld $16, %xmm3, %xmm4
; AVX2-NEXT: vphminposuw %xmm2, %xmm2
; AVX2-NEXT: vmovd %xmm2, %eax
-; AVX2-NEXT: vpminuw %xmm4, %xmm3, %xmm2
; AVX2-NEXT: vpbroadcastw %xmm2, %ymm2
; AVX2-NEXT: vpcmpeqw %ymm2, %ymm0, %ymm0
; AVX2-NEXT: vpcmpeqd %ymm2, %ymm2, %ymm2
@@ -71,19 +58,12 @@ define { i16, i16 } @test_reduce_v16i16_with_umin(<16 x i16> %x, <16 x i16> %y)
define { i16, i16 } @test_reduce_v16i16_with_add(<16 x i16> %x, <16 x i16> %y) {
; SSE41-LABEL: test_reduce_v16i16_with_add:
; SSE41: # %bb.0: # %start
-; SSE41-NEXT: movdqa %xmm0, %xmm4
-; SSE41-NEXT: paddw %xmm1, %xmm4
-; SSE41-NEXT: pshufd {{.*#+}} xmm5 = xmm4[2,3,2,3]
-; SSE41-NEXT: paddw %xmm4, %xmm5
-; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm5[1,1,1,1]
-; SSE41-NEXT: paddw %xmm5, %xmm4
+; SSE41-NEXT: movdqa %xmm1, %xmm4
+; SSE41-NEXT: phaddw %xmm0, %xmm4
+; SSE41-NEXT: phaddw %xmm4, %xmm4
; SSE41-NEXT: phaddw %xmm4, %xmm4
-; SSE41-NEXT: movdqa %xmm1, %xmm5
-; SSE41-NEXT: phaddw %xmm0, %xmm5
-; SSE41-NEXT: phaddw %xmm5, %xmm5
-; SSE41-NEXT: phaddw %xmm5, %xmm5
-; SSE41-NEXT: phaddw %xmm5, %xmm5
-; SSE41-NEXT: movd %xmm5, %eax
+; SSE41-NEXT: phaddw %xmm4, %xmm4
+; SSE41-NEXT: movd %xmm4, %eax
; SSE41-NEXT: pshuflw {{.*#+}} xmm4 = xmm4[0,0,0,0,4,5,6,7]
; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,1,0,1]
; SSE41-NEXT: pcmpeqw %xmm4, %xmm1
@@ -103,18 +83,12 @@ define { i16, i16 } @test_reduce_v16i16_with_add(<16 x i16> %x, <16 x i16> %y) {
; AVX2-LABEL: test_reduce_v16i16_with_add:
; AVX2: # %bb.0: # %start
; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm2
-; AVX2-NEXT: vpaddw %xmm2, %xmm0, %xmm3
-; AVX2-NEXT: vpshufd {{.*#+}} xmm4 = xmm3[2,3,2,3]
-; AVX2-NEXT: vpaddw %xmm4, %xmm3, %xmm3
-; AVX2-NEXT: vpshufd {{.*#+}} xmm4 = xmm3[1,1,1,1]
-; AVX2-NEXT: vpaddw %xmm4, %xmm3, %xmm3
-; AVX2-NEXT: vphaddw %xmm3, %xmm3, %xmm3
; AVX2-NEXT: vphaddw %xmm0, %xmm2, %xmm2
; AVX2-NEXT: vphaddw %xmm2, %xmm2, %xmm2
; AVX2-NEXT: vphaddw %xmm2, %xmm2, %xmm2
; AVX2-NEXT: vphaddw %xmm2, %xmm2, %xmm2
; AVX2-NEXT: vmovd %xmm2, %eax
-; AVX2-NEXT: vpbroadcastw %xmm3, %ymm2
+; AVX2-NEXT: vpbroadcastw %xmm2, %ymm2
; AVX2-NEXT: vpcmpeqw %ymm2, %ymm0, %ymm0
; AVX2-NEXT: vpcmpeqd %ymm2, %ymm2, %ymm2
; AVX2-NEXT: vpxor %ymm2, %ymm0, %ymm0
>From 5a07a2332a9479ede536e3a9a4d7be96edbcbe9b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tam=C3=A1s=20Suhajda?= <sutajo at gmail.com>
Date: Sun, 15 Feb 2026 18:50:12 +0100
Subject: [PATCH 3/3] Refactor
---
llvm/lib/Target/X86/X86ISelLowering.cpp | 113 ++++++++++++--------
llvm/test/CodeGen/X86/optimize-reduction.ll | 5 +-
2 files changed, 68 insertions(+), 50 deletions(-)
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index e6769142fab71..c1483350cf075 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -46976,27 +46976,31 @@ static SDValue createPSADBW(SelectionDAG &DAG, SDValue N0, SDValue N1,
// Attempt to replace an min/max v8i16/v16i8 horizontal reduction with
// PHMINPOSUW.
-static SDValue combineMinMaxReduction(SDNode *Extract, SelectionDAG &DAG,
+static std::pair<SDValue, bool> combineMinMaxReduction(SDNode *Extract, SelectionDAG &DAG,
const X86Subtarget &Subtarget) {
+ const std::pair<SDValue, bool> NoMatch{};
+
// Bail without SSE41.
if (!Subtarget.hasSSE41())
- return SDValue();
+ return NoMatch;
EVT ExtractVT = Extract->getValueType(0);
if (ExtractVT != MVT::i16 && ExtractVT != MVT::i8)
- return SDValue();
+ return NoMatch;
// Check for SMAX/SMIN/UMAX/UMIN horizontal reduction patterns.
ISD::NodeType BinOp;
SDValue Src = DAG.matchBinOpReduction(
Extract, BinOp, {ISD::SMAX, ISD::SMIN, ISD::UMAX, ISD::UMIN}, true);
if (!Src)
- return SDValue();
+ return NoMatch;
+
+ bool FoundPartialReduction = Src.getOpcode() == ISD::EXTRACT_SUBVECTOR;
EVT SrcVT = Src.getValueType();
EVT SrcSVT = SrcVT.getScalarType();
if (SrcSVT != ExtractVT || (SrcVT.getSizeInBits() % 128) != 0)
- return SDValue();
+ return NoMatch;
SDLoc DL(Extract);
SDValue MinPos = Src;
@@ -47045,8 +47049,8 @@ static SDValue combineMinMaxReduction(SDNode *Extract, SelectionDAG &DAG,
if (Mask)
MinPos = DAG.getNode(ISD::XOR, DL, SrcVT, Mask, MinPos);
- return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ExtractVT, MinPos,
- DAG.getVectorIdxConstant(0, DL));
+ return { DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ExtractVT, MinPos,
+ DAG.getVectorIdxConstant(0, DL)), FoundPartialReduction };
}
// Attempt to replace an all_of/any_of/parity style horizontal reduction with a MOVMSK.
@@ -47260,6 +47264,9 @@ static SDValue combineVPDPBUSDPattern(SDNode *Extract, SelectionDAG &DAG,
Extract->getOperand(1));
}
+// Check whether this extract is the root of a sum of absolute differences
+// pattern. This has to be done here because we really want it to happen
+// pre-legalizatio.
static SDValue combineBasicSADPattern(SDNode *Extract, SelectionDAG &DAG,
const X86Subtarget &Subtarget) {
using namespace SDPatternMatch;
@@ -47697,19 +47704,23 @@ static SDValue scalarizeExtEltFP(SDNode *ExtElt, SelectionDAG &DAG,
/// Try to convert a vector reduction sequence composed of binops and shuffles
/// into horizontal ops.
-static SDValue combineArithReduction(SDNode *ExtElt, SelectionDAG &DAG,
+static std::pair<SDValue, bool> combineArithReduction(SDNode *ExtElt, SelectionDAG &DAG,
const X86Subtarget &Subtarget) {
assert(ExtElt->getOpcode() == ISD::EXTRACT_VECTOR_ELT && "Unexpected caller");
+ const std::pair<SDValue, bool> NoMatch{};
+
// We need at least SSE2 to anything here.
if (!Subtarget.hasSSE2())
- return SDValue();
+ return NoMatch;
ISD::NodeType Opc;
SDValue Rdx = DAG.matchBinOpReduction(ExtElt, Opc,
{ISD::ADD, ISD::MUL, ISD::FADD}, true);
if (!Rdx)
- return SDValue();
+ return NoMatch;
+
+ bool FoundPartialReduction = Rdx.getOpcode() == ISD::EXTRACT_SUBVECTOR;
SDValue Index = ExtElt->getOperand(1);
assert(isNullConstant(Index) &&
@@ -47718,7 +47729,7 @@ static SDValue combineArithReduction(SDNode *ExtElt, SelectionDAG &DAG,
EVT VT = ExtElt->getValueType(0);
EVT VecVT = Rdx.getValueType();
if (VecVT.getScalarType() != VT)
- return SDValue();
+ return NoMatch;
SDLoc DL(ExtElt);
unsigned NumElts = VecVT.getVectorNumElements();
@@ -47745,7 +47756,7 @@ static SDValue combineArithReduction(SDNode *ExtElt, SelectionDAG &DAG,
// vXi8 mul reduction - promote to vXi16 mul reduction.
if (Opc == ISD::MUL) {
if (VT != MVT::i8 || NumElts < 4 || !isPowerOf2_32(NumElts))
- return SDValue();
+ return NoMatch;
if (VecVT.getSizeInBits() >= 128) {
EVT WideVT = EVT::getVectorVT(*DAG.getContext(), MVT::i16, NumElts / 2);
SDValue Lo = getUnpackl(DAG, DL, VecVT, Rdx, DAG.getUNDEF(VecVT));
@@ -47773,7 +47784,10 @@ static SDValue combineArithReduction(SDNode *ExtElt, SelectionDAG &DAG,
DAG.getVectorShuffle(MVT::v8i16, DL, Rdx, Rdx,
{1, -1, -1, -1, -1, -1, -1, -1}));
Rdx = DAG.getBitcast(MVT::v16i8, Rdx);
- return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Rdx, Index);
+ return {
+ DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Rdx, Index),
+ FoundPartialReduction
+ };
}
// vXi8 add reduction - sub 128-bit vector.
@@ -47782,12 +47796,15 @@ static SDValue combineArithReduction(SDNode *ExtElt, SelectionDAG &DAG,
Rdx = DAG.getNode(X86ISD::PSADBW, DL, MVT::v2i64, Rdx,
DAG.getConstant(0, DL, MVT::v16i8));
Rdx = DAG.getBitcast(MVT::v16i8, Rdx);
- return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Rdx, Index);
+ return {
+ DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Rdx, Index),
+ FoundPartialReduction
+ };
}
// Must be a >=128-bit vector with pow2 elements.
if ((VecVT.getSizeInBits() % 128) != 0 || !isPowerOf2_32(NumElts))
- return SDValue();
+ return NoMatch;
// vXi8 add reduction - sum lo/hi halves then use PSADBW.
if (VT == MVT::i8) {
@@ -47806,7 +47823,10 @@ static SDValue combineArithReduction(SDNode *ExtElt, SelectionDAG &DAG,
Rdx = DAG.getNode(X86ISD::PSADBW, DL, MVT::v2i64, Rdx,
getZeroVector(MVT::v16i8, Subtarget, DAG, DL));
Rdx = DAG.getBitcast(MVT::v16i8, Rdx);
- return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Rdx, Index);
+ return {
+ DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Rdx, Index),
+ FoundPartialReduction
+ };
}
// See if we can use vXi8 PSADBW add reduction for larger zext types.
@@ -47853,12 +47873,15 @@ static SDValue combineArithReduction(SDNode *ExtElt, SelectionDAG &DAG,
VecVT = MVT::getVectorVT(VT.getSimpleVT(), 128 / VT.getSizeInBits());
Rdx = DAG.getBitcast(VecVT, Rdx);
- return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Rdx, Index);
+ return {
+ DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Rdx, Index),
+ FoundPartialReduction
+ };
}
// Only use (F)HADD opcodes if they aren't microcoded or minimizes codesize.
if (!shouldUseHorizontalOp(true, DAG, Subtarget))
- return SDValue();
+ return NoMatch;
unsigned HorizOpcode = Opc == ISD::ADD ? X86ISD::HADD : X86ISD::FHADD;
@@ -47876,14 +47899,14 @@ static SDValue combineArithReduction(SDNode *ExtElt, SelectionDAG &DAG,
}
if (!((VecVT == MVT::v8i16 || VecVT == MVT::v4i32) && Subtarget.hasSSSE3()) &&
!((VecVT == MVT::v4f32 || VecVT == MVT::v2f64) && Subtarget.hasSSE3()))
- return SDValue();
+ return NoMatch;
// extract (add (shuf X), X), 0 --> extract (hadd X, X), 0
unsigned ReductionSteps = Log2_32(VecVT.getVectorNumElements());
for (unsigned i = 0; i != ReductionSteps; ++i)
Rdx = DAG.getNode(HorizOpcode, DL, VecVT, Rdx, Rdx);
- return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Rdx, Index);
+ return { DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Rdx, Index), FoundPartialReduction };
}
/// Detect vector gather/scatter index generation and convert it from being a
@@ -47978,35 +48001,31 @@ static SDValue combineExtractVectorElt(SDNode *N, SelectionDAG &DAG,
return DAG.getNode(X86ISD::MMX_MOVD2W, dl, MVT::i32,
InputVector.getOperand(0));
- // Check whether this extract is the root of a sum of absolute differences
- // pattern. This has to be done here because we really want it to happen
- // pre-legalization,
- if (SDValue SAD = combineBasicSADPattern(N, DAG, Subtarget)) {
- TransformedBinOpReduction = true;
- return SAD;
- }
-
- if (SDValue VPDPBUSD = combineVPDPBUSDPattern(N, DAG, Subtarget)) {
- TransformedBinOpReduction = true;
- return VPDPBUSD;
- }
-
- // Attempt to replace an all_of/any_of horizontal reduction with a MOVMSK.
- if (SDValue Cmp = combinePredicateReduction(N, DAG, Subtarget)) {
- TransformedBinOpReduction = true;
- return Cmp;
- }
+ SDValue BinOpReduction;
+ bool IsPartialReduction = false;
+ if (auto Result = std::tie(BinOpReduction, IsPartialReduction);
+ (BinOpReduction = combineBasicSADPattern(N, DAG, Subtarget)) ||
+ (BinOpReduction = combineVPDPBUSDPattern(N, DAG, Subtarget)) ||
+ (BinOpReduction = combinePredicateReduction(N, DAG, Subtarget)) ||
+ ((Result = combineMinMaxReduction(N, DAG, Subtarget)), BinOpReduction) ||
+ ((Result = combineArithReduction(N, DAG, Subtarget)), BinOpReduction)
+ )
+ {
+ SDValue ExtractEltOperand = N->getOperand(0);
+ DCI.CombineTo(N, BinOpReduction);
- // Attempt to replace min/max v8i16/v16i8 reductions with PHMINPOSUW.
- if (SDValue MinMax = combineMinMaxReduction(N, DAG, Subtarget)) {
- TransformedBinOpReduction = true;
- return MinMax;
- }
+ if (!IsPartialReduction)
+ {
+ // Replace also ExtractEltOperand.
+ // This is safe to do, because N resulted directly from a full reduction,
+ // whch means all the elements are undefined except for the 0th element.
+ SDValue V =
+ DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(BinOpReduction),
+ ExtractEltOperand->getValueType(0), BinOpReduction);
+ DCI.CombineTo(ExtractEltOperand.getNode(), V);
+ }
- // Attempt to optimize ADD/FADD/MUL reductions with HADD, promotion etc..
- if (SDValue V = combineArithReduction(N, DAG, Subtarget)) {
- TransformedBinOpReduction = true;
- return V;
+ return SDValue(N,0); // Return N so it doesn't get rechecked!
}
if (SDValue V = scalarizeExtEltFP(N, DAG, Subtarget, DCI))
diff --git a/llvm/test/CodeGen/X86/optimize-reduction.ll b/llvm/test/CodeGen/X86/optimize-reduction.ll
index e51ac1bd3c13c..1cedacf6e9ce1 100644
--- a/llvm/test/CodeGen/X86/optimize-reduction.ll
+++ b/llvm/test/CodeGen/X86/optimize-reduction.ll
@@ -57,7 +57,7 @@ define { i16, i16 } @test_reduce_v16i16_with_umin(<16 x i16> %x, <16 x i16> %y)
define { i16, i16 } @test_reduce_v16i16_with_add(<16 x i16> %x, <16 x i16> %y) {
; SSE41-LABEL: test_reduce_v16i16_with_add:
-; SSE41: # %bb.0: # %start
+; SSE41: # %bb.0:
; SSE41-NEXT: movdqa %xmm1, %xmm4
; SSE41-NEXT: phaddw %xmm0, %xmm4
; SSE41-NEXT: phaddw %xmm4, %xmm4
@@ -81,7 +81,7 @@ define { i16, i16 } @test_reduce_v16i16_with_add(<16 x i16> %x, <16 x i16> %y) {
; SSE41-NEXT: retq
;
; AVX2-LABEL: test_reduce_v16i16_with_add:
-; AVX2: # %bb.0: # %start
+; AVX2: # %bb.0:
; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm2
; AVX2-NEXT: vphaddw %xmm0, %xmm2, %xmm2
; AVX2-NEXT: vphaddw %xmm2, %xmm2, %xmm2
@@ -101,7 +101,6 @@ define { i16, i16 } @test_reduce_v16i16_with_add(<16 x i16> %x, <16 x i16> %y) {
; AVX2-NEXT: # kill: def $dx killed $dx killed $edx
; AVX2-NEXT: vzeroupper
; AVX2-NEXT: retq
-start:
%sum_x = tail call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> %x)
%sum_x_vec = insertelement <1 x i16> poison, i16 %sum_x, i64 0
%sum_x_splat = shufflevector <1 x i16> %sum_x_vec, <1 x i16> poison, <16 x i32> zeroinitializer
More information about the llvm-commits
mailing list