[llvm] r345037 - [SLPVectorizer] Add basic support for mul/and/or/xor horizontal reductions
Simon Pilgrim via llvm-commits
llvm-commits at lists.llvm.org
Tue Oct 23 08:13:10 PDT 2018
Author: rksimon
Date: Tue Oct 23 08:13:09 2018
New Revision: 345037
URL: http://llvm.org/viewvc/llvm-project?rev=345037&view=rev
Log:
[SLPVectorizer] Add basic support for mul/and/or/xor horizontal reductions
Expand arithmetic reduction to include mul/and/or/xor instructions.
This patch just fixes the SLPVectorizer - the effective reduction costs for AVX1+ are still poor (see rL344846) and will need to be improved before SLP sees this as a valid transform - but we can already see the effect on SSE2 tests.
This partially helps PR37731, but doesn't fix it all as it still falls over on the extraction/reduction order for some reason.
Differential Revision: https://reviews.llvm.org/D53473
Modified:
llvm/trunk/lib/Transforms/Vectorize/SLPVectorizer.cpp
llvm/trunk/test/Transforms/SLPVectorizer/X86/reduction_unrolled.ll
Modified: llvm/trunk/lib/Transforms/Vectorize/SLPVectorizer.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Transforms/Vectorize/SLPVectorizer.cpp?rev=345037&r1=345036&r2=345037&view=diff
==============================================================================
--- llvm/trunk/lib/Transforms/Vectorize/SLPVectorizer.cpp (original)
+++ llvm/trunk/lib/Transforms/Vectorize/SLPVectorizer.cpp Tue Oct 23 08:13:09 2018
@@ -5126,9 +5126,12 @@ class HorizontalReduction {
/// Checks if the reduction operation can be vectorized.
bool isVectorizable() const {
return LHS && RHS &&
- // We currently only support adds && min/max reductions.
+ // We currently only support add/mul/logical && min/max reductions.
((Kind == RK_Arithmetic &&
- (Opcode == Instruction::Add || Opcode == Instruction::FAdd)) ||
+ (Opcode == Instruction::Add || Opcode == Instruction::FAdd ||
+ Opcode == Instruction::Mul || Opcode == Instruction::FMul ||
+ Opcode == Instruction::And || Opcode == Instruction::Or ||
+ Opcode == Instruction::Xor)) ||
((Opcode == Instruction::ICmp || Opcode == Instruction::FCmp) &&
(Kind == RK_Min || Kind == RK_Max)) ||
(Opcode == Instruction::ICmp &&
Modified: llvm/trunk/test/Transforms/SLPVectorizer/X86/reduction_unrolled.ll
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/test/Transforms/SLPVectorizer/X86/reduction_unrolled.ll?rev=345037&r1=345036&r2=345037&view=diff
==============================================================================
--- llvm/trunk/test/Transforms/SLPVectorizer/X86/reduction_unrolled.ll (original)
+++ llvm/trunk/test/Transforms/SLPVectorizer/X86/reduction_unrolled.ll Tue Oct 23 08:13:09 2018
@@ -217,29 +217,30 @@ define i32 @test_and(i32* nocapture read
;
; SSE2-LABEL: @test_and(
; SSE2-NEXT: entry:
-; SSE2-NEXT: [[TMP0:%.*]] = load i32, i32* [[P:%.*]], align 4
-; SSE2-NEXT: [[ARRAYIDX_1:%.*]] = getelementptr inbounds i32, i32* [[P]], i64 1
-; SSE2-NEXT: [[TMP1:%.*]] = load i32, i32* [[ARRAYIDX_1]], align 4
-; SSE2-NEXT: [[MUL_18:%.*]] = and i32 [[TMP1]], [[TMP0]]
+; SSE2-NEXT: [[ARRAYIDX_1:%.*]] = getelementptr inbounds i32, i32* [[P:%.*]], i64 1
; SSE2-NEXT: [[ARRAYIDX_2:%.*]] = getelementptr inbounds i32, i32* [[P]], i64 2
-; SSE2-NEXT: [[TMP2:%.*]] = load i32, i32* [[ARRAYIDX_2]], align 4
-; SSE2-NEXT: [[MUL_29:%.*]] = and i32 [[TMP2]], [[MUL_18]]
; SSE2-NEXT: [[ARRAYIDX_3:%.*]] = getelementptr inbounds i32, i32* [[P]], i64 3
-; SSE2-NEXT: [[TMP3:%.*]] = load i32, i32* [[ARRAYIDX_3]], align 4
-; SSE2-NEXT: [[MUL_310:%.*]] = and i32 [[TMP3]], [[MUL_29]]
; SSE2-NEXT: [[ARRAYIDX_4:%.*]] = getelementptr inbounds i32, i32* [[P]], i64 4
-; SSE2-NEXT: [[TMP4:%.*]] = load i32, i32* [[ARRAYIDX_4]], align 4
-; SSE2-NEXT: [[MUL_411:%.*]] = and i32 [[TMP4]], [[MUL_310]]
; SSE2-NEXT: [[ARRAYIDX_5:%.*]] = getelementptr inbounds i32, i32* [[P]], i64 5
-; SSE2-NEXT: [[TMP5:%.*]] = load i32, i32* [[ARRAYIDX_5]], align 4
-; SSE2-NEXT: [[MUL_512:%.*]] = and i32 [[TMP5]], [[MUL_411]]
; SSE2-NEXT: [[ARRAYIDX_6:%.*]] = getelementptr inbounds i32, i32* [[P]], i64 6
-; SSE2-NEXT: [[TMP6:%.*]] = load i32, i32* [[ARRAYIDX_6]], align 4
-; SSE2-NEXT: [[MUL_613:%.*]] = and i32 [[TMP6]], [[MUL_512]]
; SSE2-NEXT: [[ARRAYIDX_7:%.*]] = getelementptr inbounds i32, i32* [[P]], i64 7
-; SSE2-NEXT: [[TMP7:%.*]] = load i32, i32* [[ARRAYIDX_7]], align 4
-; SSE2-NEXT: [[MUL_714:%.*]] = and i32 [[TMP7]], [[MUL_613]]
-; SSE2-NEXT: ret i32 [[MUL_714]]
+; SSE2-NEXT: [[TMP0:%.*]] = bitcast i32* [[P]] to <8 x i32>*
+; SSE2-NEXT: [[TMP1:%.*]] = load <8 x i32>, <8 x i32>* [[TMP0]], align 4
+; SSE2-NEXT: [[MUL_18:%.*]] = and i32 undef, undef
+; SSE2-NEXT: [[MUL_29:%.*]] = and i32 undef, [[MUL_18]]
+; SSE2-NEXT: [[MUL_310:%.*]] = and i32 undef, [[MUL_29]]
+; SSE2-NEXT: [[MUL_411:%.*]] = and i32 undef, [[MUL_310]]
+; SSE2-NEXT: [[MUL_512:%.*]] = and i32 undef, [[MUL_411]]
+; SSE2-NEXT: [[MUL_613:%.*]] = and i32 undef, [[MUL_512]]
+; SSE2-NEXT: [[RDX_SHUF:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSE2-NEXT: [[BIN_RDX:%.*]] = and <8 x i32> [[TMP1]], [[RDX_SHUF]]
+; SSE2-NEXT: [[RDX_SHUF1:%.*]] = shufflevector <8 x i32> [[BIN_RDX]], <8 x i32> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSE2-NEXT: [[BIN_RDX2:%.*]] = and <8 x i32> [[BIN_RDX]], [[RDX_SHUF1]]
+; SSE2-NEXT: [[RDX_SHUF3:%.*]] = shufflevector <8 x i32> [[BIN_RDX2]], <8 x i32> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSE2-NEXT: [[BIN_RDX4:%.*]] = and <8 x i32> [[BIN_RDX2]], [[RDX_SHUF3]]
+; SSE2-NEXT: [[TMP2:%.*]] = extractelement <8 x i32> [[BIN_RDX4]], i32 0
+; SSE2-NEXT: [[MUL_714:%.*]] = and i32 undef, [[MUL_613]]
+; SSE2-NEXT: ret i32 [[TMP2]]
;
entry:
%0 = load i32, i32* %p, align 4
@@ -303,29 +304,30 @@ define i32 @test_or(i32* nocapture reado
;
; SSE2-LABEL: @test_or(
; SSE2-NEXT: entry:
-; SSE2-NEXT: [[TMP0:%.*]] = load i32, i32* [[P:%.*]], align 4
-; SSE2-NEXT: [[ARRAYIDX_1:%.*]] = getelementptr inbounds i32, i32* [[P]], i64 1
-; SSE2-NEXT: [[TMP1:%.*]] = load i32, i32* [[ARRAYIDX_1]], align 4
-; SSE2-NEXT: [[MUL_18:%.*]] = or i32 [[TMP1]], [[TMP0]]
+; SSE2-NEXT: [[ARRAYIDX_1:%.*]] = getelementptr inbounds i32, i32* [[P:%.*]], i64 1
; SSE2-NEXT: [[ARRAYIDX_2:%.*]] = getelementptr inbounds i32, i32* [[P]], i64 2
-; SSE2-NEXT: [[TMP2:%.*]] = load i32, i32* [[ARRAYIDX_2]], align 4
-; SSE2-NEXT: [[MUL_29:%.*]] = or i32 [[TMP2]], [[MUL_18]]
; SSE2-NEXT: [[ARRAYIDX_3:%.*]] = getelementptr inbounds i32, i32* [[P]], i64 3
-; SSE2-NEXT: [[TMP3:%.*]] = load i32, i32* [[ARRAYIDX_3]], align 4
-; SSE2-NEXT: [[MUL_310:%.*]] = or i32 [[TMP3]], [[MUL_29]]
; SSE2-NEXT: [[ARRAYIDX_4:%.*]] = getelementptr inbounds i32, i32* [[P]], i64 4
-; SSE2-NEXT: [[TMP4:%.*]] = load i32, i32* [[ARRAYIDX_4]], align 4
-; SSE2-NEXT: [[MUL_411:%.*]] = or i32 [[TMP4]], [[MUL_310]]
; SSE2-NEXT: [[ARRAYIDX_5:%.*]] = getelementptr inbounds i32, i32* [[P]], i64 5
-; SSE2-NEXT: [[TMP5:%.*]] = load i32, i32* [[ARRAYIDX_5]], align 4
-; SSE2-NEXT: [[MUL_512:%.*]] = or i32 [[TMP5]], [[MUL_411]]
; SSE2-NEXT: [[ARRAYIDX_6:%.*]] = getelementptr inbounds i32, i32* [[P]], i64 6
-; SSE2-NEXT: [[TMP6:%.*]] = load i32, i32* [[ARRAYIDX_6]], align 4
-; SSE2-NEXT: [[MUL_613:%.*]] = or i32 [[TMP6]], [[MUL_512]]
; SSE2-NEXT: [[ARRAYIDX_7:%.*]] = getelementptr inbounds i32, i32* [[P]], i64 7
-; SSE2-NEXT: [[TMP7:%.*]] = load i32, i32* [[ARRAYIDX_7]], align 4
-; SSE2-NEXT: [[MUL_714:%.*]] = or i32 [[TMP7]], [[MUL_613]]
-; SSE2-NEXT: ret i32 [[MUL_714]]
+; SSE2-NEXT: [[TMP0:%.*]] = bitcast i32* [[P]] to <8 x i32>*
+; SSE2-NEXT: [[TMP1:%.*]] = load <8 x i32>, <8 x i32>* [[TMP0]], align 4
+; SSE2-NEXT: [[MUL_18:%.*]] = or i32 undef, undef
+; SSE2-NEXT: [[MUL_29:%.*]] = or i32 undef, [[MUL_18]]
+; SSE2-NEXT: [[MUL_310:%.*]] = or i32 undef, [[MUL_29]]
+; SSE2-NEXT: [[MUL_411:%.*]] = or i32 undef, [[MUL_310]]
+; SSE2-NEXT: [[MUL_512:%.*]] = or i32 undef, [[MUL_411]]
+; SSE2-NEXT: [[MUL_613:%.*]] = or i32 undef, [[MUL_512]]
+; SSE2-NEXT: [[RDX_SHUF:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSE2-NEXT: [[BIN_RDX:%.*]] = or <8 x i32> [[TMP1]], [[RDX_SHUF]]
+; SSE2-NEXT: [[RDX_SHUF1:%.*]] = shufflevector <8 x i32> [[BIN_RDX]], <8 x i32> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSE2-NEXT: [[BIN_RDX2:%.*]] = or <8 x i32> [[BIN_RDX]], [[RDX_SHUF1]]
+; SSE2-NEXT: [[RDX_SHUF3:%.*]] = shufflevector <8 x i32> [[BIN_RDX2]], <8 x i32> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSE2-NEXT: [[BIN_RDX4:%.*]] = or <8 x i32> [[BIN_RDX2]], [[RDX_SHUF3]]
+; SSE2-NEXT: [[TMP2:%.*]] = extractelement <8 x i32> [[BIN_RDX4]], i32 0
+; SSE2-NEXT: [[MUL_714:%.*]] = or i32 undef, [[MUL_613]]
+; SSE2-NEXT: ret i32 [[TMP2]]
;
entry:
%0 = load i32, i32* %p, align 4
@@ -389,29 +391,30 @@ define i32 @test_xor(i32* nocapture read
;
; SSE2-LABEL: @test_xor(
; SSE2-NEXT: entry:
-; SSE2-NEXT: [[TMP0:%.*]] = load i32, i32* [[P:%.*]], align 4
-; SSE2-NEXT: [[ARRAYIDX_1:%.*]] = getelementptr inbounds i32, i32* [[P]], i64 1
-; SSE2-NEXT: [[TMP1:%.*]] = load i32, i32* [[ARRAYIDX_1]], align 4
-; SSE2-NEXT: [[MUL_18:%.*]] = xor i32 [[TMP1]], [[TMP0]]
+; SSE2-NEXT: [[ARRAYIDX_1:%.*]] = getelementptr inbounds i32, i32* [[P:%.*]], i64 1
; SSE2-NEXT: [[ARRAYIDX_2:%.*]] = getelementptr inbounds i32, i32* [[P]], i64 2
-; SSE2-NEXT: [[TMP2:%.*]] = load i32, i32* [[ARRAYIDX_2]], align 4
-; SSE2-NEXT: [[MUL_29:%.*]] = xor i32 [[TMP2]], [[MUL_18]]
; SSE2-NEXT: [[ARRAYIDX_3:%.*]] = getelementptr inbounds i32, i32* [[P]], i64 3
-; SSE2-NEXT: [[TMP3:%.*]] = load i32, i32* [[ARRAYIDX_3]], align 4
-; SSE2-NEXT: [[MUL_310:%.*]] = xor i32 [[TMP3]], [[MUL_29]]
; SSE2-NEXT: [[ARRAYIDX_4:%.*]] = getelementptr inbounds i32, i32* [[P]], i64 4
-; SSE2-NEXT: [[TMP4:%.*]] = load i32, i32* [[ARRAYIDX_4]], align 4
-; SSE2-NEXT: [[MUL_411:%.*]] = xor i32 [[TMP4]], [[MUL_310]]
; SSE2-NEXT: [[ARRAYIDX_5:%.*]] = getelementptr inbounds i32, i32* [[P]], i64 5
-; SSE2-NEXT: [[TMP5:%.*]] = load i32, i32* [[ARRAYIDX_5]], align 4
-; SSE2-NEXT: [[MUL_512:%.*]] = xor i32 [[TMP5]], [[MUL_411]]
; SSE2-NEXT: [[ARRAYIDX_6:%.*]] = getelementptr inbounds i32, i32* [[P]], i64 6
-; SSE2-NEXT: [[TMP6:%.*]] = load i32, i32* [[ARRAYIDX_6]], align 4
-; SSE2-NEXT: [[MUL_613:%.*]] = xor i32 [[TMP6]], [[MUL_512]]
; SSE2-NEXT: [[ARRAYIDX_7:%.*]] = getelementptr inbounds i32, i32* [[P]], i64 7
-; SSE2-NEXT: [[TMP7:%.*]] = load i32, i32* [[ARRAYIDX_7]], align 4
-; SSE2-NEXT: [[MUL_714:%.*]] = xor i32 [[TMP7]], [[MUL_613]]
-; SSE2-NEXT: ret i32 [[MUL_714]]
+; SSE2-NEXT: [[TMP0:%.*]] = bitcast i32* [[P]] to <8 x i32>*
+; SSE2-NEXT: [[TMP1:%.*]] = load <8 x i32>, <8 x i32>* [[TMP0]], align 4
+; SSE2-NEXT: [[MUL_18:%.*]] = xor i32 undef, undef
+; SSE2-NEXT: [[MUL_29:%.*]] = xor i32 undef, [[MUL_18]]
+; SSE2-NEXT: [[MUL_310:%.*]] = xor i32 undef, [[MUL_29]]
+; SSE2-NEXT: [[MUL_411:%.*]] = xor i32 undef, [[MUL_310]]
+; SSE2-NEXT: [[MUL_512:%.*]] = xor i32 undef, [[MUL_411]]
+; SSE2-NEXT: [[MUL_613:%.*]] = xor i32 undef, [[MUL_512]]
+; SSE2-NEXT: [[RDX_SHUF:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSE2-NEXT: [[BIN_RDX:%.*]] = xor <8 x i32> [[TMP1]], [[RDX_SHUF]]
+; SSE2-NEXT: [[RDX_SHUF1:%.*]] = shufflevector <8 x i32> [[BIN_RDX]], <8 x i32> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSE2-NEXT: [[BIN_RDX2:%.*]] = xor <8 x i32> [[BIN_RDX]], [[RDX_SHUF1]]
+; SSE2-NEXT: [[RDX_SHUF3:%.*]] = shufflevector <8 x i32> [[BIN_RDX2]], <8 x i32> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSE2-NEXT: [[BIN_RDX4:%.*]] = xor <8 x i32> [[BIN_RDX2]], [[RDX_SHUF3]]
+; SSE2-NEXT: [[TMP2:%.*]] = extractelement <8 x i32> [[BIN_RDX4]], i32 0
+; SSE2-NEXT: [[MUL_714:%.*]] = xor i32 undef, [[MUL_613]]
+; SSE2-NEXT: ret i32 [[TMP2]]
;
entry:
%0 = load i32, i32* %p, align 4
More information about the llvm-commits
mailing list