[llvm] 9d35594 - Reland "[LV] Use lookThroughAnd with logical reductions"
Kerry McLaughlin via llvm-commits
llvm-commits at lists.llvm.org
Fri Jul 30 10:04:37 PDT 2021
Author: Kerry McLaughlin
Date: 2021-07-30T18:04:09+01:00
New Revision: 9d355949937038c32c7608ebb558bbc3984f6340
URL: https://github.com/llvm/llvm-project/commit/9d355949937038c32c7608ebb558bbc3984f6340
DIFF: https://github.com/llvm/llvm-project/commit/9d355949937038c32c7608ebb558bbc3984f6340.diff
LOG: Reland "[LV] Use lookThroughAnd with logical reductions"
If a reduction Phi has a single user which `AND`s the Phi with a type mask,
`lookThroughAnd` will return the user of the Phi and the narrower type represented
by the mask. Currently this is only used for arithmetic reductions, whereas loops
containing logical reductions will create a reduction intrinsic using the widened
type, for example:
for.body:
%phi = phi i32 [ %and, %for.body ], [ 255, %entry ]
%mask = and i32 %phi, 255
%gep = getelementptr inbounds i8, i8* %ptr, i32 %iv
%load = load i8, i8* %gep
%ext = zext i8 %load to i32
%and = and i32 %mask, %ext
...
^ this will generate an and reduction intrinsic such as the following:
call i32 @llvm.vector.reduce.and.v8i32(<8 x i32>...)
The same example for an add instruction would create an intrinsic of type i8:
call i8 @llvm.vector.reduce.add.v8i8(<8 x i8>...)
This patch changes AddReductionVar to call lookThroughAnd for other integer
reductions, allowing loops similar to the example above with reductions such
as and, or & xor to vectorize.
Reviewed By: david-arm, dmgreen
Differential Revision: https://reviews.llvm.org/D105632
Added:
llvm/test/Transforms/LoopVectorize/trunc-reductions.ll
Modified:
llvm/lib/Analysis/IVDescriptors.cpp
llvm/test/Transforms/LoopVectorize/reduction-inloop-pred.ll
llvm/test/Transforms/LoopVectorize/reduction-inloop.ll
Removed:
################################################################################
diff --git a/llvm/lib/Analysis/IVDescriptors.cpp b/llvm/lib/Analysis/IVDescriptors.cpp
index fc6051b35efcd..d8b8f961136e8 100644
--- a/llvm/lib/Analysis/IVDescriptors.cpp
+++ b/llvm/lib/Analysis/IVDescriptors.cpp
@@ -272,7 +272,7 @@ bool RecurrenceDescriptor::AddReductionVar(PHINode *Phi, RecurKind Kind,
} else if (RecurrenceType->isIntegerTy()) {
if (!isIntegerRecurrenceKind(Kind))
return false;
- if (isArithmeticRecurrenceKind(Kind))
+ if (!isMinMaxRecurrenceKind(Kind))
Start = lookThroughAnd(Phi, RecurrenceType, VisitedInsts, CastInsts);
} else {
// Pointer min/max may exist, but it is not supported as a reduction op.
diff --git a/llvm/test/Transforms/LoopVectorize/reduction-inloop-pred.ll b/llvm/test/Transforms/LoopVectorize/reduction-inloop-pred.ll
index 90f3323941c3b..81dee6e2bcd3d 100644
--- a/llvm/test/Transforms/LoopVectorize/reduction-inloop-pred.ll
+++ b/llvm/test/Transforms/LoopVectorize/reduction-inloop-pred.ll
@@ -1541,72 +1541,69 @@ define i8 @reduction_and_trunc(i8* noalias nocapture %A) {
; CHECK: vector.body:
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_LOAD_CONTINUE6:%.*]] ]
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_LOAD_CONTINUE6]] ]
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i32 [ 255, [[VECTOR_PH]] ], [ [[TMP34:%.*]], [[PRED_LOAD_CONTINUE6]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ <i32 255, i32 -1, i32 -1, i32 -1>, [[VECTOR_PH]] ], [ [[TMP29:%.*]], [[PRED_LOAD_CONTINUE6]] ]
; CHECK-NEXT: [[TMP0:%.*]] = icmp ult <4 x i32> [[VEC_IND]], <i32 257, i32 257, i32 257, i32 257>
-; CHECK-NEXT: [[TMP1:%.*]] = select <4 x i1> [[TMP0]], <4 x i32> <i32 255, i32 255, i32 255, i32 255>, <4 x i32> <i32 -1, i32 -1, i32 -1, i32 -1>
-; CHECK-NEXT: [[TMP2:%.*]] = call i32 @llvm.vector.reduce.and.v4i32(<4 x i32> [[TMP1]])
-; CHECK-NEXT: [[TMP3:%.*]] = and i32 [[TMP2]], [[VEC_PHI]]
-; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x i1> [[TMP0]], i32 0
-; CHECK-NEXT: br i1 [[TMP4]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]]
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <4 x i1> [[TMP0]], i32 0
+; CHECK-NEXT: br i1 [[TMP1]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]]
; CHECK: pred.load.if:
-; CHECK-NEXT: [[TMP5:%.*]] = sext i32 [[INDEX]] to i64
-; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i8, i8* [[A:%.*]], i64 [[TMP5]]
-; CHECK-NEXT: [[TMP7:%.*]] = load i8, i8* [[TMP6]], align 4
-; CHECK-NEXT: [[TMP8:%.*]] = insertelement <4 x i8> poison, i8 [[TMP7]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = sext i32 [[INDEX]] to i64
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, i8* [[A:%.*]], i64 [[TMP2]]
+; CHECK-NEXT: [[TMP4:%.*]] = load i8, i8* [[TMP3]], align 4
+; CHECK-NEXT: [[TMP5:%.*]] = insertelement <4 x i8> poison, i8 [[TMP4]], i32 0
; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE]]
; CHECK: pred.load.continue:
-; CHECK-NEXT: [[TMP9:%.*]] = phi <4 x i8> [ poison, [[VECTOR_BODY]] ], [ [[TMP8]], [[PRED_LOAD_IF]] ]
-; CHECK-NEXT: [[TMP10:%.*]] = extractelement <4 x i1> [[TMP0]], i32 1
-; CHECK-NEXT: br i1 [[TMP10]], label [[PRED_LOAD_IF1:%.*]], label [[PRED_LOAD_CONTINUE2:%.*]]
+; CHECK-NEXT: [[TMP6:%.*]] = phi <4 x i8> [ poison, [[VECTOR_BODY]] ], [ [[TMP5]], [[PRED_LOAD_IF]] ]
+; CHECK-NEXT: [[TMP7:%.*]] = extractelement <4 x i1> [[TMP0]], i32 1
+; CHECK-NEXT: br i1 [[TMP7]], label [[PRED_LOAD_IF1:%.*]], label [[PRED_LOAD_CONTINUE2:%.*]]
; CHECK: pred.load.if1:
-; CHECK-NEXT: [[TMP11:%.*]] = or i32 [[INDEX]], 1
-; CHECK-NEXT: [[TMP12:%.*]] = sext i32 [[TMP11]] to i64
-; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds i8, i8* [[A]], i64 [[TMP12]]
-; CHECK-NEXT: [[TMP14:%.*]] = load i8, i8* [[TMP13]], align 4
-; CHECK-NEXT: [[TMP15:%.*]] = insertelement <4 x i8> [[TMP9]], i8 [[TMP14]], i32 1
+; CHECK-NEXT: [[TMP8:%.*]] = or i32 [[INDEX]], 1
+; CHECK-NEXT: [[TMP9:%.*]] = sext i32 [[TMP8]] to i64
+; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i8, i8* [[A]], i64 [[TMP9]]
+; CHECK-NEXT: [[TMP11:%.*]] = load i8, i8* [[TMP10]], align 4
+; CHECK-NEXT: [[TMP12:%.*]] = insertelement <4 x i8> [[TMP6]], i8 [[TMP11]], i32 1
; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE2]]
; CHECK: pred.load.continue2:
-; CHECK-NEXT: [[TMP16:%.*]] = phi <4 x i8> [ [[TMP9]], [[PRED_LOAD_CONTINUE]] ], [ [[TMP15]], [[PRED_LOAD_IF1]] ]
-; CHECK-NEXT: [[TMP17:%.*]] = extractelement <4 x i1> [[TMP0]], i32 2
-; CHECK-NEXT: br i1 [[TMP17]], label [[PRED_LOAD_IF3:%.*]], label [[PRED_LOAD_CONTINUE4:%.*]]
+; CHECK-NEXT: [[TMP13:%.*]] = phi <4 x i8> [ [[TMP6]], [[PRED_LOAD_CONTINUE]] ], [ [[TMP12]], [[PRED_LOAD_IF1]] ]
+; CHECK-NEXT: [[TMP14:%.*]] = extractelement <4 x i1> [[TMP0]], i32 2
+; CHECK-NEXT: br i1 [[TMP14]], label [[PRED_LOAD_IF3:%.*]], label [[PRED_LOAD_CONTINUE4:%.*]]
; CHECK: pred.load.if3:
-; CHECK-NEXT: [[TMP18:%.*]] = or i32 [[INDEX]], 2
-; CHECK-NEXT: [[TMP19:%.*]] = sext i32 [[TMP18]] to i64
-; CHECK-NEXT: [[TMP20:%.*]] = getelementptr inbounds i8, i8* [[A]], i64 [[TMP19]]
-; CHECK-NEXT: [[TMP21:%.*]] = load i8, i8* [[TMP20]], align 4
-; CHECK-NEXT: [[TMP22:%.*]] = insertelement <4 x i8> [[TMP16]], i8 [[TMP21]], i32 2
+; CHECK-NEXT: [[TMP15:%.*]] = or i32 [[INDEX]], 2
+; CHECK-NEXT: [[TMP16:%.*]] = sext i32 [[TMP15]] to i64
+; CHECK-NEXT: [[TMP17:%.*]] = getelementptr inbounds i8, i8* [[A]], i64 [[TMP16]]
+; CHECK-NEXT: [[TMP18:%.*]] = load i8, i8* [[TMP17]], align 4
+; CHECK-NEXT: [[TMP19:%.*]] = insertelement <4 x i8> [[TMP13]], i8 [[TMP18]], i32 2
; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE4]]
; CHECK: pred.load.continue4:
-; CHECK-NEXT: [[TMP23:%.*]] = phi <4 x i8> [ [[TMP16]], [[PRED_LOAD_CONTINUE2]] ], [ [[TMP22]], [[PRED_LOAD_IF3]] ]
-; CHECK-NEXT: [[TMP24:%.*]] = extractelement <4 x i1> [[TMP0]], i32 3
-; CHECK-NEXT: br i1 [[TMP24]], label [[PRED_LOAD_IF5:%.*]], label [[PRED_LOAD_CONTINUE6]]
+; CHECK-NEXT: [[TMP20:%.*]] = phi <4 x i8> [ [[TMP13]], [[PRED_LOAD_CONTINUE2]] ], [ [[TMP19]], [[PRED_LOAD_IF3]] ]
+; CHECK-NEXT: [[TMP21:%.*]] = extractelement <4 x i1> [[TMP0]], i32 3
+; CHECK-NEXT: br i1 [[TMP21]], label [[PRED_LOAD_IF5:%.*]], label [[PRED_LOAD_CONTINUE6]]
; CHECK: pred.load.if5:
-; CHECK-NEXT: [[TMP25:%.*]] = or i32 [[INDEX]], 3
-; CHECK-NEXT: [[TMP26:%.*]] = sext i32 [[TMP25]] to i64
-; CHECK-NEXT: [[TMP27:%.*]] = getelementptr inbounds i8, i8* [[A]], i64 [[TMP26]]
-; CHECK-NEXT: [[TMP28:%.*]] = load i8, i8* [[TMP27]], align 4
-; CHECK-NEXT: [[TMP29:%.*]] = insertelement <4 x i8> [[TMP23]], i8 [[TMP28]], i32 3
+; CHECK-NEXT: [[TMP22:%.*]] = or i32 [[INDEX]], 3
+; CHECK-NEXT: [[TMP23:%.*]] = sext i32 [[TMP22]] to i64
+; CHECK-NEXT: [[TMP24:%.*]] = getelementptr inbounds i8, i8* [[A]], i64 [[TMP23]]
+; CHECK-NEXT: [[TMP25:%.*]] = load i8, i8* [[TMP24]], align 4
+; CHECK-NEXT: [[TMP26:%.*]] = insertelement <4 x i8> [[TMP20]], i8 [[TMP25]], i32 3
; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE6]]
; CHECK: pred.load.continue6:
-; CHECK-NEXT: [[TMP30:%.*]] = phi <4 x i8> [ [[TMP23]], [[PRED_LOAD_CONTINUE4]] ], [ [[TMP29]], [[PRED_LOAD_IF5]] ]
-; CHECK-NEXT: [[TMP31:%.*]] = zext <4 x i8> [[TMP30]] to <4 x i32>
-; CHECK-NEXT: [[TMP32:%.*]] = select <4 x i1> [[TMP0]], <4 x i32> [[TMP31]], <4 x i32> <i32 -1, i32 -1, i32 -1, i32 -1>
-; CHECK-NEXT: [[TMP33:%.*]] = call i32 @llvm.vector.reduce.and.v4i32(<4 x i32> [[TMP32]])
-; CHECK-NEXT: [[TMP34]] = and i32 [[TMP33]], [[TMP3]]
+; CHECK-NEXT: [[TMP27:%.*]] = phi <4 x i8> [ [[TMP20]], [[PRED_LOAD_CONTINUE4]] ], [ [[TMP26]], [[PRED_LOAD_IF5]] ]
+; CHECK-NEXT: [[TMP28:%.*]] = zext <4 x i8> [[TMP27]] to <4 x i32>
+; CHECK-NEXT: [[TMP29]] = and <4 x i32> [[VEC_PHI]], [[TMP28]]
; CHECK-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 4
; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], <i32 4, i32 4, i32 4, i32 4>
-; CHECK-NEXT: [[TMP35:%.*]] = icmp eq i32 [[INDEX_NEXT]], 260
-; CHECK-NEXT: br i1 [[TMP35]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP32:![0-9]+]]
+; CHECK-NEXT: [[TMP30:%.*]] = icmp eq i32 [[INDEX_NEXT]], 260
+; CHECK-NEXT: br i1 [[TMP30]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP32:![0-9]+]]
; CHECK: middle.block:
+; CHECK-NEXT: [[TMP31:%.*]] = select <4 x i1> [[TMP0]], <4 x i32> [[TMP29]], <4 x i32> [[VEC_PHI]]
+; CHECK-NEXT: [[TMP32:%.*]] = trunc <4 x i32> [[TMP31]] to <4 x i8>
+; CHECK-NEXT: [[TMP33:%.*]] = call i8 @llvm.vector.reduce.and.v4i8(<4 x i8> [[TMP32]])
; CHECK-NEXT: br i1 true, label [[DOT_CRIT_EDGE:%.*]], label [[SCALAR_PH]]
; CHECK: scalar.ph:
; CHECK-NEXT: br label [[DOTLR_PH:%.*]]
; CHECK: .lr.ph:
; CHECK-NEXT: br i1 undef, label [[DOT_CRIT_EDGE]], label [[DOTLR_PH]], !llvm.loop [[LOOP33:![0-9]+]]
; CHECK: ._crit_edge:
-; CHECK-NEXT: [[SUM_0_LCSSA:%.*]] = phi i32 [ undef, [[DOTLR_PH]] ], [ [[TMP34]], [[MIDDLE_BLOCK]] ]
-; CHECK-NEXT: [[RET:%.*]] = trunc i32 [[SUM_0_LCSSA]] to i8
-; CHECK-NEXT: ret i8 [[RET]]
+; CHECK-NEXT: [[SUM_0_LCSSA:%.*]] = phi i8 [ undef, [[DOTLR_PH]] ], [ [[TMP33]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: ret i8 [[SUM_0_LCSSA]]
;
entry:
br label %.lr.ph
diff --git a/llvm/test/Transforms/LoopVectorize/reduction-inloop.ll b/llvm/test/Transforms/LoopVectorize/reduction-inloop.ll
index 07a5eb10753e0..15c5f774a7711 100644
--- a/llvm/test/Transforms/LoopVectorize/reduction-inloop.ll
+++ b/llvm/test/Transforms/LoopVectorize/reduction-inloop.ll
@@ -1052,28 +1052,25 @@ define i8 @reduction_and_trunc(i8* noalias nocapture %A) {
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK: vector.body:
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i32 [ 255, [[VECTOR_PH]] ], [ [[TMP6:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = and i32 [[VEC_PHI]], 255
-; CHECK-NEXT: [[TMP1:%.*]] = sext i32 [[INDEX]] to i64
-; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, i8* [[A:%.*]], i64 [[TMP1]]
-; CHECK-NEXT: [[TMP3:%.*]] = bitcast i8* [[TMP2]] to <4 x i8>*
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, <4 x i8>* [[TMP3]], align 4
-; CHECK-NEXT: [[TMP4:%.*]] = zext <4 x i8> [[WIDE_LOAD]] to <4 x i32>
-; CHECK-NEXT: [[TMP5:%.*]] = call i32 @llvm.vector.reduce.and.v4i32(<4 x i32> [[TMP4]])
-; CHECK-NEXT: [[TMP6]] = and i32 [[TMP5]], [[TMP0]]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i8> [ <i8 -1, i8 -1, i8 -1, i8 -1>, [[VECTOR_PH]] ], [ [[TMP3:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = sext i32 [[INDEX]] to i64
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, i8* [[A:%.*]], i64 [[TMP0]]
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast i8* [[TMP1]] to <4 x i8>*
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, <4 x i8>* [[TMP2]], align 4
+; CHECK-NEXT: [[TMP3]] = and <4 x i8> [[VEC_PHI]], [[WIDE_LOAD]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
-; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i32 [[INDEX_NEXT]], 256
-; CHECK-NEXT: br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP38:![0-9]+]]
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i32 [[INDEX_NEXT]], 256
+; CHECK-NEXT: br i1 [[TMP4]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP38:![0-9]+]]
; CHECK: middle.block:
+; CHECK-NEXT: [[TMP5:%.*]] = call i8 @llvm.vector.reduce.and.v4i8(<4 x i8> [[TMP3]])
; CHECK-NEXT: br i1 true, label [[DOT_CRIT_EDGE:%.*]], label [[SCALAR_PH]]
; CHECK: scalar.ph:
; CHECK-NEXT: br label [[DOTLR_PH:%.*]]
; CHECK: .lr.ph:
; CHECK-NEXT: br i1 undef, label [[DOT_CRIT_EDGE]], label [[DOTLR_PH]], !llvm.loop [[LOOP39:![0-9]+]]
; CHECK: ._crit_edge:
-; CHECK-NEXT: [[SUM_0_LCSSA:%.*]] = phi i32 [ undef, [[DOTLR_PH]] ], [ [[TMP6]], [[MIDDLE_BLOCK]] ]
-; CHECK-NEXT: [[RET:%.*]] = trunc i32 [[SUM_0_LCSSA]] to i8
-; CHECK-NEXT: ret i8 [[RET]]
+; CHECK-NEXT: [[SUM_0_LCSSA:%.*]] = phi i8 [ undef, [[DOTLR_PH]] ], [ [[TMP5]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: ret i8 [[SUM_0_LCSSA]]
;
entry:
br label %.lr.ph
diff --git a/llvm/test/Transforms/LoopVectorize/trunc-reductions.ll b/llvm/test/Transforms/LoopVectorize/trunc-reductions.ll
new file mode 100644
index 0000000000000..92cb0ba8b10ea
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/trunc-reductions.ll
@@ -0,0 +1,250 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -loop-vectorize -dce -instcombine -force-vector-interleave=1 -force-vector-width=8 -S < %s | FileCheck %s
+
+define i8 @reduction_and_trunc(i8* noalias nocapture %ptr) {
+; CHECK-LABEL: @reduction_and_trunc(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK: vector.ph:
+; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK: vector.body:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <8 x i8> [ <i8 0, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>, [[VECTOR_PH]] ], [ [[TMP3:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = sext i32 [[INDEX]] to i64
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, i8* [[PTR:%.*]], i64 [[TMP0]]
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast i8* [[TMP1]] to <8 x i8>*
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i8>, <8 x i8>* [[TMP2]], align 1
+; CHECK-NEXT: [[TMP3]] = and <8 x i8> [[VEC_PHI]], [[WIDE_LOAD]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i32 [[INDEX_NEXT]], 256
+; CHECK-NEXT: br i1 [[TMP4]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: middle.block:
+; CHECK-NEXT: [[TMP5:%.*]] = call i8 @llvm.vector.reduce.and.v8i8(<8 x i8> [[TMP3]])
+; CHECK-NEXT: br i1 true, label [[FOR_END:%.*]], label [[SCALAR_PH]]
+; CHECK: scalar.ph:
+; CHECK-NEXT: br label [[FOR_BODY:%.*]]
+; CHECK: for.body:
+; CHECK-NEXT: br i1 undef, label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP2:![0-9]+]]
+; CHECK: for.end:
+; CHECK-NEXT: [[AND_LCSSA_OFF0:%.*]] = phi i8 [ undef, [[FOR_BODY]] ], [ [[TMP5]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: ret i8 [[AND_LCSSA_OFF0]]
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i32 [ %iv.next, %for.body ], [ 0, %entry ]
+ %sum.02p = phi i32 [ %and, %for.body ], [ 0, %entry ]
+ %sum.02 = and i32 %sum.02p, 255
+ %gep = getelementptr inbounds i8, i8* %ptr, i32 %iv
+ %load = load i8, i8* %gep
+ %ext = zext i8 %load to i32
+ %and = and i32 %sum.02, %ext
+ %iv.next = add i32 %iv, 1
+ %exitcond = icmp eq i32 %iv.next, 256
+ br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+ %ret = trunc i32 %and to i8
+ ret i8 %ret
+}
+
+define i16 @reduction_or_trunc(i16* noalias nocapture %ptr) {
+; CHECK-LABEL: @reduction_or_trunc(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK: vector.ph:
+; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK: vector.body:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <8 x i16> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP3:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = sext i32 [[INDEX]] to i64
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i16, i16* [[PTR:%.*]], i64 [[TMP0]]
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast i16* [[TMP1]] to <8 x i16>*
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i16>, <8 x i16>* [[TMP2]], align 2
+; CHECK-NEXT: [[TMP3]] = or <8 x i16> [[VEC_PHI]], [[WIDE_LOAD]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i32 [[INDEX_NEXT]], 256
+; CHECK-NEXT: br i1 [[TMP4]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: middle.block:
+; CHECK-NEXT: [[TMP5:%.*]] = call i16 @llvm.vector.reduce.or.v8i16(<8 x i16> [[TMP3]])
+; CHECK-NEXT: br i1 true, label [[FOR_END:%.*]], label [[SCALAR_PH]]
+; CHECK: scalar.ph:
+; CHECK-NEXT: br label [[FOR_BODY:%.*]]
+; CHECK: for.body:
+; CHECK-NEXT: br i1 undef, label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK: for.end:
+; CHECK-NEXT: [[XOR_LCSSA_OFF0:%.*]] = phi i16 [ undef, [[FOR_BODY]] ], [ [[TMP5]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: ret i16 [[XOR_LCSSA_OFF0]]
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i32 [ %iv.next, %for.body ], [ 0, %entry ]
+ %sum.02p = phi i32 [ %xor, %for.body ], [ 0, %entry ]
+ %sum.02 = and i32 %sum.02p, 65535
+ %gep = getelementptr inbounds i16, i16* %ptr, i32 %iv
+ %load = load i16, i16* %gep
+ %ext = zext i16 %load to i32
+ %xor = or i32 %sum.02, %ext
+ %iv.next = add i32 %iv, 1
+ %exitcond = icmp eq i32 %iv.next, 256
+ br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+ %ret = trunc i32 %xor to i16
+ ret i16 %ret
+}
+
+define i16 @reduction_xor_trunc(i16* noalias nocapture %ptr) {
+; CHECK-LABEL: @reduction_xor_trunc(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK: vector.ph:
+; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK: vector.body:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <8 x i16> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP3:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = sext i32 [[INDEX]] to i64
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i16, i16* [[PTR:%.*]], i64 [[TMP0]]
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast i16* [[TMP1]] to <8 x i16>*
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i16>, <8 x i16>* [[TMP2]], align 2
+; CHECK-NEXT: [[TMP3]] = xor <8 x i16> [[VEC_PHI]], [[WIDE_LOAD]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i32 [[INDEX_NEXT]], 256
+; CHECK-NEXT: br i1 [[TMP4]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK: middle.block:
+; CHECK-NEXT: [[TMP5:%.*]] = call i16 @llvm.vector.reduce.xor.v8i16(<8 x i16> [[TMP3]])
+; CHECK-NEXT: br i1 true, label [[FOR_END:%.*]], label [[SCALAR_PH]]
+; CHECK: scalar.ph:
+; CHECK-NEXT: br label [[FOR_BODY:%.*]]
+; CHECK: for.body:
+; CHECK-NEXT: br i1 undef, label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK: for.end:
+; CHECK-NEXT: [[XOR_LCSSA_OFF0:%.*]] = phi i16 [ undef, [[FOR_BODY]] ], [ [[TMP5]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: ret i16 [[XOR_LCSSA_OFF0]]
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i32 [ %iv.next, %for.body ], [ 0, %entry ]
+ %sum.02p = phi i32 [ %xor, %for.body ], [ 0, %entry ]
+ %sum.02 = and i32 %sum.02p, 65535
+ %gep = getelementptr inbounds i16, i16* %ptr, i32 %iv
+ %load = load i16, i16* %gep
+ %ext = zext i16 %load to i32
+ %xor = xor i32 %sum.02, %ext
+ %iv.next = add i32 %iv, 1
+ %exitcond = icmp eq i32 %iv.next, 256
+ br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+ %ret = trunc i32 %xor to i16
+ ret i16 %ret
+}
+
+define i8 @reduction_smin_trunc(i8* noalias nocapture %ptr) {
+; CHECK-LABEL: @reduction_smin_trunc(
+; CHECK-NOT: vector.body
+; CHECK-NOT: <8 x
+; CHECK: ret
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i32 [ %iv.next, %for.body ], [ 0, %entry ]
+ %sum.02p = phi i32 [ %min, %for.body ], [ 256, %entry ]
+ %sum.02 = and i32 %sum.02p, 255
+ %gep = getelementptr inbounds i8, i8* %ptr, i32 %iv
+ %load = load i8, i8* %gep
+ %ext = sext i8 %load to i32
+ %icmp = icmp slt i32 %sum.02, %ext
+ %min = select i1 %icmp, i32 %sum.02, i32 %ext
+ %iv.next = add i32 %iv, 1
+ %exitcond = icmp eq i32 %iv.next, 256
+ br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+ %ret = trunc i32 %min to i8
+ ret i8 %ret
+}
+
+define i8 @reduction_umin_trunc(i8* noalias nocapture %ptr) {
+; CHECK-LABEL: @reduction_umin_trunc(
+; CHECK-NOT: vector.body
+; CHECK-NOT: <8 x
+; CHECK: ret
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i32 [ %iv.next, %for.body ], [ 0, %entry ]
+ %sum.02p = phi i32 [ %min, %for.body ], [ 0, %entry ]
+ %sum.02 = and i32 %sum.02p, 255
+ %gep = getelementptr inbounds i8, i8* %ptr, i32 %iv
+ %load = load i8, i8* %gep
+ %ext = zext i8 %load to i32
+ %icmp = icmp ult i32 %sum.02, %ext
+ %min = select i1 %icmp, i32 %sum.02, i32 %ext
+ %iv.next = add i32 %iv, 1
+ %exitcond = icmp eq i32 %iv.next, 256
+ br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+ %ret = trunc i32 %min to i8
+ ret i8 %ret
+}
+
+define i16 @reduction_smax_trunc(i16* noalias nocapture %ptr) {
+; CHECK-LABEL: @reduction_smax_trunc(
+; CHECK-NOT: vector.body
+; CHECK-NOT: <8 x
+; CHECK: ret
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i32 [ %iv.next, %for.body ], [ 0, %entry ]
+ %sum.02p = phi i32 [ %min, %for.body ], [ 0, %entry ]
+ %sum.02 = and i32 %sum.02p, 65535
+ %gep = getelementptr inbounds i16, i16* %ptr, i32 %iv
+ %load = load i16, i16* %gep
+ %ext = sext i16 %load to i32
+ %icmp = icmp sgt i32 %sum.02, %ext
+ %min = select i1 %icmp, i32 %sum.02, i32 %ext
+ %iv.next = add i32 %iv, 1
+ %exitcond = icmp eq i32 %iv.next, 256
+ br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+ %ret = trunc i32 %min to i16
+ ret i16 %ret
+}
+
+define i16 @reduction_umax_trunc(i16* noalias nocapture %ptr) {
+; CHECK-LABEL: @reduction_umax_trunc(
+; CHECK-NOT: vector.body
+; CHECK-NOT: <8 x
+; CHECK: ret
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i32 [ %iv.next, %for.body ], [ 0, %entry ]
+ %sum.02p = phi i32 [ %min, %for.body ], [ 0, %entry ]
+ %sum.02 = and i32 %sum.02p, 65535
+ %gep = getelementptr inbounds i16, i16* %ptr, i32 %iv
+ %load = load i16, i16* %gep
+ %ext = zext i16 %load to i32
+ %icmp = icmp ugt i32 %sum.02, %ext
+ %min = select i1 %icmp, i32 %sum.02, i32 %ext
+ %iv.next = add i32 %iv, 1
+ %exitcond = icmp eq i32 %iv.next, 256
+ br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+ %ret = trunc i32 %min to i16
+ ret i16 %ret
+}
More information about the llvm-commits
mailing list