<div dir="ltr">This change broke the modules buildbot:<div><br></div><div> <a href="http://lab.llvm.org:8011/builders/clang-x86_64-linux-selfhost-modules/builds/15321/steps/compile.llvm.stage2/logs/stdio">http://lab.llvm.org:8011/builders/clang-x86_64-linux-selfhost-modules/builds/15321/steps/compile.llvm.stage2/logs/stdio</a></div><div><br></div><div>Looks like you're missing a #include for the type SCEV that you use in LoopUtils.h. Please fix.</div></div><div class="gmail_extra"><br><div class="gmail_quote">On Thu, May 5, 2016 at 8:20 AM, Silviu Baranga via llvm-commits <span dir="ltr"><<a href="mailto:llvm-commits@lists.llvm.org" target="_blank">llvm-commits@lists.llvm.org</a>></span> wrote:<br><blockquote class="gmail_quote" style="margin:0 0 0 .8ex;border-left:1px #ccc solid;padding-left:1ex">Author: sbaranga<br>
Date: Thu May 5 10:20:39 2016<br>
New Revision: 268633<br>
<br>
URL: <a href="http://llvm.org/viewvc/llvm-project?rev=268633&view=rev" rel="noreferrer" target="_blank">http://llvm.org/viewvc/llvm-project?rev=268633&view=rev</a><br>
Log:<br>
[LV] Identify more induction PHIs by coercing expressions to AddRecExprs<br>
<br>
Summary:<br>
Some PHIs can have expressions that are not AddRecExprs due to the presence<br>
of sext/zext instructions. In order to prevent the Loop Vectorizer from<br>
bailing out when encountering these PHIs, we now coerce the SCEV<br>
expressions to AddRecExprs using SCEV predicates (when possible).<br>
<br>
We only do this when the alternative would be to not vectorize.<br>
<br>
Reviewers: mzolotukhin, anemet<br>
<br>
Subscribers: mssimpso, sanjoy, mzolotukhin, llvm-commits<br>
<br>
Differential Revision: <a href="http://reviews.llvm.org/D17153" rel="noreferrer" target="_blank">http://reviews.llvm.org/D17153</a><br>
<br>
Modified:<br>
llvm/trunk/include/llvm/Transforms/Utils/LoopUtils.h<br>
llvm/trunk/lib/Transforms/Utils/LoopUtils.cpp<br>
llvm/trunk/lib/Transforms/Vectorize/LoopVectorize.cpp<br>
llvm/trunk/test/Transforms/LoopVectorize/induction.ll<br>
<br>
Modified: llvm/trunk/include/llvm/Transforms/Utils/LoopUtils.h<br>
URL: <a href="http://llvm.org/viewvc/llvm-project/llvm/trunk/include/llvm/Transforms/Utils/LoopUtils.h?rev=268633&r1=268632&r2=268633&view=diff" rel="noreferrer" target="_blank">http://llvm.org/viewvc/llvm-project/llvm/trunk/include/llvm/Transforms/Utils/LoopUtils.h?rev=268633&r1=268632&r2=268633&view=diff</a><br>
==============================================================================<br>
--- llvm/trunk/include/llvm/Transforms/Utils/LoopUtils.h (original)<br>
+++ llvm/trunk/include/llvm/Transforms/Utils/LoopUtils.h Thu May 5 10:20:39 2016<br>
@@ -30,6 +30,7 @@ class DominatorTree;<br>
class Loop;<br>
class LoopInfo;<br>
class Pass;<br>
+class PredicatedScalarEvolution;<br>
class PredIteratorCache;<br>
class ScalarEvolution;<br>
class TargetLibraryInfo;<br>
@@ -287,8 +288,22 @@ public:<br>
InductionKind getKind() const { return IK; }<br>
ConstantInt *getStepValue() const { return StepValue; }<br>
<br>
+ /// Returns true if \p Phi is an induction. If \p Phi is an induction,<br>
+ /// the induction descriptor \p D will contain the data describing this<br>
+ /// induction. If by some other means the caller has a better SCEV<br>
+ /// expression for \p Phi than the one returned by the ScalarEvolution<br>
+ /// analysis, it can be passed through \p Expr.<br>
static bool isInductionPHI(PHINode *Phi, ScalarEvolution *SE,<br>
- InductionDescriptor &D);<br>
+ InductionDescriptor &D,<br>
+ const SCEV *Expr = nullptr);<br>
+<br>
+ /// Returns true if \p Phi is an induction, in the context associated with<br>
+ /// the run-time predicate of PSE. If \p Assume is true, this can add further<br>
+ /// SCEV predicates to \p PSE in order to prove that \p Phi is an induction.<br>
+ /// If \p Phi is an induction, \p D will contain the data describing this<br>
+ /// induction.<br>
+ static bool isInductionPHI(PHINode *Phi, PredicatedScalarEvolution &PSE,<br>
+ InductionDescriptor &D, bool Assume = false);<br>
<br>
private:<br>
/// Private constructor - used by \c isInductionPHI.<br>
<br>
Modified: llvm/trunk/lib/Transforms/Utils/LoopUtils.cpp<br>
URL: <a href="http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Transforms/Utils/LoopUtils.cpp?rev=268633&r1=268632&r2=268633&view=diff" rel="noreferrer" target="_blank">http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Transforms/Utils/LoopUtils.cpp?rev=268633&r1=268632&r2=268633&view=diff</a><br>
==============================================================================<br>
--- llvm/trunk/lib/Transforms/Utils/LoopUtils.cpp (original)<br>
+++ llvm/trunk/lib/Transforms/Utils/LoopUtils.cpp Thu May 5 10:20:39 2016<br>
@@ -698,16 +698,43 @@ Value *InductionDescriptor::transform(IR<br>
llvm_unreachable("invalid enum");<br>
}<br>
<br>
-bool InductionDescriptor::isInductionPHI(PHINode *Phi, ScalarEvolution *SE,<br>
- InductionDescriptor &D) {<br>
+bool InductionDescriptor::isInductionPHI(PHINode *Phi,<br>
+ PredicatedScalarEvolution &PSE,<br>
+ InductionDescriptor &D,<br>
+ bool Assume) {<br>
+ Type *PhiTy = Phi->getType();<br>
+ // We only handle integer and pointer inductions variables.<br>
+ if (!PhiTy->isIntegerTy() && !PhiTy->isPointerTy())<br>
+ return false;<br>
+<br>
+ const SCEV *PhiScev = PSE.getSCEV(Phi);<br>
+ const auto *AR = dyn_cast<SCEVAddRecExpr>(PhiScev);<br>
+<br>
+ // We need this expression to be an AddRecExpr.<br>
+ if (Assume && !AR)<br>
+ AR = PSE.getAsAddRec(Phi);<br>
+<br>
+ if (!AR) {<br>
+ DEBUG(dbgs() << "LV: PHI is not a poly recurrence.\n");<br>
+ return false;<br>
+ }<br>
+<br>
+ return isInductionPHI(Phi, PSE.getSE(), D, AR);<br>
+}<br>
+<br>
+bool InductionDescriptor::isInductionPHI(PHINode *Phi,<br>
+ ScalarEvolution *SE,<br>
+ InductionDescriptor &D,<br>
+ const SCEV *Expr) {<br>
Type *PhiTy = Phi->getType();<br>
// We only handle integer and pointer inductions variables.<br>
if (!PhiTy->isIntegerTy() && !PhiTy->isPointerTy())<br>
return false;<br>
<br>
// Check that the PHI is consecutive.<br>
- const SCEV *PhiScev = SE->getSCEV(Phi);<br>
+ const SCEV *PhiScev = Expr ? Expr : SE->getSCEV(Phi);<br>
const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(PhiScev);<br>
+<br>
if (!AR) {<br>
DEBUG(dbgs() << "LV: PHI is not a poly recurrence.\n");<br>
return false;<br>
<br>
Modified: llvm/trunk/lib/Transforms/Vectorize/LoopVectorize.cpp<br>
URL: <a href="http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Transforms/Vectorize/LoopVectorize.cpp?rev=268633&r1=268632&r2=268633&view=diff" rel="noreferrer" target="_blank">http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Transforms/Vectorize/LoopVectorize.cpp?rev=268633&r1=268632&r2=268633&view=diff</a><br>
==============================================================================<br>
--- llvm/trunk/lib/Transforms/Vectorize/LoopVectorize.cpp (original)<br>
+++ llvm/trunk/lib/Transforms/Vectorize/LoopVectorize.cpp Thu May 5 10:20:39 2016<br>
@@ -4673,13 +4673,6 @@ bool LoopVectorizationLegality::canVecto<br>
return false;<br>
}<br>
<br>
- InductionDescriptor ID;<br>
- if (InductionDescriptor::isInductionPHI(Phi, PSE.getSE(), ID)) {<br>
- if (!addInductionPhi(Phi, ID))<br>
- return false;<br>
- continue;<br>
- }<br>
-<br>
RecurrenceDescriptor RedDes;<br>
if (RecurrenceDescriptor::isReductionPHI(Phi, TheLoop, RedDes)) {<br>
if (RedDes.hasUnsafeAlgebra())<br>
@@ -4689,11 +4682,26 @@ bool LoopVectorizationLegality::canVecto<br>
continue;<br>
}<br>
<br>
+ InductionDescriptor ID;<br>
+ if (InductionDescriptor::isInductionPHI(Phi, PSE, ID)) {<br>
+ if (!addInductionPhi(Phi, ID))<br>
+ return false;<br>
+ continue;<br>
+ }<br>
+<br>
if (RecurrenceDescriptor::isFirstOrderRecurrence(Phi, TheLoop, DT)) {<br>
FirstOrderRecurrences.insert(Phi);<br>
continue;<br>
}<br>
<br>
+ // As a last resort, coerce the PHI to a AddRec expression<br>
+ // and re-try classifying it a an induction PHI.<br>
+ if (InductionDescriptor::isInductionPHI(Phi, PSE, ID, true)) {<br>
+ if (!addInductionPhi(Phi, ID))<br>
+ return false;<br>
+ continue;<br>
+ }<br>
+<br>
emitAnalysis(VectorizationReport(&*it)<br>
<< "value that could not be identified as "<br>
"reduction is used outside the loop");<br>
<br>
Modified: llvm/trunk/test/Transforms/LoopVectorize/induction.ll<br>
URL: <a href="http://llvm.org/viewvc/llvm-project/llvm/trunk/test/Transforms/LoopVectorize/induction.ll?rev=268633&r1=268632&r2=268633&view=diff" rel="noreferrer" target="_blank">http://llvm.org/viewvc/llvm-project/llvm/trunk/test/Transforms/LoopVectorize/induction.ll?rev=268633&r1=268632&r2=268633&view=diff</a><br>
==============================================================================<br>
--- llvm/trunk/test/Transforms/LoopVectorize/induction.ll (original)<br>
+++ llvm/trunk/test/Transforms/LoopVectorize/induction.ll Thu May 5 10:20:39 2016<br>
@@ -166,3 +166,78 @@ cond.end.i:<br>
loopexit:<br>
ret i32 %and.i<br>
}<br>
+<br>
+; The SCEV expression of %sphi is (zext i8 {%t,+,1}<%loop> to i32)<br>
+; In order to recognize %sphi as an induction PHI and vectorize this loop,<br>
+; we need to convert the SCEV expression into an AddRecExpr.<br>
+; The expression gets converted to {zext i8 %t to i32,+,1}.<br>
+<br>
+; CHECK-LABEL: wrappingindvars1<br>
+; CHECK-LABEL: vector.scevcheck<br>
+; CHECK-LABEL: vector.body<br>
+; CHECK: add <2 x i32> {{%[^ ]*}}, <i32 0, i32 1><br>
+define void @wrappingindvars1(i8 %t, i32 %len, i32 *%A) {<br>
+ entry:<br>
+ %st = zext i8 %t to i16<br>
+ %ext = zext i8 %t to i32<br>
+ %ecmp = icmp ult i16 %st, 42<br>
+ br i1 %ecmp, label %loop, label %exit<br>
+<br>
+ loop:<br>
+<br>
+ %idx = phi i8 [ %t, %entry ], [ %idx.inc, %loop ]<br>
+ %idx.b = phi i32 [ 0, %entry ], [ %idx.b.inc, %loop ]<br>
+ %sphi = phi i32 [ %ext, %entry ], [%idx.inc.ext, %loop]<br>
+<br>
+ %ptr = getelementptr inbounds i32, i32* %A, i8 %idx<br>
+ store i32 %sphi, i32* %ptr<br>
+<br>
+ %idx.inc = add i8 %idx, 1<br>
+ %idx.inc.ext = zext i8 %idx.inc to i32<br>
+ %idx.b.inc = add nuw nsw i32 %idx.b, 1<br>
+<br>
+ %c = icmp ult i32 %idx.b, %len<br>
+ br i1 %c, label %loop, label %exit<br>
+<br>
+ exit:<br>
+ ret void<br>
+}<br>
+<br>
+; The SCEV expression of %sphi is (4 * (zext i8 {%t,+,1}<%loop> to i32))<br>
+; In order to recognize %sphi as an induction PHI and vectorize this loop,<br>
+; we need to convert the SCEV expression into an AddRecExpr.<br>
+; The expression gets converted to ({4 * (zext %t to i32),+,4}).<br>
+; CHECK-LABEL: wrappingindvars2<br>
+; CHECK-LABEL: vector.scevcheck<br>
+; CHECK-LABEL: vector.body<br>
+; CHECK: add <2 x i32> {{%[^ ]*}}, <i32 0, i32 4><br>
+define void @wrappingindvars2(i8 %t, i32 %len, i32 *%A) {<br>
+<br>
+entry:<br>
+ %st = zext i8 %t to i16<br>
+ %ext = zext i8 %t to i32<br>
+ %ext.mul = mul i32 %ext, 4<br>
+<br>
+ %ecmp = icmp ult i16 %st, 42<br>
+ br i1 %ecmp, label %loop, label %exit<br>
+<br>
+ loop:<br>
+<br>
+ %idx = phi i8 [ %t, %entry ], [ %idx.inc, %loop ]<br>
+ %sphi = phi i32 [ %ext.mul, %entry ], [%mul, %loop]<br>
+ %idx.b = phi i32 [ 0, %entry ], [ %idx.b.inc, %loop ]<br>
+<br>
+ %ptr = getelementptr inbounds i32, i32* %A, i8 %idx<br>
+ store i32 %sphi, i32* %ptr<br>
+<br>
+ %idx.inc = add i8 %idx, 1<br>
+ %idx.inc.ext = zext i8 %idx.inc to i32<br>
+ %mul = mul i32 %idx.inc.ext, 4<br>
+ %idx.b.inc = add nuw nsw i32 %idx.b, 1<br>
+<br>
+ %c = icmp ult i32 %idx.b, %len<br>
+ br i1 %c, label %loop, label %exit<br>
+<br>
+ exit:<br>
+ ret void<br>
+}<br>
<br>
<br>
_______________________________________________<br>
llvm-commits mailing list<br>
<a href="mailto:llvm-commits@lists.llvm.org">llvm-commits@lists.llvm.org</a><br>
<a href="http://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-commits" rel="noreferrer" target="_blank">http://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-commits</a><br>
</blockquote></div><br></div>