[llvm] r179930 - SLPVectorizer: Improve the cost model for loop invariant broadcast values.
Nadav Rotem
nrotem at apple.com
Fri Apr 19 23:13:48 PDT 2013
Author: nadav
Date: Sat Apr 20 01:13:47 2013
New Revision: 179930
URL: http://llvm.org/viewvc/llvm-project?rev=179930&view=rev
Log:
SLPVectorizer: Improve the cost model for loop invariant broadcast values.
Added:
llvm/trunk/test/Transforms/SLPVectorizer/X86/loopinvariant.ll
Modified:
llvm/trunk/lib/Transforms/Vectorize/SLPVectorizer.cpp
llvm/trunk/lib/Transforms/Vectorize/VecUtils.cpp
llvm/trunk/lib/Transforms/Vectorize/VecUtils.h
Modified: llvm/trunk/lib/Transforms/Vectorize/SLPVectorizer.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Transforms/Vectorize/SLPVectorizer.cpp?rev=179930&r1=179929&r2=179930&view=diff
==============================================================================
--- llvm/trunk/lib/Transforms/Vectorize/SLPVectorizer.cpp (original)
+++ llvm/trunk/lib/Transforms/Vectorize/SLPVectorizer.cpp Sat Apr 20 01:13:47 2013
@@ -83,7 +83,7 @@ struct SLPVectorizer : public FunctionPa
// Use the bollom up slp vectorizer to construct chains that start with
// he store instructions.
- BoUpSLP R(BB, SE, DL, TTI, AA);
+ BoUpSLP R(BB, SE, DL, TTI, AA, LI->getLoopFor(BB));
// Vectorize trees that end at reductions.
BBChanged |= vectorizeReductions(BB, R);
Modified: llvm/trunk/lib/Transforms/Vectorize/VecUtils.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Transforms/Vectorize/VecUtils.cpp?rev=179930&r1=179929&r2=179930&view=diff
==============================================================================
--- llvm/trunk/lib/Transforms/Vectorize/VecUtils.cpp (original)
+++ llvm/trunk/lib/Transforms/Vectorize/VecUtils.cpp Sat Apr 20 01:13:47 2013
@@ -18,6 +18,7 @@
#include "llvm/Analysis/ScalarEvolutionExpressions.h"
#include "llvm/Analysis/TargetTransformInfo.h"
#include "llvm/Analysis/Verifier.h"
+#include "llvm/Analysis/LoopInfo.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/DataLayout.h"
#include "llvm/IR/Function.h"
@@ -44,8 +45,8 @@ static const unsigned RecursionMaxDepth
namespace llvm {
BoUpSLP::BoUpSLP(BasicBlock *Bb, ScalarEvolution *S, DataLayout *Dl,
- TargetTransformInfo *Tti, AliasAnalysis *Aa) :
- BB(Bb), SE(S), DL(Dl), TTI(Tti), AA(Aa) {
+ TargetTransformInfo *Tti, AliasAnalysis *Aa, Loop *Lp) :
+ BB(Bb), SE(S), DL(Dl), TTI(Tti), AA(Aa), L(Lp) {
numberInstructions();
}
@@ -121,7 +122,7 @@ bool BoUpSLP::vectorizeStoreChain(ValueL
if (Cost < CostThreshold) {
DEBUG(dbgs() << "SLP: Decided to vectorize cost=" << Cost << "\n");
vectorizeTree(Operands, VF);
- i += VF;
+ i += VF - 1;
Changed = true;
}
}
@@ -381,13 +382,15 @@ int BoUpSLP::getTreeCost_rec(ValueList &
// Check if all of the operands are constants.
bool AllConst = true;
bool AllSameScalar = true;
+ bool MustScalarizeFlag = false;
for (unsigned i = 0, e = VL.size(); i < e; ++i) {
AllConst &= isa<Constant>(VL[i]);
AllSameScalar &= (VL[0] == VL[i]);
// Must have a single use.
Instruction *I = dyn_cast<Instruction>(VL[i]);
- // This instruction is outside the basic block or if it is a known hazard.
- if (MustScalarize.count(VL[i]) || (I && I->getParent() != BB))
+ MustScalarizeFlag |= MustScalarize.count(VL[i]);
+ // This instruction is outside the basic block.
+ if (I && I->getParent() != BB)
return getScalarizationCost(VecTy);
}
@@ -395,11 +398,23 @@ int BoUpSLP::getTreeCost_rec(ValueList &
if (AllConst) return 0;
// If all of the operands are identical we can broadcast them.
- if (AllSameScalar)
+ Instruction *VL0 = dyn_cast<Instruction>(VL[0]);
+ if (AllSameScalar) {
+ // If we are in a loop, and this is not an instruction (e.g. constant or
+ // argument) or the instruction is defined outside the loop then assume
+ // that the cost is zero.
+ if (L && (!VL0 || !L->contains(VL0)))
+ return 0;
+
+ // We need to broadcast the scalar.
return TTI->getShuffleCost(TargetTransformInfo::SK_Broadcast, VecTy, 0);
+ }
+
+ // If this is not a constant, or a scalar from outside the loop then we
+ // need to scalarize it.
+ if (MustScalarizeFlag)
+ return getScalarizationCost(VecTy);
- // Scalarize unknown structures.
- Instruction *VL0 = dyn_cast<Instruction>(VL[0]);
if (!VL0) return getScalarizationCost(VecTy);
assert(VL0->getParent() == BB && "Wrong BB");
Modified: llvm/trunk/lib/Transforms/Vectorize/VecUtils.h
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Transforms/Vectorize/VecUtils.h?rev=179930&r1=179929&r2=179930&view=diff
==============================================================================
--- llvm/trunk/lib/Transforms/Vectorize/VecUtils.h (original)
+++ llvm/trunk/lib/Transforms/Vectorize/VecUtils.h Sat Apr 20 01:13:47 2013
@@ -27,6 +27,7 @@ class BasicBlock; class Instruction; cla
class VectorType; class StoreInst; class Value;
class ScalarEvolution; class DataLayout;
class TargetTransformInfo; class AliasAnalysis;
+class Loop;
/// Bottom Up SLP vectorization utility class.
struct BoUpSLP {
@@ -37,7 +38,7 @@ struct BoUpSLP {
// \brief C'tor.
BoUpSLP(BasicBlock *Bb, ScalarEvolution *Se, DataLayout *Dl,
- TargetTransformInfo *Tti, AliasAnalysis *Aa);
+ TargetTransformInfo *Tti, AliasAnalysis *Aa, Loop *Lp);
/// \brief Take the pointer operand from the Load/Store instruction.
/// \returns NULL if this is not a valid Load/Store instruction.
@@ -112,7 +113,7 @@ private:
/// \returns a vector from a collection of scalars in \p VL.
Value *Scalarize(ValueList &VL, VectorType *Ty);
-
+
private:
/// Maps instructions to numbers and back.
SmallDenseMap<Value*, int> InstrIdx;
@@ -155,6 +156,7 @@ private:
DataLayout *DL;
TargetTransformInfo *TTI;
AliasAnalysis *AA;
+ Loop *L;
};
} // end of namespace
Added: llvm/trunk/test/Transforms/SLPVectorizer/X86/loopinvariant.ll
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/test/Transforms/SLPVectorizer/X86/loopinvariant.ll?rev=179930&view=auto
==============================================================================
--- llvm/trunk/test/Transforms/SLPVectorizer/X86/loopinvariant.ll (added)
+++ llvm/trunk/test/Transforms/SLPVectorizer/X86/loopinvariant.ll Sat Apr 20 01:13:47 2013
@@ -0,0 +1,73 @@
+; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.8.0"
+
+;CHECK: @foo
+;CHECK: load <4 x i32>
+;CHECK: add <4 x i32>
+;CHECK: store <4 x i32>
+;CHECK: load <4 x i32>
+;CHECK: add <4 x i32>
+;CHECK: store <4 x i32>
+;CHECK: ret
+define i32 @foo(i32* nocapture %A, i32 %n) #0 {
+entry:
+ %cmp62 = icmp sgt i32 %n, 0
+ br i1 %cmp62, label %for.body, label %for.end
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+ %arrayidx = getelementptr inbounds i32* %A, i64 %indvars.iv
+ %0 = load i32* %arrayidx, align 4, !tbaa !0
+ %add1 = add nsw i32 %0, %n
+ store i32 %add1, i32* %arrayidx, align 4, !tbaa !0
+ %1 = or i64 %indvars.iv, 1
+ %arrayidx4 = getelementptr inbounds i32* %A, i64 %1
+ %2 = load i32* %arrayidx4, align 4, !tbaa !0
+ %add5 = add nsw i32 %2, %n
+ store i32 %add5, i32* %arrayidx4, align 4, !tbaa !0
+ %3 = or i64 %indvars.iv, 2
+ %arrayidx8 = getelementptr inbounds i32* %A, i64 %3
+ %4 = load i32* %arrayidx8, align 4, !tbaa !0
+ %add9 = add nsw i32 %4, %n
+ store i32 %add9, i32* %arrayidx8, align 4, !tbaa !0
+ %5 = or i64 %indvars.iv, 3
+ %arrayidx12 = getelementptr inbounds i32* %A, i64 %5
+ %6 = load i32* %arrayidx12, align 4, !tbaa !0
+ %add13 = add nsw i32 %6, %n
+ store i32 %add13, i32* %arrayidx12, align 4, !tbaa !0
+ %7 = or i64 %indvars.iv, 4
+ %arrayidx16 = getelementptr inbounds i32* %A, i64 %7
+ %8 = load i32* %arrayidx16, align 4, !tbaa !0
+ %add17 = add nsw i32 %8, %n
+ store i32 %add17, i32* %arrayidx16, align 4, !tbaa !0
+ %9 = or i64 %indvars.iv, 5
+ %arrayidx20 = getelementptr inbounds i32* %A, i64 %9
+ %10 = load i32* %arrayidx20, align 4, !tbaa !0
+ %add21 = add nsw i32 %10, %n
+ store i32 %add21, i32* %arrayidx20, align 4, !tbaa !0
+ %11 = or i64 %indvars.iv, 6
+ %arrayidx24 = getelementptr inbounds i32* %A, i64 %11
+ %12 = load i32* %arrayidx24, align 4, !tbaa !0
+ %add25 = add nsw i32 %12, %n
+ store i32 %add25, i32* %arrayidx24, align 4, !tbaa !0
+ %13 = or i64 %indvars.iv, 7
+ %arrayidx28 = getelementptr inbounds i32* %A, i64 %13
+ %14 = load i32* %arrayidx28, align 4, !tbaa !0
+ %add29 = add nsw i32 %14, %n
+ store i32 %add29, i32* %arrayidx28, align 4, !tbaa !0
+ %indvars.iv.next = add i64 %indvars.iv, 8
+ %15 = trunc i64 %indvars.iv.next to i32
+ %cmp = icmp slt i32 %15, %n
+ br i1 %cmp, label %for.body, label %for.end
+
+for.end: ; preds = %for.body, %entry
+ ret i32 undef
+}
+
+attributes #0 = { nounwind ssp uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf"="true" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
+
+!0 = metadata !{metadata !"int", metadata !1}
+!1 = metadata !{metadata !"omnipotent char", metadata !2}
+!2 = metadata !{metadata !"Simple C/C++ TBAA"}
More information about the llvm-commits
mailing list