[llvm-commits] [llvm] r173219 - in /llvm/trunk: lib/Transforms/Vectorize/LoopVectorize.cpp test/Transforms/LoopVectorize/bzip_reverse_loops.ll test/Transforms/LoopVectorize/ptr_loops.ll test/Transforms/LoopVectorize/struct_access.ll
Nadav Rotem
nrotem at apple.com
Tue Jan 22 17:35:00 PST 2013
Author: nadav
Date: Tue Jan 22 19:35:00 2013
New Revision: 173219
URL: http://llvm.org/viewvc/llvm-project?rev=173219&view=rev
Log:
Add support for reverse pointer induction variables. These are loops that contain pointers that count backwards.
For example, this is the hot loop in BZIP:
do {
m = *--p;
*p = ( ... );
} while (--n);
Added:
llvm/trunk/test/Transforms/LoopVectorize/bzip_reverse_loops.ll
llvm/trunk/test/Transforms/LoopVectorize/ptr_loops.ll
llvm/trunk/test/Transforms/LoopVectorize/struct_access.ll
Modified:
llvm/trunk/lib/Transforms/Vectorize/LoopVectorize.cpp
Modified: llvm/trunk/lib/Transforms/Vectorize/LoopVectorize.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Transforms/Vectorize/LoopVectorize.cpp?rev=173219&r1=173218&r2=173219&view=diff
==============================================================================
--- llvm/trunk/lib/Transforms/Vectorize/LoopVectorize.cpp (original)
+++ llvm/trunk/lib/Transforms/Vectorize/LoopVectorize.cpp Tue Jan 22 19:35:00 2013
@@ -327,7 +327,8 @@
IK_NoInduction, ///< Not an induction variable.
IK_IntInduction, ///< Integer induction variable. Step = 1.
IK_ReverseIntInduction, ///< Reverse int induction variable. Step = -1.
- IK_PtrInduction ///< Pointer induction variable. Step = sizeof(elem).
+ IK_PtrInduction, ///< Pointer induction var. Step = sizeof(elem).
+ IK_ReversePtrInduction ///< Reverse ptr indvar. Step = - sizeof(elem).
};
/// This POD struct holds information about reduction variables.
@@ -734,6 +735,9 @@
int LoopVectorizationLegality::isConsecutivePtr(Value *Ptr) {
assert(Ptr->getType()->isPointerTy() && "Unexpected non ptr");
+ // Make sure that the pointer does not point to structs.
+ if (cast<PointerType>(Ptr->getType())->getElementType()->isAggregateType())
+ return 0;
// If this value is a pointer induction variable we know it is consecutive.
PHINode *Phi = dyn_cast_or_null<PHINode>(Ptr);
@@ -741,6 +745,8 @@
InductionInfo II = Inductions[Phi];
if (IK_PtrInduction == II.IK)
return 1;
+ else if (IK_ReversePtrInduction == II.IK)
+ return -1;
}
GetElementPtrInst *Gep = dyn_cast_or_null<GetElementPtrInst>(Ptr);
@@ -750,6 +756,29 @@
unsigned NumOperands = Gep->getNumOperands();
Value *LastIndex = Gep->getOperand(NumOperands - 1);
+ Value *GpPtr = Gep->getPointerOperand();
+ // If this GEP value is a consecutive pointer induction variable and all of
+ // the indices are constant then we know it is consecutive. We can
+ Phi = dyn_cast<PHINode>(GpPtr);
+ if (Phi && Inductions.count(Phi)) {
+
+ // Make sure that the pointer does not point to structs.
+ PointerType *GepPtrType = cast<PointerType>(GpPtr->getType());
+ if (GepPtrType->getElementType()->isAggregateType())
+ return 0;
+
+ // Make sure that all of the index operands are loop invariant.
+ for (unsigned i = 1; i < NumOperands; ++i)
+ if (!SE->isLoopInvariant(SE->getSCEV(Gep->getOperand(i)), TheLoop))
+ return 0;
+
+ InductionInfo II = Inductions[Phi];
+ if (IK_PtrInduction == II.IK)
+ return 1;
+ else if (IK_ReversePtrInduction == II.IK)
+ return -1;
+ }
+
// Check that all of the gep indices are uniform except for the last.
for (unsigned i = 0; i < NumOperands - 1; ++i)
if (!SE->isLoopInvariant(SE->getSCEV(Gep->getOperand(i)), TheLoop))
@@ -1148,6 +1177,18 @@
LoopBypassBlocks.back()->getTerminator());
break;
}
+ case LoopVectorizationLegality::IK_ReversePtrInduction: {
+ // The value at the end of the loop for the reverse pointer is calculated
+ // by creating a GEP with a negative index starting from the start value.
+ Value *Zero = ConstantInt::get(CountRoundDown->getType(), 0);
+ Value *NegIdx = BinaryOperator::CreateSub(Zero, CountRoundDown,
+ "rev.ind.end",
+ LoopBypassBlocks.back()->getTerminator());
+ EndValue = GetElementPtrInst::Create(II.StartValue, NegIdx,
+ "rev.ptr.ind.end",
+ LoopBypassBlocks.back()->getTerminator());
+ break;
+ }
}// end of case
// The new PHI merges the original incoming value, in case of a bypass,
@@ -1625,6 +1666,7 @@
}
case LoopVectorizationLegality::IK_ReverseIntInduction:
case LoopVectorizationLegality::IK_PtrInduction:
+ case LoopVectorizationLegality::IK_ReversePtrInduction:
// Handle reverse integer and pointer inductions.
Value *StartIdx = 0;
// If we have a single integer induction variable then use it.
@@ -1660,15 +1702,23 @@
// Handle the pointer induction variable case.
assert(P->getType()->isPointerTy() && "Unexpected type.");
+ // Is this a reverse induction ptr or a consecutive induction ptr.
+ bool Reverse = (LoopVectorizationLegality::IK_ReversePtrInduction ==
+ II.IK);
+
// This is the vector of results. Notice that we don't generate
// vector geps because scalar geps result in better code.
for (unsigned part = 0; part < UF; ++part) {
Value *VecVal = UndefValue::get(VectorType::get(P->getType(), VF));
for (unsigned int i = 0; i < VF; ++i) {
- Constant *Idx = ConstantInt::get(Induction->getType(),
- i + part * VF);
- Value *GlobalIdx = Builder.CreateAdd(NormalizedIdx, Idx,
- "gep.idx");
+ int EltIndex = (i + part * VF) * (Reverse ? -1 : 1);
+ Constant *Idx = ConstantInt::get(Induction->getType(), EltIndex);
+ Value *GlobalIdx;
+ if (!Reverse)
+ GlobalIdx = Builder.CreateAdd(NormalizedIdx, Idx, "gep.idx");
+ else
+ GlobalIdx = Builder.CreateSub(Idx, NormalizedIdx, "gep.ridx");
+
Value *SclrGep = Builder.CreateGEP(II.StartValue, GlobalIdx,
"next.gep");
VecVal = Builder.CreateInsertElement(VecVal, SclrGep,
@@ -1786,7 +1836,19 @@
// Handle consecutive stores.
GetElementPtrInst *Gep = dyn_cast<GetElementPtrInst>(Ptr);
- if (Gep) {
+ if (Gep && Legal->isInductionVariable(Gep->getPointerOperand())) {
+ Value *PtrOperand = Gep->getPointerOperand();
+ Value *FirstBasePtr = getVectorValue(PtrOperand)[0];
+ FirstBasePtr = Builder.CreateExtractElement(FirstBasePtr, Zero);
+
+ // Create the new GEP with the new induction variable.
+ GetElementPtrInst *Gep2 = cast<GetElementPtrInst>(Gep->clone());
+ Gep2->setOperand(0, FirstBasePtr);
+ Ptr = Builder.Insert(Gep2);
+ } else if (Gep) {
+ assert(SE->isLoopInvariant(SE->getSCEV(Gep->getPointerOperand()),
+ OrigLoop) && "Base ptr must be invariant");
+
// The last index does not have to be the induction. It can be
// consecutive and be a function of the index. For example A[I+1];
unsigned NumOperands = Gep->getNumOperands();
@@ -1844,7 +1906,18 @@
}
GetElementPtrInst *Gep = dyn_cast<GetElementPtrInst>(Ptr);
- if (Gep) {
+ if (Gep && Legal->isInductionVariable(Gep->getPointerOperand())) {
+ Value *PtrOperand = Gep->getPointerOperand();
+ Value *FirstBasePtr = getVectorValue(PtrOperand)[0];
+ FirstBasePtr = Builder.CreateExtractElement(FirstBasePtr, Zero);
+ // Create the new GEP with the new induction variable.
+ GetElementPtrInst *Gep2 = cast<GetElementPtrInst>(Gep->clone());
+ Gep2->setOperand(0, FirstBasePtr);
+ Ptr = Builder.Insert(Gep2);
+ } else if (Gep) {
+ assert(SE->isLoopInvariant(SE->getSCEV(Gep->getPointerOperand()),
+ OrigLoop) && "Base ptr must be invariant");
+
// The last index does not have to be the induction. It can be
// consecutive and be a function of the index. For example A[I+1];
unsigned NumOperands = Gep->getNumOperands();
@@ -2589,6 +2662,8 @@
uint64_t Size = DL->getTypeAllocSize(PhiTy->getPointerElementType());
if (C->getValue()->equalsInt(Size))
return IK_PtrInduction;
+ else if (C->getValue()->equalsInt(0 - Size))
+ return IK_ReversePtrInduction;
return IK_NoInduction;
}
Added: llvm/trunk/test/Transforms/LoopVectorize/bzip_reverse_loops.ll
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/test/Transforms/LoopVectorize/bzip_reverse_loops.ll?rev=173219&view=auto
==============================================================================
--- llvm/trunk/test/Transforms/LoopVectorize/bzip_reverse_loops.ll (added)
+++ llvm/trunk/test/Transforms/LoopVectorize/bzip_reverse_loops.ll Tue Jan 22 19:35:00 2013
@@ -0,0 +1,71 @@
+; RUN: opt < %s -loop-vectorize -force-vector-unroll=1 -force-vector-width=4 -dce -instcombine -S -enable-if-conversion | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.8.0"
+
+;CHECK: fc
+;CHECK: load <4 x i16>
+;CHECK-NEXT: shufflevector <4 x i16>
+;CHECK: select <4 x i1>
+;CHECK: store <4 x i16>
+;CHECK: ret
+define void @fc(i16* nocapture %p, i32 %n, i32 %size) nounwind uwtable ssp {
+entry:
+ br label %do.body
+
+do.body: ; preds = %cond.end, %entry
+ %n.addr.0 = phi i32 [ %n, %entry ], [ %dec, %cond.end ]
+ %p.addr.0 = phi i16* [ %p, %entry ], [ %incdec.ptr, %cond.end ]
+ %incdec.ptr = getelementptr inbounds i16* %p.addr.0, i64 -1
+ %0 = load i16* %incdec.ptr, align 2, !tbaa !0
+ %conv = zext i16 %0 to i32
+ %cmp = icmp ult i32 %conv, %size
+ br i1 %cmp, label %cond.end, label %cond.true
+
+cond.true: ; preds = %do.body
+ %sub = sub i32 %conv, %size
+ %phitmp = trunc i32 %sub to i16
+ br label %cond.end
+
+cond.end: ; preds = %do.body, %cond.true
+ %cond = phi i16 [ %phitmp, %cond.true ], [ 0, %do.body ]
+ store i16 %cond, i16* %incdec.ptr, align 2, !tbaa !0
+ %dec = add i32 %n.addr.0, -1
+ %tobool = icmp eq i32 %dec, 0
+ br i1 %tobool, label %do.end, label %do.body
+
+do.end: ; preds = %cond.end
+ ret void
+}
+
+;CHECK: example1
+;CHECK: load <4 x i32>
+;CHECK-NEXT: shufflevector <4 x i32>
+;CHECK: select <4 x i1>
+;CHECK: store <4 x i32>
+;CHECK: ret
+define void @example1(i32* nocapture %a, i32 %n, i32 %wsize) nounwind uwtable ssp {
+entry:
+ br label %do.body
+
+do.body: ; preds = %do.body, %entry
+ %n.addr.0 = phi i32 [ %n, %entry ], [ %dec, %do.body ]
+ %p.0 = phi i32* [ %a, %entry ], [ %incdec.ptr, %do.body ]
+ %incdec.ptr = getelementptr inbounds i32* %p.0, i64 -1
+ %0 = load i32* %incdec.ptr, align 4, !tbaa !3
+ %cmp = icmp slt i32 %0, %wsize
+ %sub = sub nsw i32 %0, %wsize
+ %cond = select i1 %cmp, i32 0, i32 %sub
+ store i32 %cond, i32* %incdec.ptr, align 4, !tbaa !3
+ %dec = add nsw i32 %n.addr.0, -1
+ %tobool = icmp eq i32 %dec, 0
+ br i1 %tobool, label %do.end, label %do.body
+
+do.end: ; preds = %do.body
+ ret void
+}
+
+!0 = metadata !{metadata !"short", metadata !1}
+!1 = metadata !{metadata !"omnipotent char", metadata !2}
+!2 = metadata !{metadata !"Simple C/C++ TBAA"}
+!3 = metadata !{metadata !"int", metadata !1}
Added: llvm/trunk/test/Transforms/LoopVectorize/ptr_loops.ll
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/test/Transforms/LoopVectorize/ptr_loops.ll?rev=173219&view=auto
==============================================================================
--- llvm/trunk/test/Transforms/LoopVectorize/ptr_loops.ll (added)
+++ llvm/trunk/test/Transforms/LoopVectorize/ptr_loops.ll Tue Jan 22 19:35:00 2013
@@ -0,0 +1,74 @@
+; RUN: opt < %s -loop-vectorize -force-vector-unroll=1 -force-vector-width=4 -dce -instcombine -S -enable-if-conversion | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.8.0"
+
+ at A = global [36 x i32] [i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35], align 16
+ at B = global [36 x i32] [i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35], align 16
+
+;CHECK:_Z5test1v
+;CHECK: load <4 x i32>
+;CHECK: shufflevector <4 x i32>
+;CHECK: store <4 x i32>
+;CHECK: ret
+define i32 @_Z5test1v() nounwind uwtable ssp {
+ br label %1
+
+; <label>:1 ; preds = %0, %1
+ %p.02 = phi i32* [ getelementptr inbounds ([36 x i32]* @A, i64 0, i64 18), %0 ], [ %4, %1 ]
+ %b.01 = phi i32* [ getelementptr inbounds ([36 x i32]* @B, i64 0, i64 0), %0 ], [ %5, %1 ]
+ %2 = load i32* %b.01, align 4
+ %3 = shl nsw i32 %2, 1
+ store i32 %3, i32* %p.02, align 4
+ %4 = getelementptr inbounds i32* %p.02, i64 -1
+ %5 = getelementptr inbounds i32* %b.01, i64 1
+ %6 = icmp eq i32* %4, getelementptr ([36 x i32]* @A, i64 128102389400760775, i64 3)
+ br i1 %6, label %7, label %1
+
+; <label>:7 ; preds = %1
+ ret i32 0
+}
+
+;CHECK:_Z5test2v
+;CHECK: load <4 x i32>
+;CHECK: shufflevector <4 x i32>
+;CHECK: store <4 x i32>
+;CHECK: ret
+define i32 @_Z5test2v() nounwind uwtable ssp {
+ br label %1
+
+; <label>:1 ; preds = %0, %1
+ %p.02 = phi i32* [ getelementptr inbounds ([36 x i32]* @A, i64 0, i64 25), %0 ], [ %3, %1 ]
+ %b.01 = phi i32* [ getelementptr inbounds ([36 x i32]* @B, i64 0, i64 2), %0 ], [ %4, %1 ]
+ %2 = load i32* %b.01, align 4
+ store i32 %2, i32* %p.02, align 4
+ %3 = getelementptr inbounds i32* %p.02, i64 -1
+ %4 = getelementptr inbounds i32* %b.01, i64 1
+ %5 = icmp eq i32* %4, getelementptr inbounds ([36 x i32]* @A, i64 0, i64 18)
+ br i1 %5, label %6, label %1
+
+; <label>:6 ; preds = %1
+ ret i32 0
+}
+
+;CHECK:_Z5test3v
+;CHECK: load <4 x i32>
+;CHECK: shufflevector <4 x i32>
+;CHECK: store <4 x i32>
+;CHECK: ret
+define i32 @_Z5test3v() nounwind uwtable ssp {
+ br label %1
+
+; <label>:1 ; preds = %0, %1
+ %p.02 = phi i32* [ getelementptr inbounds ([36 x i32]* @A, i64 0, i64 29), %0 ], [ %3, %1 ]
+ %b.01 = phi i32* [ getelementptr inbounds ([36 x i32]* @B, i64 0, i64 5), %0 ], [ %4, %1 ]
+ %2 = load i32* %b.01, align 4
+ store i32 %2, i32* %p.02, align 4
+ %3 = getelementptr inbounds i32* %p.02, i64 -1
+ %4 = getelementptr inbounds i32* %b.01, i64 1
+ %5 = icmp eq i32* %3, getelementptr ([36 x i32]* @A, i64 128102389400760775, i64 3)
+ br i1 %5, label %6, label %1
+
+; <label>:6 ; preds = %1
+ ret i32 0
+}
Added: llvm/trunk/test/Transforms/LoopVectorize/struct_access.ll
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/test/Transforms/LoopVectorize/struct_access.ll?rev=173219&view=auto
==============================================================================
--- llvm/trunk/test/Transforms/LoopVectorize/struct_access.ll (added)
+++ llvm/trunk/test/Transforms/LoopVectorize/struct_access.ll Tue Jan 22 19:35:00 2013
@@ -0,0 +1,50 @@
+; RUN: opt < %s -loop-vectorize -force-vector-unroll=1 -force-vector-width=4 -instcombine -S | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.9.0"
+
+%struct.coordinate = type { i32, i32 }
+
+; Make sure that we don't generate a wide load when accessing the struct.
+; struct coordinate {
+; int x;
+; int y;
+; };
+;
+;
+; int foo(struct coordinate *A, int n) {
+;
+; int sum = 0;
+; for (int i = 0; i < n; ++i)
+; sum += A[i].x;
+;
+; return sum;
+; }
+
+;CHECK: @foo
+;CHECK-NOT: load <4 x i32>
+;CHECK: ret
+define i32 @foo(%struct.coordinate* nocapture %A, i32 %n) nounwind uwtable readonly ssp {
+entry:
+ %cmp4 = icmp sgt i32 %n, 0
+ br i1 %cmp4, label %for.body, label %for.end
+
+for.body: ; preds = %entry, %for.body
+ %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+ %sum.05 = phi i32 [ %add, %for.body ], [ 0, %entry ]
+ %x = getelementptr inbounds %struct.coordinate* %A, i64 %indvars.iv, i32 0
+ %0 = load i32* %x, align 4, !tbaa !0
+ %add = add nsw i32 %0, %sum.05
+ %indvars.iv.next = add i64 %indvars.iv, 1
+ %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+ %exitcond = icmp eq i32 %lftr.wideiv, %n
+ br i1 %exitcond, label %for.end, label %for.body
+
+for.end: ; preds = %for.body, %entry
+ %sum.0.lcssa = phi i32 [ 0, %entry ], [ %add, %for.body ]
+ ret i32 %sum.0.lcssa
+}
+
+!0 = metadata !{metadata !"int", metadata !1}
+!1 = metadata !{metadata !"omnipotent char", metadata !2}
+!2 = metadata !{metadata !"Simple C/C++ TBAA"}
More information about the llvm-commits
mailing list