<div dir="ltr">hope r207514 fixes it.</div><div class="gmail_extra"><br><br><div class="gmail_quote">On Tue, Apr 29, 2014 at 1:33 PM, Kostya Serebryany <span dir="ltr"><<a href="mailto:kcc@google.com" target="_blank">kcc@google.com</a>></span> wrote:<br>
<blockquote class="gmail_quote" style="margin:0 0 0 .8ex;border-left:1px #ccc solid;padding-left:1ex"><div dir="ltr">This causes build breakage in release mode: <div><div>lib/Transforms/Vectorize/LoopVectorize.cpp:5099:15: error: unused variable 'ScalarCost' [-Werror,-Wunused-variable]</div>
<div class=""><div> const float ScalarCost = Cost;</div>
<div><br></div></div><div>(The constant is used in DEBUG() statement which is void in Release mode)</div><div><br></div></div></div><div class="HOEnZb"><div class="h5"><div class="gmail_extra"><br><br><div class="gmail_quote">
On Tue, Apr 29, 2014 at 12:55 PM, Zinovy Nis <span dir="ltr"><<a href="mailto:zinovy.nis@gmail.com" target="_blank">zinovy.nis@gmail.com</a>></span> wrote:<br>
<blockquote class="gmail_quote" style="margin:0 0 0 .8ex;border-left:1px #ccc solid;padding-left:1ex">Author: zinovy.nis<br>
Date: Tue Apr 29 03:55:11 2014<br>
New Revision: 207512<br>
<br>
URL: <a href="http://llvm.org/viewvc/llvm-project?rev=207512&view=rev" target="_blank">http://llvm.org/viewvc/llvm-project?rev=207512&view=rev</a><br>
Log:<br>
[OPENMP][LV][D3423] Respect Hints.Force meta-data for loops in LoopVectorizer<br>
<br>
Added:<br>
llvm/trunk/test/Transforms/LoopVectorize/X86/vect.omp.force.ll<br>
llvm/trunk/test/Transforms/LoopVectorize/X86/vect.omp.force.small-tc.ll<br>
Modified:<br>
llvm/trunk/lib/Transforms/Vectorize/LoopVectorize.cpp<br>
<br>
Modified: llvm/trunk/lib/Transforms/Vectorize/LoopVectorize.cpp<br>
URL: <a href="http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Transforms/Vectorize/LoopVectorize.cpp?rev=207512&r1=207511&r2=207512&view=diff" target="_blank">http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Transforms/Vectorize/LoopVectorize.cpp?rev=207512&r1=207511&r2=207512&view=diff</a><br>
==============================================================================<br>
--- llvm/trunk/lib/Transforms/Vectorize/LoopVectorize.cpp (original)<br>
+++ llvm/trunk/lib/Transforms/Vectorize/LoopVectorize.cpp Tue Apr 29 03:55:11 2014<br>
@@ -819,7 +819,8 @@ public:<br>
/// then this vectorization factor will be selected if vectorization is<br>
/// possible.<br>
VectorizationFactor selectVectorizationFactor(bool OptForSize,<br>
- unsigned UserVF);<br>
+ unsigned UserVF,<br>
+ bool ForceVectorization);<br>
<br>
/// \return The size (in bits) of the widest type in the code that<br>
/// needs to be vectorized. We ignore values that remain scalar such as<br>
@@ -891,13 +892,17 @@ struct LoopVectorizeHints {<br>
unsigned Width;<br>
/// Vectorization unroll factor.<br>
unsigned Unroll;<br>
- /// Vectorization forced (-1 not selected, 0 force disabled, 1 force enabled)<br>
- int Force;<br>
+ /// Vectorization forced<br>
+ enum ForceKind {<br>
+ FK_Undefined = -1, ///< Not selected.<br>
+ FK_Disabled = 0, ///< Forcing disabled.<br>
+ FK_Enabled = 1, ///< Forcing enabled.<br>
+ } Force;<br>
<br>
LoopVectorizeHints(const Loop *L, bool DisableUnrolling)<br>
: Width(VectorizationFactor)<br>
, Unroll(DisableUnrolling ? 1 : VectorizationUnroll)<br>
- , Force(-1)<br>
+ , Force(FK_Undefined)<br>
, LoopID(L->getLoopID()) {<br>
getHints(L);<br>
// The command line options override any loop metadata except for when<br>
@@ -1010,7 +1015,8 @@ private:<br>
DEBUG(dbgs() << "LV: ignoring invalid unroll hint metadata\n");<br>
} else if (Hint == "enable") {<br>
if (C->getBitWidth() == 1)<br>
- Force = Val;<br>
+ Force = Val == 1 ? LoopVectorizeHints::FK_Enabled<br>
+ : LoopVectorizeHints::FK_Disabled;<br>
else<br>
DEBUG(dbgs() << "LV: ignoring invalid enable hint metadata\n");<br>
} else {<br>
@@ -1106,18 +1112,20 @@ struct LoopVectorize : public FunctionPa<br>
LoopVectorizeHints Hints(L, DisableUnrolling);<br>
<br>
DEBUG(dbgs() << "LV: Loop hints:"<br>
- << " force=" << (Hints.Force == 0<br>
- ? "disabled"<br>
- : (Hints.Force == 1 ? "enabled" : "?"))<br>
- << " width=" << Hints.Width << " unroll=" << Hints.Unroll<br>
- << "\n");<br>
+ << " force="<br>
+ << (Hints.Force == LoopVectorizeHints::FK_Disabled<br>
+ ? "disabled"<br>
+ : (Hints.Force == LoopVectorizeHints::FK_Enabled<br>
+ ? "enabled"<br>
+ : "?")) << " width=" << Hints.Width<br>
+ << " unroll=" << Hints.Unroll << "\n");<br>
<br>
- if (Hints.Force == 0) {<br>
+ if (Hints.Force == LoopVectorizeHints::FK_Disabled) {<br>
DEBUG(dbgs() << "LV: Not vectorizing: #pragma vectorize disable.\n");<br>
return false;<br>
}<br>
<br>
- if (!AlwaysVectorize && Hints.Force != 1) {<br>
+ if (!AlwaysVectorize && Hints.Force != LoopVectorizeHints::FK_Enabled) {<br>
DEBUG(dbgs() << "LV: Not vectorizing: No #pragma vectorize enable.\n");<br>
return false;<br>
}<br>
@@ -1127,6 +1135,21 @@ struct LoopVectorize : public FunctionPa<br>
return false;<br>
}<br>
<br>
+ // Check the loop for a trip count threshold:<br>
+ // do not vectorize loops with a tiny trip count.<br>
+ BasicBlock *Latch = L->getLoopLatch();<br>
+ const unsigned TC = SE->getSmallConstantTripCount(L, Latch);<br>
+ if (TC > 0u && TC < TinyTripCountVectorThreshold) {<br>
+ DEBUG(dbgs() << "LV: Found a loop with a very small trip count. "<br>
+ << "This loop is not worth vectorizing.");<br>
+ if (Hints.Force == LoopVectorizeHints::FK_Enabled)<br>
+ DEBUG(dbgs() << " But vectorizing was explicitly forced.\n");<br>
+ else {<br>
+ DEBUG(dbgs() << "\n");<br>
+ return false;<br>
+ }<br>
+ }<br>
+<br>
// Check if it is legal to vectorize the loop.<br>
LoopVectorizationLegality LVL(L, SE, DL, DT, TLI);<br>
if (!LVL.canVectorize()) {<br>
@@ -1140,8 +1163,8 @@ struct LoopVectorize : public FunctionPa<br>
// Check the function attributes to find out if this function should be<br>
// optimized for size.<br>
Function *F = L->getHeader()->getParent();<br>
- bool OptForSize =<br>
- Hints.Force != 1 && F->hasFnAttribute(Attribute::OptimizeForSize);<br>
+ bool OptForSize = Hints.Force != LoopVectorizeHints::FK_Enabled &&<br>
+ F->hasFnAttribute(Attribute::OptimizeForSize);<br>
<br>
// Compute the weighted frequency of this loop being executed and see if it<br>
// is less than 20% of the function entry baseline frequency. Note that we<br>
@@ -1150,7 +1173,8 @@ struct LoopVectorize : public FunctionPa<br>
// exactly what block frequency models.<br>
if (LoopVectorizeWithBlockFrequency) {<br>
BlockFrequency LoopEntryFreq = BFI->getBlockFreq(L->getLoopPreheader());<br>
- if (Hints.Force != 1 && LoopEntryFreq < ColdEntryFreq)<br>
+ if (Hints.Force != LoopVectorizeHints::FK_Enabled &&<br>
+ LoopEntryFreq < ColdEntryFreq)<br>
OptForSize = true;<br>
}<br>
<br>
@@ -1166,7 +1190,10 @@ struct LoopVectorize : public FunctionPa<br>
<br>
// Select the optimal vectorization factor.<br>
const LoopVectorizationCostModel::VectorizationFactor VF =<br>
- CM.selectVectorizationFactor(OptForSize, Hints.Width);<br>
+ CM.selectVectorizationFactor(OptForSize, Hints.Width,<br>
+ Hints.Force ==<br>
+ LoopVectorizeHints::FK_Enabled);<br>
+<br>
// Select the unroll factor.<br>
const unsigned UF = CM.selectUnrollFactor(OptForSize, Hints.Unroll, VF.Width,<br>
VF.Cost);<br>
@@ -3300,15 +3327,6 @@ bool LoopVectorizationLegality::canVecto<br>
return false;<br>
}<br>
<br>
- // Do not loop-vectorize loops with a tiny trip count.<br>
- BasicBlock *Latch = TheLoop->getLoopLatch();<br>
- unsigned TC = SE->getSmallConstantTripCount(TheLoop, Latch);<br>
- if (TC > 0u && TC < TinyTripCountVectorThreshold) {<br>
- DEBUG(dbgs() << "LV: Found a loop with a very small trip count. " <<<br>
- "This loop is not worth vectorizing.\n");<br>
- return false;<br>
- }<br>
-<br>
// Check if we can vectorize the instructions and CFG in this loop.<br>
if (!canVectorizeInstrs()) {<br>
DEBUG(dbgs() << "LV: Can't vectorize the instructions or CFG\n");<br>
@@ -5007,7 +5025,8 @@ bool LoopVectorizationLegality::blockCan<br>
<br>
LoopVectorizationCostModel::VectorizationFactor<br>
LoopVectorizationCostModel::selectVectorizationFactor(bool OptForSize,<br>
- unsigned UserVF) {<br>
+ unsigned UserVF,<br>
+ bool ForceVectorization) {<br>
// Width 1 means no vectorize<br>
VectorizationFactor Factor = { 1U, 0U };<br>
if (OptForSize && Legal->getRuntimePointerCheck()->Need) {<br>
@@ -5077,8 +5096,16 @@ LoopVectorizationCostModel::selectVector<br>
}<br>
<br>
float Cost = expectedCost(1);<br>
+ const float ScalarCost = Cost;<br>
unsigned Width = 1;<br>
DEBUG(dbgs() << "LV: Scalar loop costs: " << (int)Cost << ".\n");<br>
+<br>
+ // Ignore scalar width, because the user explicitly wants vectorization.<br>
+ if (ForceVectorization && VF > 1) {<br>
+ Width = 2;<br>
+ Cost = expectedCost(Width) / (float)Width;<br>
+ }<br>
+<br>
for (unsigned i=2; i <= VF; i*=2) {<br>
// Notice that the vector loop needs to be executed less times, so<br>
// we need to divide the cost of the vector loops by the width of<br>
@@ -5092,6 +5119,9 @@ LoopVectorizationCostModel::selectVector<br>
}<br>
}<br>
<br>
+ DEBUG(if (ForceVectorization && Width > 1 && Cost >= ScalarCost) dbgs()<br>
+ << "LV: Vectorization seems to be not beneficial, "<br>
+ << "but was forced by a user.\n");<br>
DEBUG(dbgs() << "LV: Selecting VF: "<< Width << ".\n");<br>
Factor.Width = Width;<br>
Factor.Cost = Width * Cost;<br>
<br>
Added: llvm/trunk/test/Transforms/LoopVectorize/X86/vect.omp.force.ll<br>
URL: <a href="http://llvm.org/viewvc/llvm-project/llvm/trunk/test/Transforms/LoopVectorize/X86/vect.omp.force.ll?rev=207512&view=auto" target="_blank">http://llvm.org/viewvc/llvm-project/llvm/trunk/test/Transforms/LoopVectorize/X86/vect.omp.force.ll?rev=207512&view=auto</a><br>
==============================================================================<br>
--- llvm/trunk/test/Transforms/LoopVectorize/X86/vect.omp.force.ll (added)<br>
+++ llvm/trunk/test/Transforms/LoopVectorize/X86/vect.omp.force.ll Tue Apr 29 03:55:11 2014<br>
@@ -0,0 +1,93 @@<br>
+; RUN: opt < %s -loop-vectorize -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx -debug-only=loop-vectorize -stats -S 2>&1 | FileCheck %s<br>
+; REQUIRES: asserts<br>
+<br>
+; CHECK: LV: Loop hints: force=enabled<br>
+; CHECK: LV: Loop hints: force=?<br>
+; No more loops in the module<br>
+; CHECK-NOT: LV: Loop hints: force=<br>
+; CHECK: 2 loop-vectorize - Number of loops analyzed for vectorization<br>
+; CHECK: 1 loop-vectorize - Number of loops vectorized<br>
+<br>
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"<br>
+target triple = "x86_64-apple-macosx10.8.0"<br>
+<br>
+;<br>
+; The source code for the test:<br>
+;<br>
+; #include <math.h><br>
+; void foo(float* restrict A, float * restrict B, int size)<br>
+; {<br>
+; for (int i = 0; i < size; ++i) A[i] = sinf(B[i]);<br>
+; }<br>
+;<br>
+<br>
+;<br>
+; This loop will be vectorized, although the scalar cost is lower than any of vector costs, but vectorization is explicitly forced in metadata.<br>
+;<br>
+<br>
+define void @vectorized(float* noalias nocapture %A, float* noalias nocapture %B, i32 %size) {<br>
+entry:<br>
+ %cmp6 = icmp sgt i32 %size, 0<br>
+ br i1 %cmp6, label %for.body.preheader, label %for.end<br>
+<br>
+for.body.preheader:<br>
+ br label %for.body<br>
+<br>
+for.body:<br>
+ %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %for.body.preheader ]<br>
+ %arrayidx = getelementptr inbounds float* %B, i64 %indvars.iv<br>
+ %0 = load float* %arrayidx, align 4, !llvm.mem.parallel_loop_access !1<br>
+ %call = tail call float @llvm.sin.f32(float %0)<br>
+ %arrayidx2 = getelementptr inbounds float* %A, i64 %indvars.iv<br>
+ store float %call, float* %arrayidx2, align 4, !llvm.mem.parallel_loop_access !1<br>
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1<br>
+ %lftr.wideiv = trunc i64 %indvars.iv.next to i32<br>
+ %exitcond = icmp eq i32 %lftr.wideiv, %size<br>
+ br i1 %exitcond, label %for.end.loopexit, label %for.body, !llvm.loop !1<br>
+<br>
+for.end.loopexit:<br>
+ br label %for.end<br>
+<br>
+for.end:<br>
+ ret void<br>
+}<br>
+<br>
+!1 = metadata !{metadata !1, metadata !2}<br>
+!2 = metadata !{metadata !"llvm.vectorizer.enable", i1 true}<br>
+<br>
+;<br>
+; This method will not be vectorized, as scalar cost is lower than any of vector costs.<br>
+;<br>
+<br>
+define void @not_vectorized(float* noalias nocapture %A, float* noalias nocapture %B, i32 %size) {<br>
+entry:<br>
+ %cmp6 = icmp sgt i32 %size, 0<br>
+ br i1 %cmp6, label %for.body.preheader, label %for.end<br>
+<br>
+for.body.preheader:<br>
+ br label %for.body<br>
+<br>
+for.body:<br>
+ %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %for.body.preheader ]<br>
+ %arrayidx = getelementptr inbounds float* %B, i64 %indvars.iv<br>
+ %0 = load float* %arrayidx, align 4, !llvm.mem.parallel_loop_access !3<br>
+ %call = tail call float @llvm.sin.f32(float %0)<br>
+ %arrayidx2 = getelementptr inbounds float* %A, i64 %indvars.iv<br>
+ store float %call, float* %arrayidx2, align 4, !llvm.mem.parallel_loop_access !3<br>
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1<br>
+ %lftr.wideiv = trunc i64 %indvars.iv.next to i32<br>
+ %exitcond = icmp eq i32 %lftr.wideiv, %size<br>
+ br i1 %exitcond, label %for.end.loopexit, label %for.body, !llvm.loop !3<br>
+<br>
+for.end.loopexit:<br>
+ br label %for.end<br>
+<br>
+for.end:<br>
+ ret void<br>
+}<br>
+<br>
+declare float @llvm.sin.f32(float) nounwind readnone<br>
+<br>
+; Dummy metadata<br>
+!3 = metadata !{metadata !3}<br>
+<br>
<br>
Added: llvm/trunk/test/Transforms/LoopVectorize/X86/vect.omp.force.small-tc.ll<br>
URL: <a href="http://llvm.org/viewvc/llvm-project/llvm/trunk/test/Transforms/LoopVectorize/X86/vect.omp.force.small-tc.ll?rev=207512&view=auto" target="_blank">http://llvm.org/viewvc/llvm-project/llvm/trunk/test/Transforms/LoopVectorize/X86/vect.omp.force.small-tc.ll?rev=207512&view=auto</a><br>
==============================================================================<br>
--- llvm/trunk/test/Transforms/LoopVectorize/X86/vect.omp.force.small-tc.ll (added)<br>
+++ llvm/trunk/test/Transforms/LoopVectorize/X86/vect.omp.force.small-tc.ll Tue Apr 29 03:55:11 2014<br>
@@ -0,0 +1,73 @@<br>
+; RUN: opt < %s -loop-vectorize -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx -debug-only=loop-vectorize -stats -S -vectorizer-min-trip-count=21 2>&1 | FileCheck %s<br>
+; REQUIRES: asserts<br>
+<br>
+; CHECK: LV: Loop hints: force=enabled<br>
+; CHECK: LV: Loop hints: force=?<br>
+; No more loops in the module<br>
+; CHECK-NOT: LV: Loop hints: force=<br>
+; CHECK: 2 loop-vectorize - Number of loops analyzed for vectorization<br>
+; CHECK: 1 loop-vectorize - Number of loops vectorized<br>
+<br>
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"<br>
+target triple = "x86_64-apple-macosx10.8.0"<br>
+<br>
+;<br>
+; The source code for the test:<br>
+;<br>
+; void foo(float* restrict A, float* restrict B)<br>
+; {<br>
+; for (int i = 0; i < 20; ++i) A[i] += B[i];<br>
+; }<br>
+;<br>
+<br>
+;<br>
+; This loop will be vectorized, although the trip count is below the threshold, but vectorization is explicitly forced in metadata.<br>
+;<br>
+define void @vectorized(float* noalias nocapture %A, float* noalias nocapture readonly %B) {<br>
+entry:<br>
+ br label %for.body<br>
+<br>
+for.body:<br>
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]<br>
+ %arrayidx = getelementptr inbounds float* %B, i64 %indvars.iv<br>
+ %0 = load float* %arrayidx, align 4, !llvm.mem.parallel_loop_access !1<br>
+ %arrayidx2 = getelementptr inbounds float* %A, i64 %indvars.iv<br>
+ %1 = load float* %arrayidx2, align 4, !llvm.mem.parallel_loop_access !1<br>
+ %add = fadd fast float %0, %1<br>
+ store float %add, float* %arrayidx2, align 4, !llvm.mem.parallel_loop_access !1<br>
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1<br>
+ %exitcond = icmp eq i64 %indvars.iv.next, 20<br>
+ br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !1<br>
+<br>
+for.end:<br>
+ ret void<br>
+}<br>
+<br>
+!1 = metadata !{metadata !1, metadata !2}<br>
+!2 = metadata !{metadata !"llvm.vectorizer.enable", i1 true}<br>
+<br>
+;<br>
+; This loop will not be vectorized as the trip count is below the threshold.<br>
+;<br>
+define void @not_vectorized(float* noalias nocapture %A, float* noalias nocapture readonly %B) {<br>
+entry:<br>
+ br label %for.body<br>
+<br>
+for.body:<br>
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]<br>
+ %arrayidx = getelementptr inbounds float* %B, i64 %indvars.iv<br>
+ %0 = load float* %arrayidx, align 4, !llvm.mem.parallel_loop_access !3<br>
+ %arrayidx2 = getelementptr inbounds float* %A, i64 %indvars.iv<br>
+ %1 = load float* %arrayidx2, align 4, !llvm.mem.parallel_loop_access !3<br>
+ %add = fadd fast float %0, %1<br>
+ store float %add, float* %arrayidx2, align 4, !llvm.mem.parallel_loop_access !3<br>
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1<br>
+ %exitcond = icmp eq i64 %indvars.iv.next, 20<br>
+ br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !3<br>
+<br>
+for.end:<br>
+ ret void<br>
+}<br>
+<br>
+!3 = metadata !{metadata !3}<br>
+<br>
<br>
<br>
_______________________________________________<br>
llvm-commits mailing list<br>
<a href="mailto:llvm-commits@cs.uiuc.edu" target="_blank">llvm-commits@cs.uiuc.edu</a><br>
<a href="http://lists.cs.uiuc.edu/mailman/listinfo/llvm-commits" target="_blank">http://lists.cs.uiuc.edu/mailman/listinfo/llvm-commits</a><br>
</blockquote></div><br></div>
</div></div></blockquote></div><br></div>