[PATCH] D128272: [LoopVectorize] Change PredicatedBBsAfterVectorization to be per VF

[David Sherwood via Phabricator via llvm-commits]

david-arm created this revision.
david-arm added reviewers: sdesmalen, kmclaughlin, peterwaller-arm, CarolineConcatto, fhahn.
Herald added subscribers: ctetreau, hiraditya, kristof.beyls.
Herald added a project: All.
david-arm requested review of this revision.
Herald added subscribers: llvm-commits, alextsao1999.
Herald added a project: LLVM.

When calculating the cost of Instruction::Br in getInstructionCost
we query PredicatedBBsAfterVectorization to see if there is a
scalar predicated block. However, this meant that the decisions
being made for a given fixed-width VF were affecting the cost for a
scalable VF. As a result we were returning InstructionCost::Invalid
pointlessly for a scalable VF that should have a low cost. I
encountered this for some loops when enabling tail-folding for
scalable VFs.

Test added here:

  Transforms/LoopVectorize/AArch64/sve-tail-folding-cost.ll


Repository:
  rG LLVM Github Monorepo

https://reviews.llvm.org/D128272

Files:
  llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
  llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-cost.ll


Index: llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-cost.ll
===================================================================
--- /dev/null
+++ llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-cost.ll
@@ -0,0 +1,34 @@
+; RUN: opt -S -loop-vectorize -prefer-predicate-over-epilogue=predicate-dont-vectorize -debug -disable-output <%s 2>%t
+; RUN: cat %t | FileCheck %s
+
+target triple = "aarch64-unknown-linux-gnu"
+
+ at globval = dso_local local_unnamed_addr global float 0x0000000000000000, align 4
+
+define dso_local i32 @uniform_load(i64 %n, i64 %boff, i64 %coff, ptr nocapture readonly %b, ptr nocapture readnone %c) #0 {
+; CHECK: LAA: Found a loop in uniform_load: for.body
+; CHECK-NOT: LV: Vector loop of width vscale x 2 costs: Invalid
+; CHECK-NOT: LV: Vector loop of width vscale x 4 costs: Invalid
+; CHECK: LV: Selecting VF: vscale x 4
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %for.body
+  %indvars.iv = phi i64 [ 1, %entry ], [ %indvars.iv.next, %for.body ]
+  %idxprom = add i64 %indvars.iv, %boff
+  %arrayidx = getelementptr inbounds float, ptr %b, i64 %idxprom
+  %load1 = load float, ptr %arrayidx, align 4
+  %load2 = load float, ptr @globval, align 4
+  %add12 = fadd fast float %load1, %load2
+  %idxprom2 = add i64 %indvars.iv, %coff
+  %arrayidx2 = getelementptr inbounds float, ptr %c, i64 %idxprom2
+  store float %add12, ptr %arrayidx2, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond.not = icmp eq i64 %indvars.iv.next, %n
+  br i1 %exitcond.not, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret i32 0
+}
+
+attributes #0 = { vscale_range(1,16) "target-features"="+sve" }
Index: llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
===================================================================
--- llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -1736,7 +1736,8 @@
 
   /// A set containing all BasicBlocks that are known to present after
   /// vectorization as a predicated block.
-  SmallPtrSet<BasicBlock *, 4> PredicatedBBsAfterVectorization;
+  DenseMap<ElementCount, SmallPtrSet<BasicBlock *, 4>>
+      PredicatedBBsAfterVectorization;
 
   /// Records whether it is allowed to have the original scalar loop execute at
   /// least once. This may be needed as a fallback loop in case runtime
@@ -6097,6 +6098,8 @@
   // map will indicate that we've analyzed it already.
   ScalarCostsTy &ScalarCostsVF = InstsToScalarize[VF];
 
+  PredicatedBBsAfterVectorization[VF].clear();
+
   // Find all the instructions that are scalar with predication in the loop and
   // determine if it would be better to not if-convert the blocks they are in.
   // If so, we also record the instructions to scalarize.
@@ -6114,7 +6117,7 @@
             computePredInstDiscount(&I, ScalarCosts, VF) >= 0)
           ScalarCostsVF.insert(ScalarCosts.begin(), ScalarCosts.end());
         // Remember that BB will remain after vectorization.
-        PredicatedBBsAfterVectorization.insert(BB);
+        PredicatedBBsAfterVectorization[VF].insert(BB);
       }
   }
 }
@@ -6974,8 +6977,8 @@
     bool ScalarPredicatedBB = false;
     BranchInst *BI = cast<BranchInst>(I);
     if (VF.isVector() && BI->isConditional() &&
-        (PredicatedBBsAfterVectorization.count(BI->getSuccessor(0)) ||
-         PredicatedBBsAfterVectorization.count(BI->getSuccessor(1))))
+        (PredicatedBBsAfterVectorization[VF].count(BI->getSuccessor(0)) ||
+         PredicatedBBsAfterVectorization[VF].count(BI->getSuccessor(1))))
       ScalarPredicatedBB = true;
 
     if (ScalarPredicatedBB) {


-------------- next part --------------
A non-text attachment was scrubbed...
Name: D128272.438677.patch
Type: text/x-patch
Size: 3739 bytes
Desc: not available
URL: <http://lists.llvm.org/pipermail/llvm-commits/attachments/20220621/98b9fe34/attachment.bin>