[llvm] [VPlan] Record estimated branch probabilities on VPlan0 for cost modeling (PR #216172)

[Mircea Trofin via llvm-commits]

================
@@ -6519,6 +6535,68 @@ VPRecipeBuilder::tryToCreateWidenNonPhiRecipe(VPSingleDefRecipe *R,
 // optimizations.
 static void printOptimizedVPlan(VPlan &) {}
 
+#ifndef NDEBUG
+/// Cross-check the probabilities vputils::computeBlockProbabilities computes
+/// for the blocks of the loop region of \p Plan against the frequencies \p BFI
+/// computed for the corresponding blocks of \p OrigLoop.
+/// FIXME: Temporary verification aid, to be removed.
+static bool verifyBlockProbabilitiesMatchBFI(VPlan &Plan, Loop *OrigLoop,
+                                             LoopInfo *LI,
+                                             BlockFrequencyInfo &BFI) {
+
+  // The verification is limited inner loops where the latch is the only exiting
+  // block and there are no extra VPBBs not mapped to IR BBs (when tailfolding).
+  if (Plan.isOuterLoop() ||
+      OrigLoop->getExitingBlock() != OrigLoop->getLoopLatch() ||
+      Plan.hasTailFolded())
+    return true;
+
+  // Visit the blocks of the loop region in the same order as
+  // introduceMasksAndLinearize does.
+  ReversePostOrderTraversal<VPBlockShallowTraversalWrapper<VPBlockBase *>> RPOT(
+      Plan.getVectorLoopRegion()->getEntryBasicBlock());
+  auto Blocks = to_vector(VPBlockUtils::blocksAs<VPBasicBlock>(RPOT));
+  assert(Blocks.size() == OrigLoop->getNumBlocks());
+
+  LoopBlocksRPO OrigRPO(OrigLoop);
+  OrigRPO.perform(LI);
+
+  uint64_t HeaderFreq = BFI.getBlockFreq(OrigLoop->getHeader()).getFrequency();
+  if (HeaderFreq == 0)
+    return true;
+
+  // BFI's fixed-point mass propagation rounds per edge, losing up to 1 ULP per
+  // block on the path from the header.
+  uint64_t Tolerance =
+      Blocks.size() + BranchProbability::getDenominator() / HeaderFreq;
+
+  DenseMap<const VPBasicBlock *, BranchProbability> Probabilities =
+      vputils::computeBlockProbabilities(Blocks);
+  for (const auto &[VPBB, BB] :
+       zip_equal(drop_begin(Blocks), drop_begin(OrigRPO))) {
+    BranchProbability Computed = Probabilities.lookup(VPBB);
+    // Currently VPlan-based probabilities are only computed when all blocks
+    // have branch-weights.
+    if (Computed.isUnknown())
+      continue;
+
+    // Clamp the frequency to the header's; it may exceed it slightly due to
+    // BFI's rounding.
+    uint64_t Freq = BFI.getBlockFreq(BB).getFrequency();
+    BranchProbability Expected = BranchProbability::getBranchProbability(
+        std::min(Freq, HeaderFreq), HeaderFreq);
+    if (AbsoluteDifference(Computed.getNumerator(), Expected.getNumerator()) <=
+        Tolerance)
+      continue;
+
+    errs() << "Block probability mismatch for " << VPBB->getName() << ": VPlan "
----------------
mtrofin wrote:

Block frequency (not probability)?

https://github.com/llvm/llvm-project/pull/216172