[llvm] [VPlan] Introduce recipes for VP loads and stores. (PR #87816)
via llvm-commits
llvm-commits at lists.llvm.org
Thu Apr 18 13:45:48 PDT 2024
================
@@ -1203,43 +1203,54 @@ static VPActiveLaneMaskPHIRecipe *addVPLaneMaskPhiAndUpdateExitBranch(
return LaneMaskPhi;
}
-/// Replaces (ICMP_ULE, WideCanonicalIV, backedge-taken-count) pattern using
-/// the given \p Idiom.
-static void
-replaceHeaderPredicateWith(VPlan &Plan, VPValue &Idiom,
- function_ref<bool(VPUser &, unsigned)> Cond = {}) {
+/// Collet all VPValues representing a header mask through the (ICMP_ULE,
+/// WideCanonicalIV, backedge-taken-count) pattern.
+static SmallVector<VPValue *> collectAllHeaderMasks(VPlan &Plan) {
+ SmallVector<VPValue *> WideCanonicalIVs;
auto *FoundWidenCanonicalIVUser =
find_if(Plan.getCanonicalIV()->users(),
[](VPUser *U) { return isa<VPWidenCanonicalIVRecipe>(U); });
- if (FoundWidenCanonicalIVUser == Plan.getCanonicalIV()->users().end())
- return;
- auto *WideCanonicalIV =
- cast<VPWidenCanonicalIVRecipe>(*FoundWidenCanonicalIVUser);
- // Walk users of WideCanonicalIV and replace all compares of the form
- // (ICMP_ULE, WideCanonicalIV, backedge-taken-count) with
- // the given idiom VPValue.
+ if (FoundWidenCanonicalIVUser != Plan.getCanonicalIV()->users().end()) {
+ auto *WideCanonicalIV =
+ cast<VPWidenCanonicalIVRecipe>(*FoundWidenCanonicalIVUser);
+ assert(all_of(Plan.getCanonicalIV()->users(),
+ [WideCanonicalIV](VPUser *U) {
+ return !isa<VPWidenCanonicalIVRecipe>(U) ||
+ U == WideCanonicalIV;
+ }) &&
+ "Must have a single WideCanonicalIV");
+ WideCanonicalIVs.push_back(WideCanonicalIV);
+ }
+
+ // Also include VPWidenIntOrFpInductionRecipes that represent a widened
+ // version of the canonical induction.
+ VPBasicBlock *HeaderVPBB = Plan.getVectorLoopRegion()->getEntryBasicBlock();
+ for (VPRecipeBase &Phi : HeaderVPBB->phis()) {
+ auto *WidenOriginalIV = dyn_cast<VPWidenIntOrFpInductionRecipe>(&Phi);
+ if (WidenOriginalIV && WidenOriginalIV->isCanonical())
+ WideCanonicalIVs.push_back(WidenOriginalIV);
+ }
+
+ // Walk users of wide canonical IVs and apply Fn to all compares of the form
+ // (ICMP_ULE, WideCanonicalIV, backedge-taken-count).
----------------
ayalz wrote:
```suggestion
// Walk users of wide canonical IVs and collect all compares of the form
// (ICMP_ULE, WideCanonicalIV, backedge-taken-count).
```
https://github.com/llvm/llvm-project/pull/87816
More information about the llvm-commits
mailing list