[llvm] [VPlan] Introduce recipes for VP loads and stores. (PR #87816)

[via llvm-commits]

================
@@ -1348,6 +1337,48 @@ void VPlanTransforms::addExplicitVectorLength(VPlan &Plan) {
   CanonicalIVIncrement->setOperand(0, CanonicalIVPHI);
   // TODO: support unroll factor > 1.
   Plan.setUF(1);
+
+  VPValue *BTC = Plan.getOrCreateBackedgeTakenCount();
+  auto IsHeaderMask = [BTC](VPValue *V) {
+    auto *CompareToReplace = dyn_cast<VPInstruction>(V);
+    return CompareToReplace &&
+           CompareToReplace->getOpcode() == Instruction::ICmp &&
+           CompareToReplace->getPredicate() == CmpInst::ICMP_ULE &&
+           CompareToReplace->getOperand(1) == BTC;
+  };
+
+  // Replace regular widened memory operations with vector-predicated versions.
+  auto Iter = vp_depth_first_deep(Header);
+  for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(Iter)) {
+    for (VPRecipeBase &R : make_early_inc_range(*VPBB)) {
+      auto *MemR = dyn_cast<VPWidenMemoryRecipe>(&R);
+      if (!MemR)
+        continue;
+      VPValue *OrigMask = MemR->getMask();
+      if (!OrigMask)
+        continue;
+      assert(!MemR->isReverse() &&
+             "Reversed memory operations not supported yet.");
+      VPValue *Mask = IsHeaderMask(OrigMask) ? nullptr : OrigMask;
+      if (auto *L = dyn_cast<VPWidenLoadRecipe>(&R)) {
+        auto *N = new VPWidenVPLoadRecipe(cast<LoadInst>(L->getIngredient()),
+                                          L->getAddr(), VPEVL, Mask,
+                                          L->isConsecutive(), L->getDebugLoc());
+        N->insertBefore(L);
+        L->replaceAllUsesWith(N);
+        L->eraseFromParent();
+      } else if (auto *S = dyn_cast<VPWidenStoreRecipe>(&R)) {
+        auto *N = new VPWidenVPStoreRecipe(
----------------
ayalz wrote:

nit: perhaps the constructor of VPWidenVPStoreRecipe should take a VPWidenStoreRecipe, VPEVL, and mask as parameters.

https://github.com/llvm/llvm-project/pull/87816