[llvm] [VPlan] Implement VPlan-based unit-strideness speculation (PR #182595)

[Luke Lau via llvm-commits]

================
@@ -5478,39 +5523,15 @@ void VPlanTransforms::makeMemOpWideningDecisions(VPlan &Plan, VFRange &Range,
                                 getLoadStoreAddressSpace(I)))
           return false;
 
-        VPBuilder Builder(VPI);
-        VPSingleDefRecipe *VectorPtr = Builder.createConsecutiveVectorPointer(
-            Ptr, ScalarTy, Reverse, VPI->getDebugLoc());
-
-        VPValue *Mask = IsPredicated ? VPI->getMask() : nullptr;
-        // Reverse the mask so it matches the reversed access order.
-        if (Reverse && Mask)
-          Mask = Builder.createNaryOp(VPInstruction::Reverse, Mask,
-                                      VPI->getDebugLoc());
-
-        if (IsLoad) {
-          VPSingleDefRecipe *Load = Builder.createWidenLoad(
-              *cast<LoadInst>(I), VectorPtr, Mask,
-              /*Consecutive=*/true, *VPI, VPI->getDebugLoc());
-          // Reverse the loaded values back into program order.
-          if (Reverse)
-            Load = Builder.createNaryOp(VPInstruction::Reverse, Load,
-                                        VPI->getDebugLoc());
-          return ReplaceWith(VPI, Load);
-        }
-
-        VPValue *StoredVal = VPI->getOperand(0);
-        if (Reverse)
-          // Reverse the stored values so they are written in descending order.
-          StoredVal = Builder.createNaryOp(VPInstruction::Reverse, StoredVal,
-                                           VPI->getDebugLoc());
-
-        auto *StoreR = Builder.createWidenStore(
-            *cast<StoreInst>(I), VectorPtr, StoredVal, Mask,
-            /*Consecutive=*/true, *VPI, VPI->getDebugLoc());
-        return ReplaceWith(VPI, StoreR);
+        widenUnitStridedLoadStore(VPI, Reverse, IsPredicated);
+        return true;
       });
 
+  if (EnableVPlanBasedStrideMV &&
+      !CostCtx.L->getHeader()->getParent()->hasOptSize())
+    RUN_VPLAN_PASS(VPlanTransforms::multiversionForUnitStridedMemOps, Plan,
----------------
lukel97 wrote:

I think after the latest rebase it might be possible to run `multiversionForUnitStridedMemOps` before the `widenConsecutiveMemOps` subpass, and just leave widening to the latter. So the multiversioning would only be replacing the stride values. Not sure if there's something prevent it or not but worth exploring probably

https://github.com/llvm/llvm-project/pull/182595