[llvm] [VPlan] Widen unit stride accesses in makeMemOpWideningDecisions (NFCI) (PR #205013)

[Ramkumar Ramachandra via llvm-commits]

================
@@ -7219,6 +7236,64 @@ void VPlanTransforms::makeMemOpWideningDecisions(
         return false;
       });
 
+  if (!RecipeBuilder.prefersVectorizedAddressing()) {
+    VPlanTransforms::runPass(
+        "makeVPlanMemOpDecision", ProcessSubset, Plan, [&](VPInstruction *VPI) {
+          Instruction *I = VPI->getUnderlyingInstr();
+          bool IsLoad = VPI->getOpcode() == Instruction::Load;
+          if (RecipeBuilder.isPredicatedInst(I) || !IsLoad ||
+              !vputils::isUsedByLoadStoreAddress(VPI))
+            return false;
+
+          // Scalarize loads used as addresses, matching the legacy CM. The load
+          // is single-scalar if the pointer is loop-invariant, otherwise it is
+          // replicated per-lane. No mask is needed as the load is not
+          // predicated.
+          VPValue *Ptr = VPI->getOperand(0);
+          const SCEV *PtrSCEV = vputils::getSCEVExprForVPValue(Ptr, PSE, L);
+          bool IsSingleScalarLoad = !isa<SCEVCouldNotCompute>(PtrSCEV) &&
+                                    PSE.getSE()->isLoopInvariant(PtrSCEV, L);
+          ReplaceWith(VPI, new VPReplicateRecipe(
+                               I, {Ptr}, /*IsSingleScalar=*/IsSingleScalarLoad,
+                               /*Mask=*/nullptr, {}, *VPI, VPI->getDebugLoc()));
+          return true;
+        });
+  }
+
+  // Widen unmasked unit-stride consecutive accesses, matching the legacy CM.
+  VPlanTransforms::runPass(
+      "widenConsecutiveMemOps", ProcessSubset, Plan, [&](VPInstruction *VPI) {
+        Instruction *I = VPI->getUnderlyingInstr();
+        if (RecipeBuilder.isPredicatedInst(I))
+          return false;
+
+        bool IsLoad = VPI->getOpcode() == Instruction::Load;
+        VPValue *Ptr = VPI->getOperand(!IsLoad);
+        Type *ScalarTy =
+            IsLoad ? VPI->getScalarType() : VPI->getOperand(0)->getScalarType();
+        if (getConstantStride(Ptr, ScalarTy, PSE, L) != 1)
+          return false;
+
+        Type *StrideTy =
+            Plan.getDataLayout().getIndexType(Ptr->getScalarType());
+        VPValue *StrideOne = Plan.getConstantInt(StrideTy, 1);
+        auto *VectorPtr = new VPVectorPointerRecipe(
+            Ptr, ScalarTy, StrideOne, vputils::getGEPFlagsForPtr(Ptr),
+            VPI->getDebugLoc());
+        VectorPtr->insertBefore(VPI);
+        VPRecipeBase *WidenedR;
+        if (IsLoad)
+          WidenedR = new VPWidenLoadRecipe(*cast<LoadInst>(I), VectorPtr,
+                                           /*Mask=*/nullptr,
+                                           /*Consecutive=*/true, *VPI,
+                                           VPI->getDebugLoc());
+        else
+          WidenedR = new VPWidenStoreRecipe(
+              *cast<StoreInst>(I), VectorPtr, VPI->getOperand(0),
+              /*Mask=*/nullptr, /*Consecutive=*/true, *VPI, VPI->getDebugLoc());
----------------
artagnon wrote:

```suggestion
        VPRecipeBase *WidenedR = IsLoad ? new VPWidenLoadRecipe(*cast<LoadInst>(I), VectorPtr,
                                           /*Mask=*/nullptr,
                                           /*Consecutive=*/true, *VPI,
                                           VPI->getDebugLoc()) : new VPWidenStoreRecipe(
              *cast<StoreInst>(I), VectorPtr, VPI->getOperand(0),
              /*Mask=*/nullptr, /*Consecutive=*/true, *VPI, VPI->getDebugLoc());
```

Perhaps clear with clang-format wrapping?

https://github.com/llvm/llvm-project/pull/205013