[llvm] [VPlan] Widen reverse unit-stride accesses in makeMemOpWideningDesc (NFCI) (PR #208807)
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Thu Jul 16 12:49:06 PDT 2026
https://github.com/fhahn updated https://github.com/llvm/llvm-project/pull/208807
>From 1ed83a7a23c997e596f02d23bd0ccf65ccc9584e Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Sun, 5 Jul 2026 08:29:17 +0100
Subject: [PATCH 1/2] [VPlan] Widen reverse unit-stride accesses in
makeMemOpWideningDecisions (NFCI)
Update widenConsecutiveMemOps to also widen reversed consecutive
accesses (stride == -1), by creating a VPVectorEndPointerRecipe via
VPBuilder, load and reverse.
---
.../Transforms/Vectorize/VPlanTransforms.cpp | 49 ++++++++++++-------
.../AArch64/vplan-memory-op-decisions.ll | 10 ++--
2 files changed, 38 insertions(+), 21 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 1185b748599d1..3924b9731669d 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -7415,6 +7415,7 @@ void VPlanTransforms::makeMemOpWideningDecisions(VPlan &Plan, VFRange &Range,
}
// Widen unmasked unit-stride consecutive accesses, matching the legacy CM.
+ // Both forward (stride +1) and reverse (stride -1) accesses are handled.
VPlanTransforms::runPass(
"widenConsecutiveMemOps", ProcessSubset, Plan, [&](VPInstruction *VPI) {
Instruction *I = VPI->getUnderlyingInstr();
@@ -7425,27 +7426,39 @@ void VPlanTransforms::makeMemOpWideningDecisions(VPlan &Plan, VFRange &Range,
VPValue *Ptr = VPI->getOperand(!IsLoad);
Type *ScalarTy =
IsLoad ? VPI->getScalarType() : VPI->getOperand(0)->getScalarType();
- if (getConstantStride(Ptr, ScalarTy, CostCtx.PSE, CostCtx.L) != 1)
+ std::optional<int64_t> Stride =
+ getConstantStride(Ptr, ScalarTy, CostCtx.PSE, CostCtx.L);
+ if (Stride != 1 && Stride != -1)
return false;
+ bool Reverse = *Stride == -1;
+
+ VPBuilder Builder(VPI);
+ VPSingleDefRecipe *VectorPtr = Builder.createConsecutiveVectorPointer(
+ Ptr, ScalarTy, Reverse, VPI->getDebugLoc());
+
+ if (IsLoad) {
+ auto *LoadR = new VPWidenLoadRecipe(*cast<LoadInst>(I), VectorPtr,
+ /*Mask=*/nullptr,
+ /*Consecutive=*/true, *VPI,
+ VPI->getDebugLoc());
+ if (!Reverse)
+ return ReplaceWith(VPI, LoadR);
+ // Reverse the loaded values back into program order.
+ Builder.insert(LoadR);
+ auto *ReverseR = new VPInstruction(VPInstruction::Reverse, {LoadR},
+ {}, {}, VPI->getDebugLoc());
+ return ReplaceWith(VPI, ReverseR);
+ }
- Type *StrideTy =
- Plan.getDataLayout().getIndexType(Ptr->getScalarType());
- VPValue *StrideOne = Plan.getConstantInt(StrideTy, 1);
- auto *VectorPtr = new VPVectorPointerRecipe(
- Ptr, ScalarTy, StrideOne, vputils::getGEPFlagsForPtr(Ptr),
- VPI->getDebugLoc());
- VectorPtr->insertBefore(VPI);
- VPRecipeBase *WidenedR;
- if (IsLoad)
- WidenedR = new VPWidenLoadRecipe(*cast<LoadInst>(I), VectorPtr,
- /*Mask=*/nullptr,
- /*Consecutive=*/true, *VPI,
+ VPValue *StoredVal = VPI->getOperand(0);
+ if (Reverse)
+ // Reverse the stored values so they are written in descending order.
+ StoredVal = Builder.createNaryOp(VPInstruction::Reverse, {StoredVal},
VPI->getDebugLoc());
- else
- WidenedR = new VPWidenStoreRecipe(
- *cast<StoreInst>(I), VectorPtr, VPI->getOperand(0),
- /*Mask=*/nullptr, /*Consecutive=*/true, *VPI, VPI->getDebugLoc());
- return ReplaceWith(VPI, WidenedR);
+ auto *StoreR = new VPWidenStoreRecipe(
+ *cast<StoreInst>(I), VectorPtr, StoredVal,
+ /*Mask=*/nullptr, /*Consecutive=*/true, *VPI, VPI->getDebugLoc());
+ return ReplaceWith(VPI, StoreR);
});
VPlanTransforms::runPass("delegateMemOpWideningToLegacyCM", ProcessSubset,
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-memory-op-decisions.ll b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-memory-op-decisions.ll
index fe15ecf35a0ab..b396b09d187c7 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-memory-op-decisions.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-memory-op-decisions.ll
@@ -252,10 +252,14 @@ define void @reverse_load_store(ptr noalias %a, ptr noalias %b) {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<1023>, ir<-1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT ir<%gep.a> = getelementptr inbounds ir<%a>, ir<%iv>
-; CHECK-NEXT: EMIT-SCALAR ir<%lv> = load ir<%gep.a>
-; CHECK-NEXT: EMIT ir<%add> = add ir<%lv>, ir<1>
+; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = vector-end-pointer inbounds ir<%gep.a>, vp<[[VP0]]>
+; CHECK-NEXT: WIDEN ir<%lv> = load vp<[[VP4]]>
+; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = reverse ir<%lv>
+; CHECK-NEXT: EMIT ir<%add> = add vp<[[VP5]]>, ir<1>
; CHECK-NEXT: EMIT ir<%gep.b> = getelementptr inbounds ir<%b>, ir<%iv>
-; CHECK-NEXT: EMIT store ir<%add>, ir<%gep.b>
+; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = vector-end-pointer inbounds ir<%gep.b>, vp<[[VP0]]>
+; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = reverse ir<%add>
+; CHECK-NEXT: WIDEN store vp<[[VP6]]>, vp<[[VP7]]>
; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<-1>
; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<0>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
>From 0f9b3c891de4445ef779fcbbe79da082bee074b7 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Thu, 16 Jul 2026 20:48:12 +0100
Subject: [PATCH 2/2] !fixup address comments, thanks
---
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 9ab28071891ba..bdb282178323f 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -7456,7 +7456,7 @@ void VPlanTransforms::makeMemOpWideningDecisions(VPlan &Plan, VFRange &Range,
getConstantStride(Ptr, ScalarTy, CostCtx.PSE, CostCtx.L);
if (Stride != 1 && Stride != -1)
return false;
- bool Reverse = *Stride == -1;
+ bool Reverse = Stride == -1;
VPBuilder Builder(VPI);
VPSingleDefRecipe *VectorPtr = Builder.createConsecutiveVectorPointer(
@@ -7471,7 +7471,7 @@ void VPlanTransforms::makeMemOpWideningDecisions(VPlan &Plan, VFRange &Range,
return ReplaceWith(VPI, LoadR);
// Reverse the loaded values back into program order.
Builder.insert(LoadR);
- auto *ReverseR = new VPInstruction(VPInstruction::Reverse, {LoadR},
+ auto *ReverseR = new VPInstruction(VPInstruction::Reverse, LoadR,
{}, {}, VPI->getDebugLoc());
return ReplaceWith(VPI, ReverseR);
}
@@ -7479,7 +7479,7 @@ void VPlanTransforms::makeMemOpWideningDecisions(VPlan &Plan, VFRange &Range,
VPValue *StoredVal = VPI->getOperand(0);
if (Reverse)
// Reverse the stored values so they are written in descending order.
- StoredVal = Builder.createNaryOp(VPInstruction::Reverse, {StoredVal},
+ StoredVal = Builder.createNaryOp(VPInstruction::Reverse, StoredVal,
VPI->getDebugLoc());
auto *StoreR = new VPWidenStoreRecipe(
*cast<StoreInst>(I), VectorPtr, StoredVal,
More information about the llvm-commits
mailing list