[llvm] [VPlan] Don't classify irregular element types as contiguous. (PR #215121)
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Sun Aug 9 13:08:17 PDT 2026
https://github.com/fhahn updated https://github.com/llvm/llvm-project/pull/215121
>From cc5d9b649845240b2ca8d44d305463aab394db64 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Sun, 9 Aug 2026 08:17:26 +0100
Subject: [PATCH] [VPlan] Don't classify irregular element types as contiguous.
Before classifying an access as continuous, check if it has an irregular
type. If so, do not widen incorrectly. This matches the logic of the
inner loop code path.
---
.../Transforms/Vectorize/VPlanTransforms.cpp | 18 ++++++++++++------
.../LoopVectorize/outer_loop_contiguous.ll | 18 +++++-------------
2 files changed, 17 insertions(+), 19 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 412f120b0de1a..5f125aecb005a 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -49,6 +49,8 @@ using namespace SCEVPatternMatch;
static std::optional<int64_t> getConstantStride(VPValue *Addr, Type *AccessTy,
PredicatedScalarEvolution &PSE,
const Loop *L) {
+ assert(!hasIrregularType(AccessTy, L->getHeader()->getDataLayout()) &&
+ "should not try to widen irregular types");
const SCEV *AddrSCEV = vputils::getSCEVExprForVPValue(Addr, PSE, L);
auto *AddRec = dyn_cast<SCEVAddRecExpr>(AddrSCEV);
if (!AddRec)
@@ -61,6 +63,13 @@ bool VPlanTransforms::tryToConvertVPInstructionsToVPRecipes(
VPlan &Plan, const TargetLibraryInfo &TLI, PredicatedScalarEvolution &PSE,
Loop *OuterLoop) {
+ // Returns true if the access of \p AccessTy at \p Addr can be widened to a
+ // consecutive vector access.
+ auto IsConsecutiveAccess = [&](VPValue *Addr, Type *AccessTy) {
+ return !hasIrregularType(AccessTy, Plan.getDataLayout()) &&
+ getConstantStride(Addr, AccessTy, PSE, OuterLoop) == 1;
+ };
+
ReversePostOrderTraversal<VPBlockDeepTraversalWrapper<VPBlockBase *>> RPOT(
Plan.getVectorLoopRegion());
for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(RPOT)) {
@@ -94,16 +103,13 @@ bool VPlanTransforms::tryToConvertVPInstructionsToVPRecipes(
// Create VPWidenMemoryRecipe for loads and stores.
if (LoadInst *Load = dyn_cast<LoadInst>(Inst)) {
bool IsConsecutive =
- getConstantStride(VPI->getOperand(0), VPI->getScalarType(), PSE,
- OuterLoop) == 1;
+ IsConsecutiveAccess(VPI->getOperand(0), VPI->getScalarType());
NewRecipe = new VPWidenLoadRecipe(*Load, Ingredient.getOperand(0),
nullptr /*Mask*/, IsConsecutive,
*VPI, Ingredient.getDebugLoc());
} else if (StoreInst *Store = dyn_cast<StoreInst>(Inst)) {
- bool IsConsecutive =
- getConstantStride(VPI->getOperand(1),
- VPI->getOperand(0)->getScalarType(), PSE,
- OuterLoop) == 1;
+ bool IsConsecutive = IsConsecutiveAccess(
+ VPI->getOperand(1), VPI->getOperand(0)->getScalarType());
NewRecipe = new VPWidenStoreRecipe(
*Store, Ingredient.getOperand(1), Ingredient.getOperand(0),
nullptr /*Mask*/, IsConsecutive, *VPI, Ingredient.getDebugLoc());
diff --git a/llvm/test/Transforms/LoopVectorize/outer_loop_contiguous.ll b/llvm/test/Transforms/LoopVectorize/outer_loop_contiguous.ll
index 56ac3f6fe523c..187a1f0f89ab1 100644
--- a/llvm/test/Transforms/LoopVectorize/outer_loop_contiguous.ll
+++ b/llvm/test/Transforms/LoopVectorize/outer_loop_contiguous.ll
@@ -247,9 +247,7 @@ exit:
;
; flags[i] has stride 1 in units of i1, but i1 is bit-packed in vectors: 4
; consecutive i1 scalars span 4 bytes, while a <4 x i1> access covers a single
-; byte.
-; FIXME: The accesses are widened to a packed <4 x i1> load and store, which
-; cover the wrong bytes. They should remain a gather and scatter.
+; byte. Should remain a gather and scatter.
define void @stride1_i1_load_store(ptr noalias %A, ptr noalias %flags, i64 %N, i64 %M) {
; CHECK-LABEL: define void @stride1_i1_load_store(
; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[FLAGS:%.*]], i64 [[N:%.*]], i64 [[M:%.*]]) {
@@ -266,8 +264,7 @@ define void @stride1_i1_load_store(ptr noalias %A, ptr noalias %flags, i64 %N, i
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[OUTER_LATCH4:.*]] ]
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[OUTER_LATCH4]] ]
; CHECK-NEXT: [[WIDE_GEP:%.*]] = getelementptr inbounds i1, ptr [[FLAGS]], <4 x i64> [[VEC_IND]]
-; CHECK-NEXT: [[TMP1:%.*]] = extractelement <4 x ptr> [[WIDE_GEP]], i64 0
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i1>, ptr [[TMP1]], align 1
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = call <4 x i1> @llvm.masked.gather.v4i1.v4p0(<4 x ptr> align 1 [[WIDE_GEP]], <4 x i1> splat (i1 true), <4 x i1> poison)
; CHECK-NEXT: [[TMP2:%.*]] = mul nsw <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP3:%.*]] = zext <4 x i1> [[WIDE_LOAD]] to <4 x i8>
; CHECK-NEXT: br label %[[INNER_BODY1:.*]]
@@ -282,8 +279,7 @@ define void @stride1_i1_load_store(ptr noalias %A, ptr noalias %flags, i64 %N, i
; CHECK-NEXT: br i1 [[TMP7]], label %[[OUTER_LATCH4]], label %[[INNER_BODY1]]
; CHECK: [[OUTER_LATCH4]]:
; CHECK-NEXT: [[TMP8:%.*]] = xor <4 x i1> [[WIDE_LOAD]], splat (i1 true)
-; CHECK-NEXT: [[TMP9:%.*]] = extractelement <4 x ptr> [[WIDE_GEP]], i64 0
-; CHECK-NEXT: store <4 x i1> [[TMP8]], ptr [[TMP9]], align 1
+; CHECK-NEXT: call void @llvm.masked.scatter.v4i1.v4p0(<4 x i1> [[TMP8]], <4 x ptr> align 1 [[WIDE_GEP]], <4 x i1> splat (i1 true))
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
@@ -327,8 +323,6 @@ exit:
; --- stride-1 i4 load and store (NOT contiguous) ---
; Same as above with i4, which is also bit-packed in vectors: 4 consecutive i4
; scalars span 4 bytes, while a <4 x i4> access covers 2 bytes.
-; FIXME: Same as above, the accesses are widened to a packed <4 x i4> load and
-; store.
define void @stride1_i4_load_store(ptr noalias %A, ptr noalias %vals, i64 %N, i64 %M) {
; CHECK-LABEL: define void @stride1_i4_load_store(
; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[VALS:%.*]], i64 [[N:%.*]], i64 [[M:%.*]]) {
@@ -345,8 +339,7 @@ define void @stride1_i4_load_store(ptr noalias %A, ptr noalias %vals, i64 %N, i6
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[OUTER_LATCH4:.*]] ]
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[OUTER_LATCH4]] ]
; CHECK-NEXT: [[WIDE_GEP:%.*]] = getelementptr inbounds i4, ptr [[VALS]], <4 x i64> [[VEC_IND]]
-; CHECK-NEXT: [[TMP1:%.*]] = extractelement <4 x ptr> [[WIDE_GEP]], i64 0
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i4>, ptr [[TMP1]], align 1
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = call <4 x i4> @llvm.masked.gather.v4i4.v4p0(<4 x ptr> align 1 [[WIDE_GEP]], <4 x i1> splat (i1 true), <4 x i4> poison)
; CHECK-NEXT: [[TMP2:%.*]] = mul nsw <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP3:%.*]] = zext <4 x i4> [[WIDE_LOAD]] to <4 x i8>
; CHECK-NEXT: br label %[[INNER_BODY1:.*]]
@@ -361,8 +354,7 @@ define void @stride1_i4_load_store(ptr noalias %A, ptr noalias %vals, i64 %N, i6
; CHECK-NEXT: br i1 [[TMP7]], label %[[OUTER_LATCH4]], label %[[INNER_BODY1]]
; CHECK: [[OUTER_LATCH4]]:
; CHECK-NEXT: [[TMP8:%.*]] = add <4 x i4> [[WIDE_LOAD]], splat (i4 1)
-; CHECK-NEXT: [[TMP9:%.*]] = extractelement <4 x ptr> [[WIDE_GEP]], i64 0
-; CHECK-NEXT: store <4 x i4> [[TMP8]], ptr [[TMP9]], align 1
+; CHECK-NEXT: call void @llvm.masked.scatter.v4i4.v4p0(<4 x i4> [[TMP8]], <4 x ptr> align 1 [[WIDE_GEP]], <4 x i1> splat (i1 true))
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
More information about the llvm-commits
mailing list