[llvm] [VPlan] Don't form VPInterleaveEVLRecipe for scalable gap-masked groups. (PR #213486)
via llvm-commits
llvm-commits at lists.llvm.org
Sat Aug 1 13:10:57 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-risc-v
Author: Florian Hahn (fhahn)
<details>
<summary>Changes</summary>
VPInterleaveEVLRecipe does not support masks for gaps. Bail out instead of creating an invalid VPInterleaveEVLRecipe. Fixes an assertion failure in the added test.
---
Full diff: https://github.com/llvm/llvm-project/pull/213486.diff
2 Files Affected:
- (modified) llvm/lib/Transforms/Vectorize/VPlanEVLTailFolding.cpp (+2-1)
- (added) llvm/test/Transforms/LoopVectorize/RISCV/interleave-group-with-gap-evl.ll (+80)
``````````diff
diff --git a/llvm/lib/Transforms/Vectorize/VPlanEVLTailFolding.cpp b/llvm/lib/Transforms/Vectorize/VPlanEVLTailFolding.cpp
index fab0765083d6f..faa7d01ae3c66 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanEVLTailFolding.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanEVLTailFolding.cpp
@@ -191,7 +191,8 @@ static VPRecipeBase *optimizeMaskToEVL(VPValue *HeaderMask,
return new VPReductionEVLRecipe(*Rdx, EVL, Mask);
if (auto *Interleave = dyn_cast<VPInterleaveRecipe>(&CurRecipe))
- if (Interleave->getMask() &&
+ // VPInterleaveEVLRecipe does not support masks for gaps
+ if (!Interleave->needsMaskForGaps() && Interleave->getMask() &&
match(Interleave->getMask(), m_RemoveMask(HeaderMask, Mask)))
return new VPInterleaveEVLRecipe(*Interleave, EVL, Mask);
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/interleave-group-with-gap-evl.ll b/llvm/test/Transforms/LoopVectorize/RISCV/interleave-group-with-gap-evl.ll
new file mode 100644
index 0000000000000..05ac69c951e87
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/interleave-group-with-gap-evl.ll
@@ -0,0 +1,80 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
+; RUN: opt -S -passes=loop-vectorize -mtriple riscv64 -mattr=+v \
+; RUN: -scalable-vectorization=on -force-vector-width=4 \
+; RUN: -epilogue-vectorization-force-VF=2 %s | FileCheck %s
+
+define i8 @interleave_gap_mask_scalable_evl(ptr %src, ptr %dst) {
+; CHECK-LABEL: define i8 @interleave_gap_mask_scalable_evl(
+; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
+; CHECK: [[VECTOR_MEMCHECK]]:
+; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 1005
+; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[SRC]], i64 1005
+; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[DST]], [[SCEVGEP1]]
+; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[SRC]], [[SCEVGEP]]
+; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP0:%.*]] = call <vscale x 4 x i32> @llvm.stepvector.nxv4i32()
+; CHECK-NEXT: [[TMP1:%.*]] = mul <vscale x 4 x i32> [[TMP0]], splat (i32 4)
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i32> [ [[TMP1]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[AVL:%.*]] = phi i32 [ 252, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = call i32 @llvm.experimental.get.vector.length.i32(i32 [[AVL]], i32 4, i1 true)
+; CHECK-NEXT: [[TMP3:%.*]] = shl i32 [[TMP2]], 2
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP3]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP4:%.*]] = sext <vscale x 4 x i32> [[VEC_IND]] to <vscale x 4 x i64>
+; CHECK-NEXT: [[TMP5:%.*]] = zext i32 [[INDEX]] to i64
+; CHECK-NEXT: [[TMP6:%.*]] = shl i64 [[TMP5]], 2
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP6]]
+; CHECK-NEXT: [[TMP8:%.*]] = call <vscale x 4 x i8> @llvm.experimental.vp.strided.load.nxv4i8.p0.i64(ptr align 1 [[TMP7]], i64 4, <vscale x 4 x i1> splat (i1 true), i32 [[TMP2]]), !alias.scope [[META0:![0-9]+]]
+; CHECK-NEXT: [[WIDE_GEP:%.*]] = getelementptr i8, ptr [[DST]], <vscale x 4 x i64> [[TMP4]]
+; CHECK-NEXT: call void @llvm.vp.scatter.nxv4i8.nxv4p0(<vscale x 4 x i8> zeroinitializer, <vscale x 4 x ptr> align 1 [[WIDE_GEP]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP2]]), !alias.scope [[META3:![0-9]+]], !noalias [[META0]]
+; CHECK-NEXT: [[CURRENT_ITERATION_NEXT]] = add nuw i32 [[TMP2]], [[INDEX]]
+; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i32 [[AVL]], [[TMP2]]
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add <vscale x 4 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i32 [[AVL_NEXT]], 0
+; CHECK-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP10:%.*]] = zext i32 [[TMP2]] to i64
+; CHECK-NEXT: [[TMP11:%.*]] = sub i64 [[TMP10]], 1
+; CHECK-NEXT: [[TMP12:%.*]] = extractelement <vscale x 4 x i8> [[TMP8]], i64 [[TMP11]]
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i32 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[IDXPROM:%.*]] = sext i32 [[IV]] to i64
+; CHECK-NEXT: [[GEP_SRC:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[IDXPROM]]
+; CHECK-NEXT: [[L:%.*]] = load i8, ptr [[GEP_SRC]], align 1
+; CHECK-NEXT: [[GEP_DST:%.*]] = getelementptr i8, ptr [[DST]], i64 [[IDXPROM]]
+; CHECK-NEXT: store i8 0, ptr [[GEP_DST]], align 1
+; CHECK-NEXT: [[IV_NEXT]] = add i32 [[IV]], 4
+; CHECK-NEXT: [[CMP:%.*]] = icmp ult i32 [[IV]], 1001
+; CHECK-NEXT: br i1 [[CMP]], label %[[LOOP]], label %[[EXIT]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[R:%.*]] = phi i8 [ [[L]], %[[LOOP]] ], [ [[TMP12]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: ret i8 [[R]]
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
+ %idxprom = sext i32 %iv to i64
+ %gep.src = getelementptr i8, ptr %src, i64 %idxprom
+ %l = load i8, ptr %gep.src, align 1
+ %gep.dst = getelementptr i8, ptr %dst, i64 %idxprom
+ store i8 0, ptr %gep.dst, align 1
+ %iv.next = add i32 %iv, 4
+ %cmp = icmp ult i32 %iv, 1001
+ br i1 %cmp, label %loop, label %exit
+
+exit:
+ %r = phi i8 [ %l, %loop ]
+ ret i8 %r
+}
``````````
</details>
https://github.com/llvm/llvm-project/pull/213486
More information about the llvm-commits
mailing list