[llvm] [LV] Add test showing lack of gather/scatter can prevent if-convert (PR #196947)
Gaëtan Bossu via llvm-commits
llvm-commits at lists.llvm.org
Wed May 13 03:51:36 PDT 2026
https://github.com/gbossu updated https://github.com/llvm/llvm-project/pull/196947
>From 75813d0fa9393e727cd5cefba0778325befbb875 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ga=C3=ABtan=20Bossu?= <gaetan.bossu at arm.com>
Date: Wed, 29 Apr 2026 15:45:20 +0000
Subject: [PATCH 1/2] [LV] Add test showing lack of gather/scatter can prevent
if-convert
This introduces a new force-target-supports-gather-scatter-ops CLI
option for testing, as well a new isLegalMaskedLoadOrStore() helper.
---
.../Vectorize/LoopVectorizationPlanner.cpp | 8 +-
.../LoopVectorize/if-conversion-scalable.ll | 120 ++++++++++++++++++
2 files changed, 127 insertions(+), 1 deletion(-)
create mode 100644 llvm/test/Transforms/LoopVectorize/if-conversion-scalable.ll
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.cpp
index 33e3187d2d336..c6997a6c8e717 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.cpp
@@ -61,6 +61,11 @@ static cl::opt<bool> ForceTargetSupportsMaskedMemoryOps(
cl::desc("Assume the target supports masked memory operations (used for "
"testing)."));
+static cl::opt<bool> ForceTargetSupportsGatherScatterOps(
+ "force-target-supports-gather-scatter-ops", cl::init(false), cl::Hidden,
+ cl::desc("Assume the target supports gather/scatter operations (used for "
+ "testing)."));
+
bool VFSelectionContext::isLegalMaskedLoadOrStore(Instruction *I,
ElementCount VF) const {
assert(isa<LoadInst>(I) || isa<StoreInst>(I));
@@ -83,7 +88,8 @@ bool VFSelectionContext::isLegalGatherOrScatter(Value *V,
Align Align = getLoadStoreAlignment(V);
if (VF.isVector())
Ty = VectorType::get(Ty, VF);
- return (LI && TTI.isLegalMaskedGather(Ty, Align)) ||
+ return ForceTargetSupportsGatherScatterOps ||
+ (LI && TTI.isLegalMaskedGather(Ty, Align)) ||
(SI && TTI.isLegalMaskedScatter(Ty, Align));
}
diff --git a/llvm/test/Transforms/LoopVectorize/if-conversion-scalable.ll b/llvm/test/Transforms/LoopVectorize/if-conversion-scalable.ll
new file mode 100644
index 0000000000000..1e0d7eafc9096
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/if-conversion-scalable.ll
@@ -0,0 +1,120 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter-out-after "^scalar.ph:"
+
+; RUN: opt -passes=loop-vectorize -force-vector-interleave=1 -force-vector-width=8 -scalable-vectorization=on \
+; RUN: -force-target-supports-masked-memory-ops -force-target-supports-scalable-vectors \
+; RUN: -enable-interleaved-mem-accesses -enable-masked-interleaved-mem-accesses -force-target-supports-gather-scatter-ops \
+; RUN: -S < %s | FileCheck %s --check-prefix=CHECK-GATHER-ENABLED
+; RUN: opt -passes=loop-vectorize -force-vector-interleave=1 -force-vector-width=8 -scalable-vectorization=on \
+; RUN: -force-target-supports-masked-memory-ops -force-target-supports-scalable-vectors \
+; RUN: -enable-interleaved-mem-accesses -enable-masked-interleaved-mem-accesses \
+; RUN: -S < %s | FileCheck %s --check-prefix=CHECK-GATHER-DISABLED
+
+; Test whether the legality of gather/scatter has an impact on if-conversion
+; for scalable VFs. In either case, the example below should be vectorized with
+; VF = vscale x 8 and a masked deinterleaved load.
+
+define void @conv_interleaved_loads_load(ptr noalias nocapture noundef writeonly %a, ptr nocapture noundef readonly %b, ptr nocapture noundef readonly %c, ptr nocapture noundef readonly %flags, i64 %cond) {
+; CHECK-GATHER-ENABLED-LABEL: @conv_interleaved_loads_load(
+; CHECK-GATHER-ENABLED-NEXT: entry:
+; CHECK-GATHER-ENABLED-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-GATHER-ENABLED-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
+; CHECK-GATHER-ENABLED-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
+; CHECK-GATHER-ENABLED-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-GATHER-ENABLED: vector.ph:
+; CHECK-GATHER-ENABLED-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-GATHER-ENABLED-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 3
+; CHECK-GATHER-ENABLED-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
+; CHECK-GATHER-ENABLED-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; CHECK-GATHER-ENABLED-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 8 x i64> poison, i64 [[COND:%.*]], i64 0
+; CHECK-GATHER-ENABLED-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 8 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer
+; CHECK-GATHER-ENABLED-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK-GATHER-ENABLED: vector.body:
+; CHECK-GATHER-ENABLED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-GATHER-ENABLED-NEXT: [[TMP4:%.*]] = getelementptr inbounds i16, ptr [[B:%.*]], i64 [[INDEX]]
+; CHECK-GATHER-ENABLED-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>, ptr [[TMP4]], align 2
+; CHECK-GATHER-ENABLED-NEXT: [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[FLAGS:%.*]], i64 [[INDEX]]
+; CHECK-GATHER-ENABLED-NEXT: [[WIDE_LOAD1:%.*]] = load <vscale x 8 x i64>, ptr [[TMP5]], align 8
+; CHECK-GATHER-ENABLED-NEXT: [[TMP6:%.*]] = icmp ne <vscale x 8 x i64> [[WIDE_LOAD1]], [[BROADCAST_SPLAT]]
+; CHECK-GATHER-ENABLED-NEXT: [[TMP7:%.*]] = getelementptr [2 x i16], ptr [[C:%.*]], i64 [[INDEX]], i64 0
+; CHECK-GATHER-ENABLED-NEXT: [[INTERLEAVED_MASK:%.*]] = call <vscale x 16 x i1> @llvm.vector.interleave2.nxv16i1(<vscale x 8 x i1> [[TMP6]], <vscale x 8 x i1> [[TMP6]])
+; CHECK-GATHER-ENABLED-NEXT: [[WIDE_MASKED_VEC:%.*]] = call <vscale x 16 x i16> @llvm.masked.load.nxv16i16.p0(ptr align 2 [[TMP7]], <vscale x 16 x i1> [[INTERLEAVED_MASK]], <vscale x 16 x i16> poison)
+; CHECK-GATHER-ENABLED-NEXT: [[STRIDED_VEC:%.*]] = call { <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.vector.deinterleave2.nxv16i16(<vscale x 16 x i16> [[WIDE_MASKED_VEC]])
+; CHECK-GATHER-ENABLED-NEXT: [[TMP8:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16> } [[STRIDED_VEC]], 0
+; CHECK-GATHER-ENABLED-NEXT: [[TMP9:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16> } [[STRIDED_VEC]], 1
+; CHECK-GATHER-ENABLED-NEXT: [[TMP10:%.*]] = add <vscale x 8 x i16> [[TMP8]], [[TMP9]]
+; CHECK-GATHER-ENABLED-NEXT: [[PREDPHI:%.*]] = select <vscale x 8 x i1> [[TMP6]], <vscale x 8 x i16> [[TMP10]], <vscale x 8 x i16> zeroinitializer
+; CHECK-GATHER-ENABLED-NEXT: [[TMP11:%.*]] = add <vscale x 8 x i16> [[WIDE_LOAD]], [[PREDPHI]]
+; CHECK-GATHER-ENABLED-NEXT: [[TMP12:%.*]] = getelementptr inbounds i16, ptr [[A:%.*]], i64 [[INDEX]]
+; CHECK-GATHER-ENABLED-NEXT: store <vscale x 8 x i16> [[TMP11]], ptr [[TMP12]], align 2
+; CHECK-GATHER-ENABLED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; CHECK-GATHER-ENABLED-NEXT: [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-GATHER-ENABLED-NEXT: br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-GATHER-ENABLED: middle.block:
+; CHECK-GATHER-ENABLED-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
+; CHECK-GATHER-ENABLED-NEXT: br i1 [[CMP_N]], label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]
+; CHECK-GATHER-ENABLED: scalar.ph:
+;
+; CHECK-GATHER-DISABLED-LABEL: @conv_interleaved_loads_load(
+; CHECK-GATHER-DISABLED-NEXT: entry:
+; CHECK-GATHER-DISABLED-NEXT: br label [[VECTOR_PH:%.*]]
+; CHECK-GATHER-DISABLED: vector.ph:
+; CHECK-GATHER-DISABLED-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i64> poison, i64 [[COND:%.*]], i64 0
+; CHECK-GATHER-DISABLED-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i64> [[BROADCAST_SPLATINSERT]], <8 x i64> poison, <8 x i32> zeroinitializer
+; CHECK-GATHER-DISABLED-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK-GATHER-DISABLED: vector.body:
+; CHECK-GATHER-DISABLED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-GATHER-DISABLED-NEXT: [[TMP0:%.*]] = getelementptr inbounds i16, ptr [[B:%.*]], i64 [[INDEX]]
+; CHECK-GATHER-DISABLED-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i16>, ptr [[TMP0]], align 2
+; CHECK-GATHER-DISABLED-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[FLAGS:%.*]], i64 [[INDEX]]
+; CHECK-GATHER-DISABLED-NEXT: [[WIDE_LOAD1:%.*]] = load <8 x i64>, ptr [[TMP1]], align 8
+; CHECK-GATHER-DISABLED-NEXT: [[TMP2:%.*]] = icmp ne <8 x i64> [[WIDE_LOAD1]], [[BROADCAST_SPLAT]]
+; CHECK-GATHER-DISABLED-NEXT: [[TMP3:%.*]] = getelementptr [2 x i16], ptr [[C:%.*]], i64 [[INDEX]], i64 0
+; CHECK-GATHER-DISABLED-NEXT: [[INTERLEAVED_MASK:%.*]] = shufflevector <8 x i1> [[TMP2]], <8 x i1> poison, <16 x i32> <i32 0, i32 0, i32 1, i32 1, i32 2, i32 2, i32 3, i32 3, i32 4, i32 4, i32 5, i32 5, i32 6, i32 6, i32 7, i32 7>
+; CHECK-GATHER-DISABLED-NEXT: [[WIDE_MASKED_VEC:%.*]] = call <16 x i16> @llvm.masked.load.v16i16.p0(ptr align 2 [[TMP3]], <16 x i1> [[INTERLEAVED_MASK]], <16 x i16> poison)
+; CHECK-GATHER-DISABLED-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <16 x i16> [[WIDE_MASKED_VEC]], <16 x i16> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
+; CHECK-GATHER-DISABLED-NEXT: [[STRIDED_VEC2:%.*]] = shufflevector <16 x i16> [[WIDE_MASKED_VEC]], <16 x i16> poison, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
+; CHECK-GATHER-DISABLED-NEXT: [[TMP4:%.*]] = add <8 x i16> [[STRIDED_VEC]], [[STRIDED_VEC2]]
+; CHECK-GATHER-DISABLED-NEXT: [[PREDPHI:%.*]] = select <8 x i1> [[TMP2]], <8 x i16> [[TMP4]], <8 x i16> zeroinitializer
+; CHECK-GATHER-DISABLED-NEXT: [[TMP5:%.*]] = add <8 x i16> [[WIDE_LOAD]], [[PREDPHI]]
+; CHECK-GATHER-DISABLED-NEXT: [[TMP6:%.*]] = getelementptr inbounds i16, ptr [[A:%.*]], i64 [[INDEX]]
+; CHECK-GATHER-DISABLED-NEXT: store <8 x i16> [[TMP5]], ptr [[TMP6]], align 2
+; CHECK-GATHER-DISABLED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; CHECK-GATHER-DISABLED-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-GATHER-DISABLED-NEXT: br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-GATHER-DISABLED: middle.block:
+; CHECK-GATHER-DISABLED-NEXT: br label [[FOR_COND_CLEANUP:%.*]]
+; CHECK-GATHER-DISABLED: for.cond.cleanup:
+; CHECK-GATHER-DISABLED-NEXT: ret void
+;
+entry:
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.body
+ ret void
+
+for.body: ; preds = %entry, %for.inc
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.inc ]
+ %arrayidx0 = getelementptr inbounds i16, ptr %b, i64 %indvars.iv
+ %0 = load i16, ptr %arrayidx0, align 2
+ %flagidx = getelementptr inbounds i64, ptr %flags, i64 %indvars.iv
+ %scl.val = load i64, ptr %flagidx, align 8
+ %do.access = icmp ne i64 %scl.val, %cond
+ br i1 %do.access, label %if.then, label %for.inc
+
+if.then:
+ %arrayidx1 = getelementptr inbounds [2 x i16], ptr %c, i64 %indvars.iv, i64 0
+ %pred.1 = load i16, ptr %arrayidx1, align 2
+ %arrayidx2 = getelementptr inbounds [2 x i16], ptr %c, i64 %indvars.iv, i64 1
+ %pred.2 = load i16, ptr %arrayidx2, align 2
+ %pred.sum = add i16 %pred.1, %pred.2
+ br label %for.inc
+
+for.inc:
+ %1 = phi i16 [ zeroinitializer, %for.body ], [ %pred.sum, %if.then ]
+ %result = add i16 %0, %1
+ %arrayidx3 = getelementptr inbounds i16, ptr %a, i64 %indvars.iv
+ store i16 %result, ptr %arrayidx3, align 2
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1024
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
>From ad352201bc5b682e480a9497bd9ce3cebf258d1f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ga=C3=ABtan=20Bossu?= <gaetan.bossu at arm.com>
Date: Wed, 13 May 2026 10:46:50 +0000
Subject: [PATCH 2/2] Add small test for
-force-target-supports-gather-scatter-ops
---
.../LoopVectorize/gather-scatter.ll | 48 +++++++++++++++++++
1 file changed, 48 insertions(+)
create mode 100644 llvm/test/Transforms/LoopVectorize/gather-scatter.ll
diff --git a/llvm/test/Transforms/LoopVectorize/gather-scatter.ll b/llvm/test/Transforms/LoopVectorize/gather-scatter.ll
new file mode 100644
index 0000000000000..c556990a47814
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/gather-scatter.ll
@@ -0,0 +1,48 @@
+; RUN: opt -passes=loop-vectorize -force-vector-width=4 -force-target-supports-gather-scatter-ops=true \
+; RUN: -S < %s | FileCheck %s --check-prefixes=CHECK,GATHER-SCATTER-ENABLED
+; RUN: opt -passes=loop-vectorize -force-vector-width=4 -force-target-supports-gather-scatter-ops=false \
+; RUN: -S < %s | FileCheck %s --check-prefixes=CHECK,GATHER-SCATTER-DISABLED
+
+; Generic test using for the force-target-supports-gather-scatter-ops option.
+
+define void @gather_i32(ptr noalias nocapture readonly %a, ptr noalias nocapture readonly %b, ptr noalias nocapture %c, i64 %n) {
+; CHECK-LABEL: define void @gather_i32(
+; GATHER-SCATTER-ENABLED: call <4 x float> @llvm.masked.gather.v4f32
+; GATHER-SCATTER-DISABLED-NOT: @llvm.masked.gather
+entry:
+ br label %for.body
+for.body:
+ %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+ %arrayidx = getelementptr inbounds i64, ptr %b, i64 %indvars.iv
+ %0 = load i64, ptr %arrayidx, align 8
+ %arrayidx3 = getelementptr inbounds float, ptr %a, i64 %0
+ %1 = load float, ptr %arrayidx3, align 4
+ %arrayidx5 = getelementptr inbounds float, ptr %c, i64 %indvars.iv
+ store float %1, ptr %arrayidx5, align 4
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, %n
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+for.cond.cleanup:
+ ret void
+}
+
+define void @scatter_i32(ptr noalias nocapture readonly %a, ptr noalias nocapture readonly %b, ptr noalias nocapture %c, i64 %n) {
+; CHECK-LABEL: define void @scatter_i32(
+; GATHER-SCATTER-ENABLED: call void @llvm.masked.scatter.v4f32
+; GATHER-SCATTER-DISABLED-NOT: @llvm.masked.scatter
+entry:
+ br label %for.body
+for.body:
+ %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+ %arrayidx = getelementptr inbounds i64, ptr %b, i64 %indvars.iv
+ %0 = load i64, ptr %arrayidx, align 8
+ %arrayidx3 = getelementptr inbounds float, ptr %a, i64 %indvars.iv
+ %1 = load float, ptr %arrayidx3, align 4
+ %arrayidx5 = getelementptr inbounds float, ptr %c, i64 %0
+ store float %1, ptr %arrayidx5, align 4
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, %n
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+for.cond.cleanup:
+ ret void
+}
More information about the llvm-commits
mailing list