[llvm] [VPlan] Detect contiguous accesses in outer loop VPlan path. (PR #203790)
via llvm-commits
llvm-commits at lists.llvm.org
Sun Jun 14 13:40:28 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-llvm-transforms
Author: Florian Hahn (fhahn)
<details>
<summary>Changes</summary>
Use SCEV analysis on VPValue pointer operands to detect stride-1 memory accesses in the outer loop vectorization path. When a load or store address is an affine AddRec w.r.t. the vectorized outer loop with stride equal to the element size, mark it as consecutive so it generates a wide load/store instead of a gather/scatter.
For example, in the column-scaling pattern:
for (i = 0; i < N; i++) // outer: vectorized
for (j = 0; j < M; j++) // inner: uniform
A[i*M+j] *= scale[i];
The access scale[i] is now correctly identified as contiguous and generates a vector load instead of a masked gather.
---
Patch is 47.23 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/203790.diff
19 Files Affected:
- (modified) llvm/include/llvm/Analysis/LoopAccessAnalysis.h (+8)
- (modified) llvm/lib/Analysis/LoopAccessAnalysis.cpp (+4-3)
- (modified) llvm/lib/Transforms/Vectorize/LoopVectorize.cpp (+1-1)
- (modified) llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp (+26-6)
- (modified) llvm/lib/Transforms/Vectorize/VPlanTransforms.h (+6-4)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/outer_loop_prefer_scalable.ll (+5-3)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/outer_loop_test1_no_explicit_vect_width.ll (+4-2)
- (modified) llvm/test/Transforms/LoopVectorize/X86/outer_loop_test1_no_explicit_vect_width.ll (+3-2)
- (modified) llvm/test/Transforms/LoopVectorize/nested-outer-loop-vect.ll (+2-1)
- (modified) llvm/test/Transforms/LoopVectorize/outer-loop-inner-latch-successors.ll (+8-6)
- (modified) llvm/test/Transforms/LoopVectorize/outer-loop-wide-phis.ll (+4-2)
- (modified) llvm/test/Transforms/LoopVectorize/outer_loop_contiguous.ll (+4-2)
- (modified) llvm/test/Transforms/LoopVectorize/outer_loop_hcfg_construction.ll (+2-1)
- (modified) llvm/test/Transforms/LoopVectorize/outer_loop_scalable.ll (+6-4)
- (modified) llvm/test/Transforms/LoopVectorize/outer_loop_test1.ll (+2-1)
- (modified) llvm/test/Transforms/LoopVectorize/outer_loop_test2.ll (+7-4)
- (modified) llvm/test/Transforms/LoopVectorize/vplan-vectorize-inner-loop-reduction.ll (+43-24)
- (modified) llvm/test/Transforms/LoopVectorize/vplan-widen-call-instruction.ll (+7-4)
- (modified) llvm/test/Transforms/LoopVectorize/vplan-widen-select-instruction.ll (+10-6)
``````````diff
diff --git a/llvm/include/llvm/Analysis/LoopAccessAnalysis.h b/llvm/include/llvm/Analysis/LoopAccessAnalysis.h
index 6c9097c92a86d..908f972ac8d9a 100644
--- a/llvm/include/llvm/Analysis/LoopAccessAnalysis.h
+++ b/llvm/include/llvm/Analysis/LoopAccessAnalysis.h
@@ -877,6 +877,14 @@ replaceSymbolicStrideSCEV(PredicatedScalarEvolution &PSE,
const DenseMap<Value *, const SCEV *> &PtrToStride,
Value *Ptr);
+/// If \p AR is an affine AddRec for \p Lp with a constant step, return the
+/// step in units of \p AccessTy's allocation size. Returns std::nullopt if the
+/// step is not constant, does not divide the access size, or \p AccessTy is a
+/// scalable vector. \p Ptr is only used for debug output and may be null.
+LLVM_ABI std::optional<int64_t>
+getStrideFromAddRec(const SCEVAddRecExpr *AR, const Loop *Lp, Type *AccessTy,
+ Value *Ptr, PredicatedScalarEvolution &PSE);
+
/// If the pointer has a constant stride return it in units of the access type
/// size. If the pointer is loop-invariant, return 0. Otherwise return
/// std::nullopt.
diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
index a2c2b0ef88da4..8cf575c6ccf86 100644
--- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp
+++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
@@ -971,9 +971,10 @@ class AccessAnalysis {
/// Try to compute a constant stride for \p AR. Used by getPtrStride and
/// isNoWrap.
-static std::optional<int64_t>
-getStrideFromAddRec(const SCEVAddRecExpr *AR, const Loop *Lp, Type *AccessTy,
- Value *Ptr, PredicatedScalarEvolution &PSE) {
+std::optional<int64_t>
+llvm::getStrideFromAddRec(const SCEVAddRecExpr *AR, const Loop *Lp,
+ Type *AccessTy, Value *Ptr,
+ PredicatedScalarEvolution &PSE) {
if (isa<ScalableVectorType>(AccessTy)) {
LLVM_DEBUG(dbgs() << "LAA: Bad stride - Scalable object: " << *AccessTy
<< "\n");
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index e815dbcd96e54..cc6c2df9fd555 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -6628,7 +6628,7 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VPlanPtr Plan,
for (ElementCount VF : Range)
Plan->addVF(VF);
if (!RUN_VPLAN_PASS(VPlanTransforms::tryToConvertVPInstructionsToVPRecipes,
- *Plan, *TLI))
+ *Plan, *TLI, &PSE, OrigLoop))
return nullptr;
RUN_VPLAN_PASS(VPlanTransforms::optimizeInductionLiveOutUsers, *Plan, PSE,
/*FoldTail=*/false);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 1d661b823ffe6..3ac27274a6365 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -30,6 +30,7 @@
#include "llvm/Analysis/IVDescriptors.h"
#include "llvm/Analysis/InstSimplifyFolder.h"
#include "llvm/Analysis/Loads.h"
+#include "llvm/Analysis/LoopAccessAnalysis.h"
#include "llvm/Analysis/LoopInfo.h"
#include "llvm/Analysis/MemoryLocation.h"
#include "llvm/Analysis/ScalarEvolutionPatternMatch.h"
@@ -47,8 +48,23 @@ using namespace llvm;
using namespace VPlanPatternMatch;
using namespace SCEVPatternMatch;
+/// Check if the pointer operand \p Addr of a memory access is consecutive
+/// w.r.t. \p OuterLoop, i.e. it is an affine AddRec with unit stride in units
+/// of \p ElemTy.
+static bool isStride1Access(VPValue *Addr, Type *ElemTy,
+ PredicatedScalarEvolution &PSE, Loop *OuterLoop) {
+ const SCEV *AddrSCEV = vputils::getSCEVExprForVPValue(Addr, PSE, OuterLoop);
+ auto *AddRec = dyn_cast<SCEVAddRecExpr>(AddrSCEV);
+ if (!AddRec)
+ return false;
+
+ return getStrideFromAddRec(AddRec, OuterLoop, ElemTy, /*Ptr=*/nullptr, PSE) ==
+ 1;
+}
+
bool VPlanTransforms::tryToConvertVPInstructionsToVPRecipes(
- VPlan &Plan, const TargetLibraryInfo &TLI) {
+ VPlan &Plan, const TargetLibraryInfo &TLI, PredicatedScalarEvolution *PSE,
+ Loop *OuterLoop) {
ReversePostOrderTraversal<VPBlockDeepTraversalWrapper<VPBlockBase *>> RPOT(
Plan.getVectorLoopRegion());
@@ -77,14 +93,18 @@ bool VPlanTransforms::tryToConvertVPInstructionsToVPRecipes(
assert(!isa<PHINode>(Inst) && "phis should be handled above");
// Create VPWidenMemoryRecipe for loads and stores.
if (LoadInst *Load = dyn_cast<LoadInst>(Inst)) {
- NewRecipe = new VPWidenLoadRecipe(
- *Load, Ingredient.getOperand(0), nullptr /*Mask*/,
- false /*Consecutive*/, *VPI, Ingredient.getDebugLoc());
+ bool IsConsecutive = isStride1Access(
+ VPI->getOperand(0), VPI->getScalarType(), *PSE, OuterLoop);
+ NewRecipe = new VPWidenLoadRecipe(*Load, Ingredient.getOperand(0),
+ nullptr /*Mask*/, IsConsecutive,
+ *VPI, Ingredient.getDebugLoc());
} else if (StoreInst *Store = dyn_cast<StoreInst>(Inst)) {
+ bool IsConsecutive = isStride1Access(
+ VPI->getOperand(1), VPI->getOperand(0)->getScalarType(), *PSE,
+ OuterLoop);
NewRecipe = new VPWidenStoreRecipe(
*Store, Ingredient.getOperand(1), Ingredient.getOperand(0),
- nullptr /*Mask*/, false /*Consecutive*/, *VPI,
- Ingredient.getDebugLoc());
+ nullptr /*Mask*/, IsConsecutive, *VPI, Ingredient.getDebugLoc());
} else if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Inst)) {
NewRecipe = new VPWidenGEPRecipe(GEP->getSourceElementType(),
Ingredient.operands(), *VPI,
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index a8c92e6d034fe..0c2752fefb5b3 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -214,10 +214,12 @@ struct VPlanTransforms {
/// Replaces the VPInstructions in \p Plan with corresponding
/// widen recipes. Returns false if any VPInstructions could not be converted
- /// to a wide recipe if needed.
- LLVM_ABI_FOR_TEST static bool
- tryToConvertVPInstructionsToVPRecipes(VPlan &Plan,
- const TargetLibraryInfo &TLI);
+ /// to a wide recipe if needed. When \p PSE and \p OuterLoop are provided,
+ /// uses SCEV to detect contiguous memory accesses w.r.t. the outer loop
+ /// induction variable.
+ LLVM_ABI_FOR_TEST static bool tryToConvertVPInstructionsToVPRecipes(
+ VPlan &Plan, const TargetLibraryInfo &TLI,
+ PredicatedScalarEvolution *PSE = nullptr, Loop *OuterLoop = nullptr);
/// Try to legalize reductions with multiple in-loop uses. Currently only
/// strict and non-strict min/max reductions used by FindLastIV reductions are
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/outer_loop_prefer_scalable.ll b/llvm/test/Transforms/LoopVectorize/AArch64/outer_loop_prefer_scalable.ll
index b111eb26b286a..860baa9e6f01f 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/outer_loop_prefer_scalable.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/outer_loop_prefer_scalable.ll
@@ -25,11 +25,12 @@ define void @foo() {
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_LATCH:.*]] ]
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[TMP4]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_LATCH]] ]
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds [1024 x float], ptr @A, i64 0, <vscale x 4 x i64> [[VEC_IND]]
-; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x float> @llvm.masked.gather.nxv4f32.nxv4p0(<vscale x 4 x ptr> align 4 [[TMP5]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x float> poison)
+; CHECK-NEXT: [[TMP14:%.*]] = extractelement <vscale x 4 x ptr> [[TMP5]], i64 0
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x float>, ptr [[TMP14]], align 4
; CHECK-NEXT: br label %[[INNER_LOOP1:.*]]
; CHECK: [[INNER_LOOP1]]:
; CHECK-NEXT: [[TMP6:%.*]] = phi <vscale x 4 x i64> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[TMP10:%.*]], %[[INNER_LOOP1]] ]
-; CHECK-NEXT: [[TMP7:%.*]] = phi <vscale x 4 x float> [ [[WIDE_MASKED_GATHER]], %[[VECTOR_BODY]] ], [ [[TMP9:%.*]], %[[INNER_LOOP1]] ]
+; CHECK-NEXT: [[TMP7:%.*]] = phi <vscale x 4 x float> [ [[WIDE_LOAD]], %[[VECTOR_BODY]] ], [ [[TMP9:%.*]], %[[INNER_LOOP1]] ]
; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds [512 x float], ptr @B, i64 0, <vscale x 4 x i64> [[TMP6]]
; CHECK-NEXT: [[WIDE_MASKED_GATHER2:%.*]] = call <vscale x 4 x float> @llvm.masked.gather.nxv4f32.nxv4p0(<vscale x 4 x ptr> align 4 [[TMP8]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x float> poison)
; CHECK-NEXT: [[TMP9]] = fmul <vscale x 4 x float> [[TMP7]], [[WIDE_MASKED_GATHER2]]
@@ -38,7 +39,8 @@ define void @foo() {
; CHECK-NEXT: [[TMP12:%.*]] = extractelement <vscale x 4 x i1> [[TMP11]], i64 0
; CHECK-NEXT: br i1 [[TMP12]], label %[[VECTOR_LATCH]], label %[[INNER_LOOP1]]
; CHECK: [[VECTOR_LATCH]]:
-; CHECK-NEXT: call void @llvm.masked.scatter.nxv4f32.nxv4p0(<vscale x 4 x float> [[TMP9]], <vscale x 4 x ptr> align 4 [[TMP5]], <vscale x 4 x i1> splat (i1 true))
+; CHECK-NEXT: [[TMP15:%.*]] = extractelement <vscale x 4 x ptr> [[TMP5]], i64 0
+; CHECK-NEXT: store <vscale x 4 x float> [[TMP9]], ptr [[TMP15]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <vscale x 4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/outer_loop_test1_no_explicit_vect_width.ll b/llvm/test/Transforms/LoopVectorize/AArch64/outer_loop_test1_no_explicit_vect_width.ll
index 1ad20d7ed0655..d9fd4e4cb033b 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/outer_loop_test1_no_explicit_vect_width.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/outer_loop_test1_no_explicit_vect_width.ll
@@ -37,7 +37,8 @@ define void @foo_i32(i32 %n) {
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_LATCH]] ]
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds [8 x i32], ptr @arr2, i64 0, <4 x i64> [[VEC_IND]]
; CHECK-NEXT: [[TMP1:%.*]] = trunc <4 x i64> [[VEC_IND]] to <4 x i32>
-; CHECK-NEXT: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> [[TMP1]], <4 x ptr> align 4 [[TMP0]], <4 x i1> splat (i1 true))
+; CHECK-NEXT: [[TMP8:%.*]] = extractelement <4 x ptr> [[TMP0]], i64 0
+; CHECK-NEXT: store <4 x i32> [[TMP1]], ptr [[TMP8]], align 4
; CHECK-NEXT: [[TMP2:%.*]] = add nsw <4 x i32> [[TMP1]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: br label %[[FOR_BODY31:.*]]
; CHECK: [[FOR_BODY31]]:
@@ -100,7 +101,8 @@ define void @foo_i64(i64 %n) {
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_LATCH:.*]] ]
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_LATCH]] ]
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds [8 x i64], ptr @arrX, i64 0, <2 x i64> [[VEC_IND]]
-; CHECK-NEXT: call void @llvm.masked.scatter.v2i64.v2p0(<2 x i64> [[VEC_IND]], <2 x ptr> align 4 [[TMP0]], <2 x i1> splat (i1 true))
+; CHECK-NEXT: [[TMP7:%.*]] = extractelement <2 x ptr> [[TMP0]], i64 0
+; CHECK-NEXT: store <2 x i64> [[VEC_IND]], ptr [[TMP7]], align 4
; CHECK-NEXT: [[TMP1:%.*]] = add nsw <2 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: br label %[[FOR_BODY31:.*]]
; CHECK: [[FOR_BODY31]]:
diff --git a/llvm/test/Transforms/LoopVectorize/X86/outer_loop_test1_no_explicit_vect_width.ll b/llvm/test/Transforms/LoopVectorize/X86/outer_loop_test1_no_explicit_vect_width.ll
index d10449327d9c0..d41c83d8c959e 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/outer_loop_test1_no_explicit_vect_width.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/outer_loop_test1_no_explicit_vect_width.ll
@@ -36,7 +36,8 @@ define void @foo(i32 %n) {
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_LATCH]] ]
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds [8 x i32], ptr @arr2, i64 0, <4 x i64> [[VEC_IND]]
; CHECK-NEXT: [[TMP1:%.*]] = trunc <4 x i64> [[VEC_IND]] to <4 x i32>
-; CHECK-NEXT: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> [[TMP1]], <4 x ptr> align 4 [[TMP0]], <4 x i1> splat (i1 true))
+; CHECK-NEXT: [[TMP8:%.*]] = extractelement <4 x ptr> [[TMP0]], i64 0
+; CHECK-NEXT: store <4 x i32> [[TMP1]], ptr [[TMP8]], align 4
; CHECK-NEXT: [[TMP2:%.*]] = add nsw <4 x i32> [[TMP1]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: br label %[[FOR_BODY31:.*]]
; CHECK: [[FOR_BODY31]]:
@@ -66,7 +67,7 @@ define void @foo(i32 %n) {
; AVX-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT]], <8 x i32> poison, <8 x i32> zeroinitializer
; AVX-NEXT: br label %[[VECTOR_BODY:.*]]
; AVX: [[VECTOR_BODY]]:
-; AVX-NEXT: call void @llvm.masked.scatter.v8i32.v8p0(<8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>, <8 x ptr> align 4 getelementptr inbounds ([8 x i32], ptr @arr2, <8 x i64> zeroinitializer, <8 x i64> <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7>), <8 x i1> splat (i1 true))
+; AVX-NEXT: store <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>, ptr @arr2, align 4
; AVX-NEXT: [[TMP2:%.*]] = add nsw <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>, [[BROADCAST_SPLAT]]
; AVX-NEXT: br label %[[FOR_BODY31:.*]]
; AVX: [[FOR_BODY31]]:
diff --git a/llvm/test/Transforms/LoopVectorize/nested-outer-loop-vect.ll b/llvm/test/Transforms/LoopVectorize/nested-outer-loop-vect.ll
index d411783fa9df8..c6bb905988dba 100644
--- a/llvm/test/Transforms/LoopVectorize/nested-outer-loop-vect.ll
+++ b/llvm/test/Transforms/LoopVectorize/nested-outer-loop-vect.ll
@@ -21,7 +21,8 @@ define void @nested_outer_loop_vect(i32 %n, i64 %outer_tc) {
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_LATCH]] ]
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds [8 x i32], ptr @arr2, i64 0, <4 x i64> [[VEC_IND]]
; CHECK-NEXT: [[TMP1:%.*]] = trunc <4 x i64> [[VEC_IND]] to <4 x i32>
-; CHECK-NEXT: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> [[TMP1]], <4 x ptr> align 4 [[TMP0]], <4 x i1> splat (i1 true))
+; CHECK-NEXT: [[TMP9:%.*]] = extractelement <4 x ptr> [[TMP0]], i64 0
+; CHECK-NEXT: store <4 x i32> [[TMP1]], ptr [[TMP9]], align 4
; CHECK-NEXT: [[TMP2:%.*]] = add nsw <4 x i32> [[TMP1]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: br label %[[LOOP2_HEADER1:.*]]
; CHECK: [[LOOP2_HEADER1]]:
diff --git a/llvm/test/Transforms/LoopVectorize/outer-loop-inner-latch-successors.ll b/llvm/test/Transforms/LoopVectorize/outer-loop-inner-latch-successors.ll
index 960a6e6445ee1..ef588bfda0aaf 100644
--- a/llvm/test/Transforms/LoopVectorize/outer-loop-inner-latch-successors.ll
+++ b/llvm/test/Transforms/LoopVectorize/outer-loop-inner-latch-successors.ll
@@ -22,12 +22,13 @@ define void @inner_latch_header_first_successor(i64 %N, i32 %c, i64 %M) {
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_LATCH:.*]] ]
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_LATCH]] ]
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds [1024 x i64], ptr @A, i64 0, <4 x i64> [[VEC_IND]]
-; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> align 4 [[TMP0]], <4 x i1> splat (i1 true), <4 x i64> poison)
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x ptr> [[TMP0]], i64 0
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP5]], align 4
; CHECK-NEXT: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> [[BROADCAST_SPLAT]], <4 x ptr> align 4 [[TMP0]], <4 x i1> splat (i1 true))
; CHECK-NEXT: br label %[[INNER3:.*]]
; CHECK: [[INNER3]]:
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[TMP4:%.*]], %[[INNER3]] ]
-; CHECK-NEXT: [[VEC_PHI4:%.*]] = phi <4 x i64> [ [[WIDE_MASKED_GATHER]], %[[VECTOR_BODY]] ], [ [[TMP3:%.*]], %[[INNER3]] ]
+; CHECK-NEXT: [[VEC_PHI4:%.*]] = phi <4 x i64> [ [[WIDE_LOAD]], %[[VECTOR_BODY]] ], [ [[TMP3:%.*]], %[[INNER3]] ]
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds [1024 x i64], ptr @B, i64 0, <4 x i64> [[VEC_PHI]]
; CHECK-NEXT: [[WIDE_MASKED_GATHER5:%.*]] = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> align 4 [[TMP1]], <4 x i1> splat (i1 true), <4 x i64> poison)
; CHECK-NEXT: [[TMP2:%.*]] = add nsw <4 x i64> [[WIDE_MASKED_GATHER5]], [[VEC_PHI4]]
@@ -37,7 +38,7 @@ define void @inner_latch_header_first_successor(i64 %N, i32 %c, i64 %M) {
; CHECK-NEXT: [[TMP9:%.*]] = extractelement <4 x i1> [[TMP6]], i64 0
; CHECK-NEXT: br i1 [[TMP9]], label %[[VECTOR_LATCH]], label %[[INNER3]]
; CHECK: [[VECTOR_LATCH]]:
-; CHECK-NEXT: call void @llvm.masked.scatter.v4i64.v4p0(<4 x i64> [[TMP3]], <4 x ptr> align 4 [[TMP0]], <4 x i1> splat (i1 true))
+; CHECK-NEXT: store <4 x i64> [[TMP3]], ptr [[TMP5]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
@@ -123,12 +124,13 @@ define void @inner_latch_header_second_successor(i64 %N, i32 %c, i64 %M) {
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_LATCH:.*]] ]
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_LATCH]] ]
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds [1024 x i64], ptr @A, i64 0, <4 x i64> [[VEC_IND]]
-; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> align 4 [[TMP0]], <4 x i1> splat (i1 true), <4 x i64> poison)
+; CHECK-NEXT: [[TMP9:%.*]] = extractelement <4 x ptr> [[TMP0]], i64 0
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP9]], align 4
; CHECK-NEXT: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> [[BROADCAST_SPLAT]], <4 x ptr> align 4 [[TMP0]], <4 x i1> splat (i1 true))
; CHECK-NEXT: br label %[[INNER3:.*]]
; CHECK: [[INNER3]]:
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[TMP4:%.*]], %[[INNER3]] ]
-; CHECK-NEXT: [[VEC_PHI4:%.*]] = phi <4 x i64> [ [[WIDE_MASKED_GATHER]], %[[VECTOR_BODY]] ], [ [[TMP3:%.*]], %[[INNER3]] ]
+; CHECK-NEXT: [[VEC_PHI4:%.*]] = phi <4 x i64> [ [[WIDE_LOAD]], %[[VECTOR_BODY]] ], [ [[TMP3:%.*]], %[[INNER3]] ]
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds [1024 x i64], ptr @B, i64 0, <4 x i64> [[VEC_PHI]]
; CHECK-NEXT: [[WIDE_MASKED_GATHER5:%.*]] = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> align 4 [[TMP1]], <4 x i1> splat (i1 true), <4 x i64> poison)
; CHECK-NEXT: [[TMP2:%.*]] = add nsw <4 x i64> [[WIDE_MASKED_GATHER5]], [[VEC_PHI4]]
@@ -138,7 +140,7 @@ define void @inner_latch_header_second_successor(i64 %N, i32 %c, i64 %M) {
; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x i1> [[TMP5]], i64 0
; CHECK-NEXT: br i1 [[TMP6]], label %[[VECTOR_LATCH]], label %[[INNER3]]
; CHECK: [[VECTOR_LATCH]]:
-; CHECK-NEXT: call void @llvm.masked.scatter.v4i64.v4p0(<4 x i64> [[TMP3]], <4 x ptr> align 4 [[TMP0]], <4 x i1> splat (i1 true))
+; CHECK-NEXT: store <4 x i64> [[TMP3]], ptr [[TMP9]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
diff --git a/llvm/tes...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/203790
More information about the llvm-commits
mailing list