[llvm] [VPlan] Support different loop-invariant ops in narrowIG. (PR #203785)
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Sun Jun 14 12:20:49 PDT 2026
https://github.com/fhahn created https://github.com/llvm/llvm-project/pull/203785
Generalize the distinct per-field operand narrowing in
narrowInterleaveGroups from plain live-ins to any value defined outside
the vector loop region.
Depends on https://github.com/llvm/llvm-project/pull/203778 (included in
PR)
>From e0a411446dacbd5a91de9d2065e1398c73835714 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Sat, 13 Jun 2026 09:41:42 +0200
Subject: [PATCH 1/2] [VPlan] Narrow interleave groups with distinct live-in
operands.
Extend narrowInterleaveGroups so bundles with live-ins can be narrowed
by using BuildVector for the operands.
This only applies to fixed VFs: for scalable VFs the number of original
iterations processed by the narrowed plan depends on vscale, so a fixed
per-field vector cannot be built.
---
.../Transforms/Vectorize/VPlanTransforms.cpp | 46 ++++--
...interleave-to-widen-memory-constant-ops.ll | 92 ++++--------
...e-to-widen-memory-with-wide-ops-chained.ll | 18 +--
.../RISCV/interleaved-accesses.ll | 19 +--
.../narrow-interleave-groups-scalable-vf.ll | 133 ++++++++++++++++++
5 files changed, 209 insertions(+), 99 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 114ea67c23221..a030b0d490163 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -5561,8 +5561,14 @@ static bool canNarrowLoad(VPSingleDefRecipe *WideMember0, unsigned OpIdx,
VPValue *OpV, unsigned Idx, bool IsScalable) {
VPValue *Member0Op = WideMember0->getOperand(OpIdx);
VPRecipeBase *Member0OpR = Member0Op->getDefiningRecipe();
- if (!Member0OpR)
- return Member0Op == OpV;
+ if (!Member0OpR) {
+ // Member0's operand is a uniform live-in, broadcast across all fields.
+ if (Member0Op == OpV)
+ return true;
+ // Otherwise distinct per-field live-ins are assembled into a BuildVector.
+ return !IsScalable && !OpV->getDefiningRecipe() &&
+ OpV->getScalarType() == Member0Op->getScalarType();
+ }
if (auto *W = dyn_cast<VPWidenLoadRecipe>(Member0OpR))
// For scalable VFs, the narrowed plan processes vscale iterations at once,
// so a shared wide load cannot be narrowed to a uniform scalar; bail out.
@@ -5661,15 +5667,33 @@ static bool isAlreadyNarrow(VPValue *VPV) {
// Convert the wide recipes defining the VPValues in \p Members feeding an
// interleave group to a single narrow variant. The first member is reused as
-// the narrowed recipe.
-static VPValue *
-narrowInterleaveGroupOp(ArrayRef<VPValue *> Members,
- SmallPtrSetImpl<VPValue *> &NarrowedOps) {
+// the narrowed recipe. Newly created loop-invariant recipes (e.g. a BuildVector
+// for distinct live-ins) are inserted into \p Preheader.
+static VPValue *narrowInterleaveGroupOp(ArrayRef<VPValue *> Members,
+ SmallPtrSetImpl<VPValue *> &NarrowedOps,
+ VPBasicBlock *Preheader) {
VPValue *V = Members.front();
auto *R = V->getDefiningRecipe();
- if (!R || NarrowedOps.contains(V))
+ if (NarrowedOps.contains(V))
return V;
+ if (!R) {
+ // V is a live-in. If all members are the same live-in, reuse it directly.
+ if (all_of(Members, equal_to(V)))
+ return V;
+ // Otherwise assemble the distinct live-ins into a BuildVector.
+ assert(all_of(Members,
+ [V](VPValue *M) {
+ return !M->getDefiningRecipe() &&
+ M->getScalarType() == V->getScalarType();
+ }) &&
+ "expected distinct live-ins of matching scalar type");
+ auto *BV = new VPInstruction(VPInstruction::BuildVector, Members);
+ Preheader->appendRecipe(BV);
+ NarrowedOps.insert(BV);
+ return BV;
+ }
+
if (isAlreadyNarrow(V))
return V;
@@ -5681,7 +5705,8 @@ narrowInterleaveGroupOp(ArrayRef<VPValue *> Members,
SmallVector<VPValue *> OpsI;
for (VPValue *Member : Members)
OpsI.push_back(Member->getDefiningRecipe()->getOperand(Idx));
- WideMember0->setOperand(Idx, narrowInterleaveGroupOp(OpsI, NarrowedOps));
+ WideMember0->setOperand(
+ Idx, narrowInterleaveGroupOp(OpsI, NarrowedOps, Preheader));
}
return V;
}
@@ -5846,10 +5871,11 @@ VPlanTransforms::narrowInterleaveGroups(VPlan &Plan,
// Convert InterleaveGroup \p R to a single VPWidenLoadRecipe.
SmallPtrSet<VPValue *, 4> NarrowedOps;
+ VPBasicBlock *Preheader = Plan.getVectorPreheader();
// Narrow operation tree rooted at store groups.
for (auto *StoreGroup : StoreGroups) {
- VPValue *Res =
- narrowInterleaveGroupOp(StoreGroup->getStoredValues(), NarrowedOps);
+ VPValue *Res = narrowInterleaveGroupOp(StoreGroup->getStoredValues(),
+ NarrowedOps, Preheader);
auto *SI =
cast<StoreInst>(StoreGroup->getInterleaveGroup()->getInsertPos());
auto *S = new VPWidenStoreRecipe(*SI, StoreGroup->getAddr(), Res, nullptr,
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-constant-ops.ll b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-constant-ops.ll
index 27d4a16c6e274..66559c3c3697a 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-constant-ops.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-constant-ops.ll
@@ -115,28 +115,18 @@ define void @test_add_double_mixed_const_args(ptr %res, ptr noalias %A, ptr noal
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 2
+; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 1
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[B]], i64 [[INDEX]]
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[B]], i64 [[TMP1]]
-; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <4 x double>, ptr [[TMP2]], align 4
-; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <4 x double> [[WIDE_VEC]], <4 x double> poison, <2 x i32> <i32 0, i32 2>
-; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <4 x double> [[WIDE_VEC]], <4 x double> poison, <2 x i32> <i32 1, i32 3>
-; CHECK-NEXT: [[WIDE_VEC2:%.*]] = load <4 x double>, ptr [[TMP3]], align 4
-; CHECK-NEXT: [[STRIDED_VEC3:%.*]] = shufflevector <4 x double> [[WIDE_VEC2]], <4 x double> poison, <2 x i32> <i32 0, i32 2>
-; CHECK-NEXT: [[STRIDED_VEC4:%.*]] = shufflevector <4 x double> [[WIDE_VEC2]], <4 x double> poison, <2 x i32> <i32 1, i32 3>
-; CHECK-NEXT: [[TMP7:%.*]] = fadd <2 x double> splat (double 1.000000e+00), [[STRIDED_VEC]]
-; CHECK-NEXT: [[TMP4:%.*]] = fadd <2 x double> splat (double 1.000000e+00), [[STRIDED_VEC3]]
-; CHECK-NEXT: [[TMP5:%.*]] = fadd <2 x double> splat (double 2.000000e+00), [[STRIDED_VEC1]]
-; CHECK-NEXT: [[TMP6:%.*]] = fadd <2 x double> splat (double 2.000000e+00), [[STRIDED_VEC4]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x double>, ptr [[TMP2]], align 4
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <2 x double>, ptr [[TMP3]], align 4
+; CHECK-NEXT: [[TMP5:%.*]] = fadd <2 x double> <double 1.000000e+00, double 2.000000e+00>, [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP4:%.*]] = fadd <2 x double> <double 1.000000e+00, double 2.000000e+00>, [[WIDE_LOAD1]]
; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[RES]], i64 [[INDEX]]
; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[RES]], i64 [[TMP1]]
-; CHECK-NEXT: [[TMP11:%.*]] = shufflevector <2 x double> [[TMP7]], <2 x double> [[TMP5]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x double> [[TMP11]], <4 x double> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
-; CHECK-NEXT: store <4 x double> [[INTERLEAVED_VEC]], ptr [[TMP8]], align 4
-; CHECK-NEXT: [[TMP10:%.*]] = shufflevector <2 x double> [[TMP4]], <2 x double> [[TMP6]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT: [[INTERLEAVED_VEC5:%.*]] = shufflevector <4 x double> [[TMP10]], <4 x double> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
-; CHECK-NEXT: store <4 x double> [[INTERLEAVED_VEC5]], ptr [[TMP9]], align 4
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: store <2 x double> [[TMP5]], ptr [[TMP8]], align 4
+; CHECK-NEXT: store <2 x double> [[TMP4]], ptr [[TMP9]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
@@ -342,35 +332,23 @@ define void @test_add_double_different_var_args_1(ptr %res, ptr noalias %A, ptr
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x double> poison, double [[X]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <2 x double> [[BROADCAST_SPLATINSERT]], <2 x double> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <2 x double> poison, double [[Y]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT3:%.*]] = shufflevector <2 x double> [[BROADCAST_SPLATINSERT1]], <2 x double> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP0:%.*]] = insertelement <2 x double> poison, double [[X]], i32 0
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x double> [[TMP0]], double [[Y]], i32 1
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 2
-; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[A]], i64 [[INDEX]]
-; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[A]], i64 [[TMP0]]
-; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <4 x double>, ptr [[TMP1]], align 4
-; CHECK-NEXT: [[STRIDED_VEC3:%.*]] = shufflevector <4 x double> [[WIDE_VEC]], <4 x double> poison, <2 x i32> <i32 0, i32 2>
-; CHECK-NEXT: [[STRIDED_VEC4:%.*]] = shufflevector <4 x double> [[WIDE_VEC]], <4 x double> poison, <2 x i32> <i32 1, i32 3>
-; CHECK-NEXT: [[WIDE_VEC4:%.*]] = load <4 x double>, ptr [[TMP2]], align 4
-; CHECK-NEXT: [[STRIDED_VEC6:%.*]] = shufflevector <4 x double> [[WIDE_VEC4]], <4 x double> poison, <2 x i32> <i32 0, i32 2>
-; CHECK-NEXT: [[STRIDED_VEC7:%.*]] = shufflevector <4 x double> [[WIDE_VEC4]], <4 x double> poison, <2 x i32> <i32 1, i32 3>
-; CHECK-NEXT: [[TMP5:%.*]] = fadd <2 x double> [[STRIDED_VEC3]], [[BROADCAST_SPLAT2]]
-; CHECK-NEXT: [[TMP6:%.*]] = fadd <2 x double> [[STRIDED_VEC6]], [[BROADCAST_SPLAT2]]
-; CHECK-NEXT: [[TMP12:%.*]] = fadd <2 x double> [[STRIDED_VEC4]], [[BROADCAST_SPLAT3]]
-; CHECK-NEXT: [[TMP13:%.*]] = fadd <2 x double> [[STRIDED_VEC7]], [[BROADCAST_SPLAT3]]
+; CHECK-NEXT: [[TMP2:%.*]] = add i64 [[INDEX]], 1
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[A]], i64 [[TMP2]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x double>, ptr [[TMP3]], align 4
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <2 x double>, ptr [[TMP4]], align 4
+; CHECK-NEXT: [[TMP5:%.*]] = fadd <2 x double> [[WIDE_LOAD]], [[TMP1]]
+; CHECK-NEXT: [[TMP6:%.*]] = fadd <2 x double> [[WIDE_LOAD1]], [[TMP1]]
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[RES]], i64 [[INDEX]]
-; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[RES]], i64 [[TMP0]]
-; CHECK-NEXT: [[TMP9:%.*]] = shufflevector <2 x double> [[TMP5]], <2 x double> [[TMP12]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x double> [[TMP9]], <4 x double> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
-; CHECK-NEXT: store <4 x double> [[INTERLEAVED_VEC]], ptr [[TMP7]], align 4
-; CHECK-NEXT: [[TMP10:%.*]] = shufflevector <2 x double> [[TMP6]], <2 x double> [[TMP13]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT: [[INTERLEAVED_VEC7:%.*]] = shufflevector <4 x double> [[TMP10]], <4 x double> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
-; CHECK-NEXT: store <4 x double> [[INTERLEAVED_VEC7]], ptr [[TMP8]], align 4
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[RES]], i64 [[TMP2]]
+; CHECK-NEXT: store <2 x double> [[TMP5]], ptr [[TMP7]], align 4
+; CHECK-NEXT: store <2 x double> [[TMP6]], ptr [[TMP8]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
; CHECK-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
@@ -407,35 +385,23 @@ define void @test_add_double_different_var_args_2(ptr %res, ptr noalias %A, ptr
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x double> poison, double [[Y]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <2 x double> [[BROADCAST_SPLATINSERT]], <2 x double> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <2 x double> poison, double [[X]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT3:%.*]] = shufflevector <2 x double> [[BROADCAST_SPLATINSERT1]], <2 x double> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP3:%.*]] = insertelement <2 x double> poison, double [[Y]], i32 0
+; CHECK-NEXT: [[BROADCAST_SPLAT3:%.*]] = insertelement <2 x double> [[TMP3]], double [[X]], i32 1
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 2
+; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 1
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[A]], i64 [[INDEX]]
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[A]], i64 [[TMP0]]
-; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <4 x double>, ptr [[TMP1]], align 4
-; CHECK-NEXT: [[STRIDED_VEC3:%.*]] = shufflevector <4 x double> [[WIDE_VEC]], <4 x double> poison, <2 x i32> <i32 0, i32 2>
-; CHECK-NEXT: [[STRIDED_VEC4:%.*]] = shufflevector <4 x double> [[WIDE_VEC]], <4 x double> poison, <2 x i32> <i32 1, i32 3>
-; CHECK-NEXT: [[WIDE_VEC4:%.*]] = load <4 x double>, ptr [[TMP2]], align 4
-; CHECK-NEXT: [[STRIDED_VEC6:%.*]] = shufflevector <4 x double> [[WIDE_VEC4]], <4 x double> poison, <2 x i32> <i32 0, i32 2>
-; CHECK-NEXT: [[STRIDED_VEC7:%.*]] = shufflevector <4 x double> [[WIDE_VEC4]], <4 x double> poison, <2 x i32> <i32 1, i32 3>
-; CHECK-NEXT: [[TMP5:%.*]] = fadd <2 x double> [[BROADCAST_SPLAT2]], [[STRIDED_VEC3]]
-; CHECK-NEXT: [[TMP6:%.*]] = fadd <2 x double> [[BROADCAST_SPLAT2]], [[STRIDED_VEC6]]
+; CHECK-NEXT: [[STRIDED_VEC4:%.*]] = load <2 x double>, ptr [[TMP1]], align 4
+; CHECK-NEXT: [[STRIDED_VEC7:%.*]] = load <2 x double>, ptr [[TMP2]], align 4
; CHECK-NEXT: [[TMP12:%.*]] = fadd <2 x double> [[BROADCAST_SPLAT3]], [[STRIDED_VEC4]]
; CHECK-NEXT: [[TMP13:%.*]] = fadd <2 x double> [[BROADCAST_SPLAT3]], [[STRIDED_VEC7]]
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[RES]], i64 [[INDEX]]
; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[RES]], i64 [[TMP0]]
-; CHECK-NEXT: [[TMP9:%.*]] = shufflevector <2 x double> [[TMP5]], <2 x double> [[TMP12]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x double> [[TMP9]], <4 x double> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
-; CHECK-NEXT: store <4 x double> [[INTERLEAVED_VEC]], ptr [[TMP7]], align 4
-; CHECK-NEXT: [[TMP10:%.*]] = shufflevector <2 x double> [[TMP6]], <2 x double> [[TMP13]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT: [[INTERLEAVED_VEC7:%.*]] = shufflevector <4 x double> [[TMP10]], <4 x double> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
-; CHECK-NEXT: store <4 x double> [[INTERLEAVED_VEC7]], ptr [[TMP8]], align 4
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: store <2 x double> [[TMP12]], ptr [[TMP7]], align 4
+; CHECK-NEXT: store <2 x double> [[TMP13]], ptr [[TMP8]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
; CHECK-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-with-wide-ops-chained.ll b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-with-wide-ops-chained.ll
index 61891ff31ffae..11fd2748dca60 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-with-wide-ops-chained.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-with-wide-ops-chained.ll
@@ -238,20 +238,16 @@ define void @test_2xi64_mul_sub_mismatched_ops1(ptr noalias %data, ptr noalias %
; VF2: [[VECTOR_BODY]]:
; VF2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; VF2-NEXT: [[TMP0:%.*]] = getelementptr inbounds i64, ptr [[FACTOR]], i64 [[INDEX]]
-; VF2-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i64>, ptr [[TMP0]], align 8
+; VF2-NEXT: [[TMP3:%.*]] = load i64, ptr [[TMP0]], align 8
+; VF2-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i64> poison, i64 [[TMP3]], i64 0
+; VF2-NEXT: [[WIDE_LOAD:%.*]] = shufflevector <2 x i64> [[BROADCAST_SPLATINSERT]], <2 x i64> poison, <2 x i32> zeroinitializer
; VF2-NEXT: [[TMP1:%.*]] = shl nsw i64 [[INDEX]], 1
; VF2-NEXT: [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[TMP1]]
-; VF2-NEXT: [[WIDE_VEC:%.*]] = load <4 x i64>, ptr [[TMP2]], align 8
-; VF2-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <4 x i64> [[WIDE_VEC]], <4 x i64> poison, <2 x i32> <i32 0, i32 2>
-; VF2-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <4 x i64> [[WIDE_VEC]], <4 x i64> poison, <2 x i32> <i32 1, i32 3>
-; VF2-NEXT: [[TMP3:%.*]] = mul <2 x i64> [[WIDE_LOAD]], [[STRIDED_VEC]]
-; VF2-NEXT: [[TMP4:%.*]] = sub <2 x i64> [[TMP3]], splat (i64 2)
+; VF2-NEXT: [[STRIDED_VEC1:%.*]] = load <2 x i64>, ptr [[TMP2]], align 8
; VF2-NEXT: [[TMP5:%.*]] = mul <2 x i64> [[WIDE_LOAD]], [[STRIDED_VEC1]]
-; VF2-NEXT: [[TMP6:%.*]] = sub <2 x i64> [[TMP5]], splat (i64 3)
-; VF2-NEXT: [[TMP7:%.*]] = shufflevector <2 x i64> [[TMP4]], <2 x i64> [[TMP6]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; VF2-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x i64> [[TMP7]], <4 x i64> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
-; VF2-NEXT: store <4 x i64> [[INTERLEAVED_VEC]], ptr [[TMP2]], align 8
-; VF2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; VF2-NEXT: [[TMP6:%.*]] = sub <2 x i64> [[TMP5]], <i64 2, i64 3>
+; VF2-NEXT: store <2 x i64> [[TMP6]], ptr [[TMP2]], align 8
+; VF2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 1
; VF2-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
; VF2-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
; VF2: [[MIDDLE_BLOCK]]:
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-accesses.ll b/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-accesses.ll
index 7ff7e6deaf503..cc65d7903c5a7 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-accesses.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-accesses.ll
@@ -524,21 +524,10 @@ define void @load_store_factor4(ptr %p) {
; FIXED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
; FIXED-NEXT: [[TMP0:%.*]] = shl i64 [[INDEX]], 2
; FIXED-NEXT: [[TMP1:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[TMP0]]
-; FIXED-NEXT: [[WIDE_VEC:%.*]] = load <16 x i64>, ptr [[TMP1]], align 8
-; FIXED-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <16 x i64> [[WIDE_VEC]], <16 x i64> poison, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
-; FIXED-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <16 x i64> [[WIDE_VEC]], <16 x i64> poison, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
-; FIXED-NEXT: [[STRIDED_VEC2:%.*]] = shufflevector <16 x i64> [[WIDE_VEC]], <16 x i64> poison, <4 x i32> <i32 2, i32 6, i32 10, i32 14>
-; FIXED-NEXT: [[STRIDED_VEC3:%.*]] = shufflevector <16 x i64> [[WIDE_VEC]], <16 x i64> poison, <4 x i32> <i32 3, i32 7, i32 11, i32 15>
-; FIXED-NEXT: [[TMP2:%.*]] = add <4 x i64> [[STRIDED_VEC]], splat (i64 1)
-; FIXED-NEXT: [[TMP3:%.*]] = add <4 x i64> [[STRIDED_VEC1]], splat (i64 2)
-; FIXED-NEXT: [[TMP4:%.*]] = add <4 x i64> [[STRIDED_VEC2]], splat (i64 3)
-; FIXED-NEXT: [[TMP5:%.*]] = add <4 x i64> [[STRIDED_VEC3]], splat (i64 4)
-; FIXED-NEXT: [[TMP6:%.*]] = shufflevector <4 x i64> [[TMP2]], <4 x i64> [[TMP3]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; FIXED-NEXT: [[TMP7:%.*]] = shufflevector <4 x i64> [[TMP4]], <4 x i64> [[TMP5]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; FIXED-NEXT: [[TMP8:%.*]] = shufflevector <8 x i64> [[TMP6]], <8 x i64> [[TMP7]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; FIXED-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <16 x i64> [[TMP8]], <16 x i64> poison, <16 x i32> <i32 0, i32 4, i32 8, i32 12, i32 1, i32 5, i32 9, i32 13, i32 2, i32 6, i32 10, i32 14, i32 3, i32 7, i32 11, i32 15>
-; FIXED-NEXT: store <16 x i64> [[INTERLEAVED_VEC]], ptr [[TMP1]], align 8
-; FIXED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; FIXED-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP1]], align 8
+; FIXED-NEXT: [[TMP2:%.*]] = add <4 x i64> [[WIDE_LOAD]], <i64 1, i64 2, i64 3, i64 4>
+; FIXED-NEXT: store <4 x i64> [[TMP2]], ptr [[TMP1]], align 8
+; FIXED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 1
; FIXED-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
; FIXED-NEXT: br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
; FIXED: middle.block:
diff --git a/llvm/test/Transforms/LoopVectorize/narrow-interleave-groups-scalable-vf.ll b/llvm/test/Transforms/LoopVectorize/narrow-interleave-groups-scalable-vf.ll
index 9a6870ee3bd0f..cf27467d7640d 100644
--- a/llvm/test/Transforms/LoopVectorize/narrow-interleave-groups-scalable-vf.ll
+++ b/llvm/test/Transforms/LoopVectorize/narrow-interleave-groups-scalable-vf.ll
@@ -101,3 +101,136 @@ loop:
exit:
ret void
}
+
+; Each field uses a distinct live-in. We cannot use BuildVector to combine them for scalable vectors.
+define void @scalablevf_distinct_args(ptr %dst.start, i8 %a0, i8 %a1, i8 %a2, i8 %a3, i8 %b0, i8 %b1, i8 %b2, i8 %b3) {
+; CHECK-LABEL: define void @scalablevf_distinct_args(
+; CHECK-SAME: ptr [[DST_START:%.*]], i8 [[A0:%.*]], i8 [[A1:%.*]], i8 [[A2:%.*]], i8 [[A3:%.*]], i8 [[B0:%.*]], i8 [[B1:%.*]], i8 [[B2:%.*]], i8 [[B3:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP13:%.*]] = insertelement <4 x i8> poison, i8 [[A0]], i32 0
+; CHECK-NEXT: [[TMP14:%.*]] = insertelement <4 x i8> [[TMP13]], i8 [[A1]], i32 1
+; CHECK-NEXT: [[TMP15:%.*]] = insertelement <4 x i8> [[TMP14]], i8 [[A2]], i32 2
+; CHECK-NEXT: [[TMP3:%.*]] = insertelement <4 x i8> [[TMP15]], i8 [[A3]], i32 3
+; CHECK-NEXT: [[TMP4:%.*]] = insertelement <4 x i8> poison, i8 [[B0]], i32 0
+; CHECK-NEXT: [[TMP5:%.*]] = insertelement <4 x i8> [[TMP4]], i8 [[B1]], i32 1
+; CHECK-NEXT: [[TMP6:%.*]] = insertelement <4 x i8> [[TMP5]], i8 [[B2]], i32 2
+; CHECK-NEXT: [[TMP7:%.*]] = insertelement <4 x i8> [[TMP6]], i8 [[B3]], i32 3
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP8:%.*]] = shl i64 [[INDEX]], 2
+; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[DST_START]], i64 [[TMP8]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[NEXT_GEP]], align 1
+; CHECK-NEXT: [[TMP9:%.*]] = mul nuw <4 x i8> [[WIDE_LOAD]], [[TMP7]]
+; CHECK-NEXT: [[TMP10:%.*]] = udiv <4 x i8> [[TMP9]], splat (i8 -1)
+; CHECK-NEXT: [[TMP11:%.*]] = add <4 x i8> [[TMP3]], [[TMP10]]
+; CHECK-NEXT: store <4 x i8> [[TMP11]], ptr [[NEXT_GEP]], align 1
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 1
+; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], 64
+; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+; SCALABLE-LABEL: define void @scalablevf_distinct_args(
+; SCALABLE-SAME: ptr [[DST_START:%.*]], i8 [[A0:%.*]], i8 [[A1:%.*]], i8 [[A2:%.*]], i8 [[A3:%.*]], i8 [[B0:%.*]], i8 [[B1:%.*]], i8 [[B2:%.*]], i8 [[B3:%.*]]) {
+; SCALABLE-NEXT: [[ENTRY:.*:]]
+; SCALABLE-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; SCALABLE-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; SCALABLE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 64, [[TMP1]]
+; SCALABLE-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; SCALABLE: [[VECTOR_PH]]:
+; SCALABLE-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 2
+; SCALABLE-NEXT: [[N_MOD_VF:%.*]] = urem i64 64, [[TMP2]]
+; SCALABLE-NEXT: [[N_VEC:%.*]] = sub i64 64, [[N_MOD_VF]]
+; SCALABLE-NEXT: [[TMP3:%.*]] = shl i64 [[N_VEC]], 2
+; SCALABLE-NEXT: [[TMP4:%.*]] = getelementptr i8, ptr [[DST_START]], i64 [[TMP3]]
+; SCALABLE-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i8> poison, i8 [[B0]], i64 0
+; SCALABLE-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i8> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i8> poison, <vscale x 4 x i32> zeroinitializer
+; SCALABLE-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 4 x i8> poison, i8 [[A0]], i64 0
+; SCALABLE-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 4 x i8> [[BROADCAST_SPLATINSERT1]], <vscale x 4 x i8> poison, <vscale x 4 x i32> zeroinitializer
+; SCALABLE-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <vscale x 4 x i8> poison, i8 [[B1]], i64 0
+; SCALABLE-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <vscale x 4 x i8> [[BROADCAST_SPLATINSERT3]], <vscale x 4 x i8> poison, <vscale x 4 x i32> zeroinitializer
+; SCALABLE-NEXT: [[BROADCAST_SPLATINSERT5:%.*]] = insertelement <vscale x 4 x i8> poison, i8 [[A1]], i64 0
+; SCALABLE-NEXT: [[BROADCAST_SPLAT6:%.*]] = shufflevector <vscale x 4 x i8> [[BROADCAST_SPLATINSERT5]], <vscale x 4 x i8> poison, <vscale x 4 x i32> zeroinitializer
+; SCALABLE-NEXT: [[BROADCAST_SPLATINSERT7:%.*]] = insertelement <vscale x 4 x i8> poison, i8 [[B2]], i64 0
+; SCALABLE-NEXT: [[BROADCAST_SPLAT8:%.*]] = shufflevector <vscale x 4 x i8> [[BROADCAST_SPLATINSERT7]], <vscale x 4 x i8> poison, <vscale x 4 x i32> zeroinitializer
+; SCALABLE-NEXT: [[BROADCAST_SPLATINSERT9:%.*]] = insertelement <vscale x 4 x i8> poison, i8 [[A2]], i64 0
+; SCALABLE-NEXT: [[BROADCAST_SPLAT10:%.*]] = shufflevector <vscale x 4 x i8> [[BROADCAST_SPLATINSERT9]], <vscale x 4 x i8> poison, <vscale x 4 x i32> zeroinitializer
+; SCALABLE-NEXT: [[BROADCAST_SPLATINSERT11:%.*]] = insertelement <vscale x 4 x i8> poison, i8 [[B3]], i64 0
+; SCALABLE-NEXT: [[BROADCAST_SPLAT12:%.*]] = shufflevector <vscale x 4 x i8> [[BROADCAST_SPLATINSERT11]], <vscale x 4 x i8> poison, <vscale x 4 x i32> zeroinitializer
+; SCALABLE-NEXT: [[BROADCAST_SPLATINSERT13:%.*]] = insertelement <vscale x 4 x i8> poison, i8 [[A3]], i64 0
+; SCALABLE-NEXT: [[BROADCAST_SPLAT14:%.*]] = shufflevector <vscale x 4 x i8> [[BROADCAST_SPLATINSERT13]], <vscale x 4 x i8> poison, <vscale x 4 x i32> zeroinitializer
+; SCALABLE-NEXT: br label %[[VECTOR_BODY:.*]]
+; SCALABLE: [[VECTOR_BODY]]:
+; SCALABLE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; SCALABLE-NEXT: [[TMP5:%.*]] = shl i64 [[INDEX]], 2
+; SCALABLE-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[DST_START]], i64 [[TMP5]]
+; SCALABLE-NEXT: [[WIDE_VEC:%.*]] = load <vscale x 16 x i8>, ptr [[NEXT_GEP]], align 1
+; SCALABLE-NEXT: [[STRIDED_VEC:%.*]] = call { <vscale x 4 x i8>, <vscale x 4 x i8>, <vscale x 4 x i8>, <vscale x 4 x i8> } @llvm.vector.deinterleave4.nxv16i8(<vscale x 16 x i8> [[WIDE_VEC]])
+; SCALABLE-NEXT: [[TMP6:%.*]] = extractvalue { <vscale x 4 x i8>, <vscale x 4 x i8>, <vscale x 4 x i8>, <vscale x 4 x i8> } [[STRIDED_VEC]], 0
+; SCALABLE-NEXT: [[TMP7:%.*]] = extractvalue { <vscale x 4 x i8>, <vscale x 4 x i8>, <vscale x 4 x i8>, <vscale x 4 x i8> } [[STRIDED_VEC]], 1
+; SCALABLE-NEXT: [[TMP8:%.*]] = extractvalue { <vscale x 4 x i8>, <vscale x 4 x i8>, <vscale x 4 x i8>, <vscale x 4 x i8> } [[STRIDED_VEC]], 2
+; SCALABLE-NEXT: [[TMP9:%.*]] = extractvalue { <vscale x 4 x i8>, <vscale x 4 x i8>, <vscale x 4 x i8>, <vscale x 4 x i8> } [[STRIDED_VEC]], 3
+; SCALABLE-NEXT: [[TMP10:%.*]] = mul nuw <vscale x 4 x i8> [[TMP6]], [[BROADCAST_SPLAT]]
+; SCALABLE-NEXT: [[TMP11:%.*]] = udiv <vscale x 4 x i8> [[TMP10]], splat (i8 -1)
+; SCALABLE-NEXT: [[TMP12:%.*]] = add <vscale x 4 x i8> [[BROADCAST_SPLAT2]], [[TMP11]]
+; SCALABLE-NEXT: [[TMP13:%.*]] = mul nuw <vscale x 4 x i8> [[TMP7]], [[BROADCAST_SPLAT4]]
+; SCALABLE-NEXT: [[TMP14:%.*]] = udiv <vscale x 4 x i8> [[TMP13]], splat (i8 -1)
+; SCALABLE-NEXT: [[TMP15:%.*]] = add <vscale x 4 x i8> [[BROADCAST_SPLAT6]], [[TMP14]]
+; SCALABLE-NEXT: [[TMP16:%.*]] = mul nuw <vscale x 4 x i8> [[TMP8]], [[BROADCAST_SPLAT8]]
+; SCALABLE-NEXT: [[TMP17:%.*]] = udiv <vscale x 4 x i8> [[TMP16]], splat (i8 -1)
+; SCALABLE-NEXT: [[TMP18:%.*]] = add <vscale x 4 x i8> [[BROADCAST_SPLAT10]], [[TMP17]]
+; SCALABLE-NEXT: [[TMP19:%.*]] = mul nuw <vscale x 4 x i8> [[TMP9]], [[BROADCAST_SPLAT12]]
+; SCALABLE-NEXT: [[TMP20:%.*]] = udiv <vscale x 4 x i8> [[TMP19]], splat (i8 -1)
+; SCALABLE-NEXT: [[TMP21:%.*]] = add <vscale x 4 x i8> [[BROADCAST_SPLAT14]], [[TMP20]]
+; SCALABLE-NEXT: [[INTERLEAVED_VEC:%.*]] = call <vscale x 16 x i8> @llvm.vector.interleave4.nxv16i8(<vscale x 4 x i8> [[TMP12]], <vscale x 4 x i8> [[TMP15]], <vscale x 4 x i8> [[TMP18]], <vscale x 4 x i8> [[TMP21]])
+; SCALABLE-NEXT: store <vscale x 16 x i8> [[INTERLEAVED_VEC]], ptr [[NEXT_GEP]], align 1
+; SCALABLE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
+; SCALABLE-NEXT: [[TMP22:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; SCALABLE-NEXT: br i1 [[TMP22]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; SCALABLE: [[MIDDLE_BLOCK]]:
+; SCALABLE-NEXT: [[CMP_N:%.*]] = icmp eq i64 64, [[N_VEC]]
+; SCALABLE-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
+; SCALABLE: [[SCALAR_PH]]:
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %dst = phi ptr [ %dst.start, %entry ], [ %dst.next, %loop ]
+ %dst.next = getelementptr inbounds nuw i8, ptr %dst, i64 4
+ %load.dst = load i8, ptr %dst, align 1
+ %mul.dst.0 = mul nuw i8 %load.dst, %b0
+ %udiv.0 = udiv i8 %mul.dst.0, 255
+ %val.0 = add i8 %a0, %udiv.0
+ store i8 %val.0, ptr %dst, align 1
+ %gep.dst.1 = getelementptr inbounds nuw i8, ptr %dst, i64 1
+ %load.dst.1 = load i8, ptr %gep.dst.1, align 1
+ %mul.dst.1 = mul nuw i8 %load.dst.1, %b1
+ %udiv.1 = udiv i8 %mul.dst.1, 255
+ %val.1 = add i8 %a1, %udiv.1
+ store i8 %val.1, ptr %gep.dst.1, align 1
+ %gep.dst.2 = getelementptr inbounds nuw i8, ptr %dst, i64 2
+ %load.dst.2 = load i8, ptr %gep.dst.2, align 1
+ %mul.dst.2 = mul nuw i8 %load.dst.2, %b2
+ %udiv.2 = udiv i8 %mul.dst.2, 255
+ %val.2 = add i8 %a2, %udiv.2
+ store i8 %val.2, ptr %gep.dst.2, align 1
+ %gep.dst.3 = getelementptr inbounds nuw i8, ptr %dst, i64 3
+ %load.dst.3 = load i8, ptr %gep.dst.3, align 1
+ %mul.dst.3 = mul nuw i8 %load.dst.3, %b3
+ %udiv.3 = udiv i8 %mul.dst.3, 255
+ %val.3 = add i8 %a3, %udiv.3
+ store i8 %val.3, ptr %gep.dst.3, align 1
+ %iv.next = add i64 %iv, 4
+ %exit.cond = icmp eq i64 %iv.next, 256
+ br i1 %exit.cond, label %exit, label %loop
+
+exit:
+ ret void
+}
>From 7794212df4361476d592dfa19932c261eeb081ad Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Sat, 13 Jun 2026 17:15:50 +0200
Subject: [PATCH 2/2] [VPlan] Support different loop-invariant ops in narrowIG.
Generalize the distinct per-field operand narrowing in
narrowInterleaveGroups from plain live-ins to any value defined outside
the vector loop region.
Depends on https://github.com/llvm/llvm-project/pull/203778 (included in
PR)
---
.../Transforms/Vectorize/VPlanTransforms.cpp | 22 +++++++--------
...interleave-to-widen-memory-constant-ops.ll | 28 ++++++-------------
2 files changed, 19 insertions(+), 31 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index a030b0d490163..e7abb453c169a 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -5560,15 +5560,15 @@ VPlanTransforms::expandSCEVs(VPlan &Plan, ScalarEvolution &SE) {
static bool canNarrowLoad(VPSingleDefRecipe *WideMember0, unsigned OpIdx,
VPValue *OpV, unsigned Idx, bool IsScalable) {
VPValue *Member0Op = WideMember0->getOperand(OpIdx);
- VPRecipeBase *Member0OpR = Member0Op->getDefiningRecipe();
- if (!Member0OpR) {
- // Member0's operand is a uniform live-in, broadcast across all fields.
+ if (Member0Op->isDefinedOutsideLoopRegions()) {
+ // Operand matches Member0, broadcast across all fields.
if (Member0Op == OpV)
return true;
- // Otherwise distinct per-field live-ins are assembled into a BuildVector.
- return !IsScalable && !OpV->getDefiningRecipe() &&
+ // Otherwise distinct per-field VPValues are assembled into a BuildVector.
+ return !IsScalable && OpV->isDefinedOutsideLoopRegions() &&
OpV->getScalarType() == Member0Op->getScalarType();
}
+ VPRecipeBase *Member0OpR = Member0Op->getDefiningRecipe();
if (auto *W = dyn_cast<VPWidenLoadRecipe>(Member0OpR))
// For scalable VFs, the narrowed plan processes vscale iterations at once,
// so a shared wide load cannot be narrowed to a uniform scalar; bail out.
@@ -5668,7 +5668,7 @@ static bool isAlreadyNarrow(VPValue *VPV) {
// Convert the wide recipes defining the VPValues in \p Members feeding an
// interleave group to a single narrow variant. The first member is reused as
// the narrowed recipe. Newly created loop-invariant recipes (e.g. a BuildVector
-// for distinct live-ins) are inserted into \p Preheader.
+// for distinct loop-invariant values) are inserted into \p Preheader.
static VPValue *narrowInterleaveGroupOp(ArrayRef<VPValue *> Members,
SmallPtrSetImpl<VPValue *> &NarrowedOps,
VPBasicBlock *Preheader) {
@@ -5677,17 +5677,17 @@ static VPValue *narrowInterleaveGroupOp(ArrayRef<VPValue *> Members,
if (NarrowedOps.contains(V))
return V;
- if (!R) {
- // V is a live-in. If all members are the same live-in, reuse it directly.
+ if (V->isDefinedOutsideLoopRegions()) {
+ // V is loop-invariant. If all members are the same, reuse it directly.
if (all_of(Members, equal_to(V)))
return V;
- // Otherwise assemble the distinct live-ins into a BuildVector.
+ // Otherwise assemble the distinct VPValues into a BuildVector.
assert(all_of(Members,
[V](VPValue *M) {
- return !M->getDefiningRecipe() &&
+ return M->isDefinedOutsideLoopRegions() &&
M->getScalarType() == V->getScalarType();
}) &&
- "expected distinct live-ins of matching scalar type");
+ "expected distinct loop-invariant values of matching scalar type");
auto *BV = new VPInstruction(VPInstruction::BuildVector, Members);
Preheader->appendRecipe(BV);
NarrowedOps.insert(BV);
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-constant-ops.ll b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-constant-ops.ll
index 66559c3c3697a..9945874df559c 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-constant-ops.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-constant-ops.ll
@@ -441,36 +441,24 @@ define void @test_add_double_different_invariant_recipe_args(ptr %res, ptr noali
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[TMP0:%.*]] = fmul double [[X]], [[Z]]
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x double> poison, double [[TMP0]], i64 0
-; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <2 x double> [[BROADCAST_SPLATINSERT]], <2 x double> poison, <2 x i32> zeroinitializer
; CHECK-NEXT: [[TMP1:%.*]] = fmul double [[Y]], [[Z]]
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT5:%.*]] = insertelement <2 x double> poison, double [[TMP1]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT6:%.*]] = shufflevector <2 x double> [[BROADCAST_SPLATINSERT5]], <2 x double> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x double> poison, double [[TMP0]], i32 0
+; CHECK-NEXT: [[BROADCAST_SPLAT6:%.*]] = insertelement <2 x double> [[TMP2]], double [[TMP1]], i32 1
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP4:%.*]] = add i64 [[INDEX]], 2
+; CHECK-NEXT: [[TMP4:%.*]] = add i64 [[INDEX]], 1
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[A]], i64 [[INDEX]]
; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[A]], i64 [[TMP4]]
-; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <4 x double>, ptr [[TMP5]], align 4
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = shufflevector <4 x double> [[WIDE_VEC]], <4 x double> poison, <2 x i32> <i32 0, i32 2>
-; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <4 x double> [[WIDE_VEC]], <4 x double> poison, <2 x i32> <i32 1, i32 3>
-; CHECK-NEXT: [[WIDE_VEC2:%.*]] = load <4 x double>, ptr [[TMP6]], align 4
-; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = shufflevector <4 x double> [[WIDE_VEC2]], <4 x double> poison, <2 x i32> <i32 0, i32 2>
-; CHECK-NEXT: [[STRIDED_VEC4:%.*]] = shufflevector <4 x double> [[WIDE_VEC2]], <4 x double> poison, <2 x i32> <i32 1, i32 3>
-; CHECK-NEXT: [[TMP7:%.*]] = fadd <2 x double> [[TMP3]], [[WIDE_LOAD]]
-; CHECK-NEXT: [[TMP8:%.*]] = fadd <2 x double> [[TMP3]], [[WIDE_LOAD1]]
+; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = load <2 x double>, ptr [[TMP5]], align 4
+; CHECK-NEXT: [[STRIDED_VEC4:%.*]] = load <2 x double>, ptr [[TMP6]], align 4
; CHECK-NEXT: [[TMP13:%.*]] = fadd <2 x double> [[BROADCAST_SPLAT6]], [[STRIDED_VEC1]]
; CHECK-NEXT: [[TMP14:%.*]] = fadd <2 x double> [[BROADCAST_SPLAT6]], [[STRIDED_VEC4]]
; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[RES]], i64 [[INDEX]]
; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[RES]], i64 [[TMP4]]
-; CHECK-NEXT: [[TMP15:%.*]] = shufflevector <2 x double> [[TMP7]], <2 x double> [[TMP13]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x double> [[TMP15]], <4 x double> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
-; CHECK-NEXT: store <4 x double> [[INTERLEAVED_VEC]], ptr [[TMP9]], align 4
-; CHECK-NEXT: [[TMP12:%.*]] = shufflevector <2 x double> [[TMP8]], <2 x double> [[TMP14]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT: [[INTERLEAVED_VEC7:%.*]] = shufflevector <4 x double> [[TMP12]], <4 x double> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
-; CHECK-NEXT: store <4 x double> [[INTERLEAVED_VEC7]], ptr [[TMP10]], align 4
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: store <2 x double> [[TMP13]], ptr [[TMP9]], align 4
+; CHECK-NEXT: store <2 x double> [[TMP14]], ptr [[TMP10]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
; CHECK-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
More information about the llvm-commits
mailing list