[llvm] [VectorCombine] fold partial interleaved load (PR #219201)
Kamlesh Kumar via llvm-commits
llvm-commits at lists.llvm.org
Thu Aug 27 05:41:44 PDT 2026
https://github.com/kamleshbhalui created https://github.com/llvm/llvm-project/pull/219201
Replace a partially used deinterleave of a load with strided shuffles of the
loaded vector when it is cheaper than an interleaved load.
>From 489642f2df9708f967740382544d70e7479a252c Mon Sep 17 00:00:00 2001
From: Kamlesh Kumar <kamlesh.kumar at arm.com>
Date: Thu, 27 Aug 2026 13:22:46 +0100
Subject: [PATCH 1/2] added a test
---
.../AArch64/partial-deinterleave.ll | 63 +++++++++++++++++++
1 file changed, 63 insertions(+)
create mode 100644 llvm/test/Transforms/VectorCombine/AArch64/partial-deinterleave.ll
diff --git a/llvm/test/Transforms/VectorCombine/AArch64/partial-deinterleave.ll b/llvm/test/Transforms/VectorCombine/AArch64/partial-deinterleave.ll
new file mode 100644
index 0000000000000..3814cc9a00dac
--- /dev/null
+++ b/llvm/test/Transforms/VectorCombine/AArch64/partial-deinterleave.ll
@@ -0,0 +1,63 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -mtriple=aarch64-linux-gnu -passes=vector-combine -S < %s | FileCheck %s
+
+define void @partial_deinterleave4_ptrs(ptr %src0, ptr %src1, ptr %dst0,
+; CHECK-LABEL: define void @partial_deinterleave4_ptrs(
+; CHECK-SAME: ptr [[SRC0:%.*]], ptr [[SRC1:%.*]], ptr [[DST0:%.*]], ptr [[DST1:%.*]]) {
+; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <8 x ptr>, ptr [[SRC0]], align 8
+; CHECK-NEXT: [[STRIDED_VEC:%.*]] = call { <2 x ptr>, <2 x ptr>, <2 x ptr>, <2 x ptr> } @llvm.vector.deinterleave4.v8p0(<8 x ptr> [[WIDE_VEC]])
+; CHECK-NEXT: [[FIELD0:%.*]] = extractvalue { <2 x ptr>, <2 x ptr>, <2 x ptr>, <2 x ptr> } [[STRIDED_VEC]], 0
+; CHECK-NEXT: [[WIDE_VEC_1:%.*]] = load <8 x ptr>, ptr [[SRC1]], align 8
+; CHECK-NEXT: [[STRIDED_VEC_1:%.*]] = call { <2 x ptr>, <2 x ptr>, <2 x ptr>, <2 x ptr> } @llvm.vector.deinterleave4.v8p0(<8 x ptr> [[WIDE_VEC_1]])
+; CHECK-NEXT: [[FIELD0_1:%.*]] = extractvalue { <2 x ptr>, <2 x ptr>, <2 x ptr>, <2 x ptr> } [[STRIDED_VEC_1]], 0
+; CHECK-NEXT: store <2 x ptr> [[FIELD0]], ptr [[DST0]], align 8
+; CHECK-NEXT: store <2 x ptr> [[FIELD0_1]], ptr [[DST1]], align 8
+; CHECK-NEXT: ret void
+;
+ ptr %dst1) {
+ %wide.vec = load <8 x ptr>, ptr %src0, align 8
+ %strided.vec = call { <2 x ptr>, <2 x ptr>, <2 x ptr>, <2 x ptr> } @llvm.vector.deinterleave4.v8p0(<8 x ptr> %wide.vec)
+ %field0 = extractvalue { <2 x ptr>, <2 x ptr>, <2 x ptr>, <2 x ptr> } %strided.vec, 0
+ %wide.vec.1 = load <8 x ptr>, ptr %src1, align 8
+ %strided.vec.1 = call { <2 x ptr>, <2 x ptr>, <2 x ptr>, <2 x ptr> } @llvm.vector.deinterleave4.v8p0(<8 x ptr> %wide.vec.1)
+ %field0.1 = extractvalue { <2 x ptr>, <2 x ptr>, <2 x ptr>, <2 x ptr> } %strided.vec.1, 0
+ store <2 x ptr> %field0, ptr %dst0, align 8
+ store <2 x ptr> %field0.1, ptr %dst1, align 8
+ ret void
+}
+
+define <2 x ptr> @partial_deinterleave4_field1(ptr %src) {
+; CHECK-LABEL: define <2 x ptr> @partial_deinterleave4_field1(
+; CHECK-SAME: ptr [[SRC:%.*]]) {
+; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <8 x ptr>, ptr [[SRC]], align 8
+; CHECK-NEXT: [[STRIDED_VEC:%.*]] = call { <2 x ptr>, <2 x ptr>, <2 x ptr>, <2 x ptr> } @llvm.vector.deinterleave4.v8p0(<8 x ptr> [[WIDE_VEC]])
+; CHECK-NEXT: [[FIELD1:%.*]] = extractvalue { <2 x ptr>, <2 x ptr>, <2 x ptr>, <2 x ptr> } [[STRIDED_VEC]], 1
+; CHECK-NEXT: ret <2 x ptr> [[FIELD1]]
+;
+ %wide.vec = load <8 x ptr>, ptr %src, align 8
+ %strided.vec = call { <2 x ptr>, <2 x ptr>, <2 x ptr>, <2 x ptr> } @llvm.vector.deinterleave4.v8p0(<8 x ptr> %wide.vec)
+ %field1 = extractvalue { <2 x ptr>, <2 x ptr>, <2 x ptr>, <2 x ptr> } %strided.vec, 1
+ ret <2 x ptr> %field1
+}
+
+
+define void @multiple_fields(ptr %src, ptr %dst0, ptr %dst1) {
+; CHECK-LABEL: define void @multiple_fields(
+; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DST0:%.*]], ptr [[DST1:%.*]]) {
+; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <8 x ptr>, ptr [[SRC]], align 8
+; CHECK-NEXT: [[STRIDED_VEC:%.*]] = call { <2 x ptr>, <2 x ptr>, <2 x ptr>, <2 x ptr> } @llvm.vector.deinterleave4.v8p0(<8 x ptr> [[WIDE_VEC]])
+; CHECK-NEXT: [[FIELD0:%.*]] = extractvalue { <2 x ptr>, <2 x ptr>, <2 x ptr>, <2 x ptr> } [[STRIDED_VEC]], 0
+; CHECK-NEXT: [[FIELD1:%.*]] = extractvalue { <2 x ptr>, <2 x ptr>, <2 x ptr>, <2 x ptr> } [[STRIDED_VEC]], 1
+; CHECK-NEXT: store <2 x ptr> [[FIELD0]], ptr [[DST0]], align 8
+; CHECK-NEXT: store <2 x ptr> [[FIELD1]], ptr [[DST1]], align 8
+; CHECK-NEXT: ret void
+;
+ %wide.vec = load <8 x ptr>, ptr %src, align 8
+ %strided.vec = call { <2 x ptr>, <2 x ptr>, <2 x ptr>, <2 x ptr> } @llvm.vector.deinterleave4.v8p0(<8 x ptr> %wide.vec)
+ %field0 = extractvalue { <2 x ptr>, <2 x ptr>, <2 x ptr>, <2 x ptr> } %strided.vec, 0
+ %field1 = extractvalue { <2 x ptr>, <2 x ptr>, <2 x ptr>, <2 x ptr> } %strided.vec, 1
+ store <2 x ptr> %field0, ptr %dst0, align 8
+ store <2 x ptr> %field1, ptr %dst1, align 8
+ ret void
+}
+
>From 43ca05f9d9df17fcd6574d1f1fdc3bc9987c6ccb Mon Sep 17 00:00:00 2001
From: Kamlesh Kumar <kamlesh.kumar at arm.com>
Date: Thu, 27 Aug 2026 13:24:07 +0100
Subject: [PATCH 2/2] [VectorCombine] fold Partial interleaved load to shuffle
If only one extract of interleaved load is used then a plain load
and shuffle is cheaper than doing interleaved load
---
.../Transforms/Vectorize/VectorCombine.cpp | 63 +++++++++++++++++++
.../AArch64/partial-deinterleave.ll | 9 +--
2 files changed, 66 insertions(+), 6 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
index dfe3070cfe87c..16e3dc2f0d482 100644
--- a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
+++ b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
@@ -164,6 +164,7 @@ class VectorCombine {
bool shrinkLoadForShuffles(Instruction &I);
bool shrinkPhiOfShuffles(Instruction &I);
bool foldDeinterleaveInterleavePair(Instruction &I);
+ bool foldPartialDeinterleave(Instruction &I);
void replaceValue(Instruction &Old, Value &New, bool Erase = true) {
LLVM_DEBUG(dbgs() << "VC: Replacing: " << Old << '\n');
@@ -5916,6 +5917,66 @@ bool VectorCombine::foldInsExtVectorToShuffle(Instruction &I) {
return true;
}
+/// Replace a partially used deinterleave of a load with strided shuffles of the
+/// loaded vector when it is cheaper than an interleaved load.
+bool VectorCombine::foldPartialDeinterleave(Instruction &I) {
+ auto *Deinterleave = dyn_cast<IntrinsicInst>(&I);
+ if (!Deinterleave)
+ return false;
+
+ unsigned Factor =
+ getDeinterleaveIntrinsicFactor(Deinterleave->getIntrinsicID());
+ if (!Factor || Deinterleave->hasOperandBundles() ||
+ !Deinterleave->hasOneUse())
+ return false;
+
+ auto *Load = dyn_cast<LoadInst>(Deinterleave->getArgOperand(0));
+ auto *LoadTy = Load ? dyn_cast<FixedVectorType>(Load->getType()) : nullptr;
+ if (!LoadTy || !Load->hasOneUse() || !Load->isSimple())
+ return false;
+
+ auto *ResultTy = dyn_cast<FixedVectorType>(
+ Deinterleave->getType()->getStructElementType(0));
+ if (!ResultTy ||
+ LoadTy->getNumElements() != Factor * ResultTy->getNumElements())
+ return false;
+
+ auto *Extract = dyn_cast<ExtractValueInst>(*Deinterleave->user_begin());
+ if (!Extract || Extract->getNumIndices() != 1)
+ return false;
+ unsigned Index = *Extract->idx_begin();
+ if (Index >= Factor)
+ return false;
+
+ SmallVector<int, 8> Mask;
+ for (unsigned Lane = 0; Lane != ResultTy->getNumElements(); ++Lane)
+ Mask.push_back(Index + Lane * Factor);
+
+ InstructionCost LoadCost = TTI.getInterleavedMemoryOpCost(
+ Instruction::Load, LoadTy, Factor, {}, Load->getAlign(),
+ Load->getPointerAddressSpace(), CostKind);
+
+ InstructionCost OldCost =
+ LoadCost * 2; // double the cost to compensate for shuffle
+
+ InstructionCost NewCost =
+ TTI.getMemoryOpCost(Instruction::Load, LoadTy, Load->getAlign(),
+ Load->getPointerAddressSpace(), CostKind) +
+ TTI.getShuffleCost(TTI::SK_PermuteSingleSrc, ResultTy, LoadTy, Mask,
+ CostKind);
+
+ LLVM_DEBUG(dbgs() << "VC: Found partially used deinterleave: " << I
+ << "\n OldCost: " << OldCost << " vs NewCost: " << NewCost
+ << "\n");
+ if (!OldCost.isValid() || !NewCost.isValid() || NewCost > OldCost)
+ return false;
+
+ Builder.SetInsertPoint(Deinterleave);
+ Value *Result = Builder.CreateShuffleVector(Load, Mask);
+ replaceValue(*Extract, *Result, false);
+ return true;
+}
+
/// Fold away a matched pair of vector.deinterleave/interleave intrinsics
/// with a chain of elementwise operations on each between the
/// deinterleave and interleave.
@@ -6215,6 +6276,8 @@ bool VectorCombine::foldInterleaveIntrinsics(Instruction &I) {
/// %merge1 = bitcast <vscale x 16 x i16> %f1 to <vscale x 8 x i32>
/// ```
bool VectorCombine::foldDeinterleaveIntrinsics(Instruction &I) {
+ if (foldPartialDeinterleave(I))
+ return true;
if (foldDeinterleaveInterleavePair(I))
return true;
diff --git a/llvm/test/Transforms/VectorCombine/AArch64/partial-deinterleave.ll b/llvm/test/Transforms/VectorCombine/AArch64/partial-deinterleave.ll
index 3814cc9a00dac..f6bfc77054411 100644
--- a/llvm/test/Transforms/VectorCombine/AArch64/partial-deinterleave.ll
+++ b/llvm/test/Transforms/VectorCombine/AArch64/partial-deinterleave.ll
@@ -5,11 +5,9 @@ define void @partial_deinterleave4_ptrs(ptr %src0, ptr %src1, ptr %dst0,
; CHECK-LABEL: define void @partial_deinterleave4_ptrs(
; CHECK-SAME: ptr [[SRC0:%.*]], ptr [[SRC1:%.*]], ptr [[DST0:%.*]], ptr [[DST1:%.*]]) {
; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <8 x ptr>, ptr [[SRC0]], align 8
-; CHECK-NEXT: [[STRIDED_VEC:%.*]] = call { <2 x ptr>, <2 x ptr>, <2 x ptr>, <2 x ptr> } @llvm.vector.deinterleave4.v8p0(<8 x ptr> [[WIDE_VEC]])
-; CHECK-NEXT: [[FIELD0:%.*]] = extractvalue { <2 x ptr>, <2 x ptr>, <2 x ptr>, <2 x ptr> } [[STRIDED_VEC]], 0
+; CHECK-NEXT: [[FIELD0:%.*]] = shufflevector <8 x ptr> [[WIDE_VEC]], <8 x ptr> poison, <2 x i32> <i32 0, i32 4>
; CHECK-NEXT: [[WIDE_VEC_1:%.*]] = load <8 x ptr>, ptr [[SRC1]], align 8
-; CHECK-NEXT: [[STRIDED_VEC_1:%.*]] = call { <2 x ptr>, <2 x ptr>, <2 x ptr>, <2 x ptr> } @llvm.vector.deinterleave4.v8p0(<8 x ptr> [[WIDE_VEC_1]])
-; CHECK-NEXT: [[FIELD0_1:%.*]] = extractvalue { <2 x ptr>, <2 x ptr>, <2 x ptr>, <2 x ptr> } [[STRIDED_VEC_1]], 0
+; CHECK-NEXT: [[FIELD0_1:%.*]] = shufflevector <8 x ptr> [[WIDE_VEC_1]], <8 x ptr> poison, <2 x i32> <i32 0, i32 4>
; CHECK-NEXT: store <2 x ptr> [[FIELD0]], ptr [[DST0]], align 8
; CHECK-NEXT: store <2 x ptr> [[FIELD0_1]], ptr [[DST1]], align 8
; CHECK-NEXT: ret void
@@ -30,8 +28,7 @@ define <2 x ptr> @partial_deinterleave4_field1(ptr %src) {
; CHECK-LABEL: define <2 x ptr> @partial_deinterleave4_field1(
; CHECK-SAME: ptr [[SRC:%.*]]) {
; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <8 x ptr>, ptr [[SRC]], align 8
-; CHECK-NEXT: [[STRIDED_VEC:%.*]] = call { <2 x ptr>, <2 x ptr>, <2 x ptr>, <2 x ptr> } @llvm.vector.deinterleave4.v8p0(<8 x ptr> [[WIDE_VEC]])
-; CHECK-NEXT: [[FIELD1:%.*]] = extractvalue { <2 x ptr>, <2 x ptr>, <2 x ptr>, <2 x ptr> } [[STRIDED_VEC]], 1
+; CHECK-NEXT: [[FIELD1:%.*]] = shufflevector <8 x ptr> [[WIDE_VEC]], <8 x ptr> poison, <2 x i32> <i32 1, i32 5>
; CHECK-NEXT: ret <2 x ptr> [[FIELD1]]
;
%wide.vec = load <8 x ptr>, ptr %src, align 8
More information about the llvm-commits
mailing list