[llvm] [VectorCombine] Fold interleave of high-half truncations (PR #211022)
Jacob Crawley via llvm-commits
llvm-commits at lists.llvm.org
Wed Jul 22 07:58:24 PDT 2026
https://github.com/jacob-crawley updated https://github.com/llvm/llvm-project/pull/211022
>From b7a4e30e03c4285b5ed954035072f8d5bdaa9037 Mon Sep 17 00:00:00 2001
From: Jacob Crawley <jacob.crawley at arm.com>
Date: Tue, 21 Jul 2026 14:53:59 +0000
Subject: [PATCH 1/2] [VectorCombine] Fold interleave of high-half truncations
Fold an interleave of shifted and truncated deinterleave results into a
deinterleave of the original vector at half the element width.
This replaces a deinterlave/interleave sequence and shift/truncates
with a single half-width deinterleave, enabling improved SVE codegen on
AArch64 targets.
---
.../Transforms/Vectorize/VectorCombine.cpp | 105 +++++++++
.../AArch64/sve-vectorcombine-interleave.ll | 65 ++++++
.../VectorCombine/fold-interleave.ll | 209 ++++++++++++++++++
3 files changed, 379 insertions(+)
create mode 100644 llvm/test/CodeGen/AArch64/sve-vectorcombine-interleave.ll
create mode 100644 llvm/test/Transforms/VectorCombine/fold-interleave.ll
diff --git a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
index 741bd4460a5ab..759c826b6e7a7 100644
--- a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
+++ b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
@@ -155,6 +155,7 @@ class VectorCombine {
bool foldEquivalentReductionCmp(Instruction &I);
bool foldReduceAddCmpZero(Instruction &I);
bool foldSelectShuffle(Instruction &I, bool FromReduction = false);
+ bool foldInterleaveOfHighHalfTruncs(Instruction &I);
bool foldInterleaveIntrinsics(Instruction &I);
bool foldDeinterleaveIntrinsics(Instruction &I);
bool foldBitcastOfVPLoad(Instruction &I);
@@ -5882,6 +5883,108 @@ bool VectorCombine::foldInsExtVectorToShuffle(Instruction &I) {
return true;
}
+/// Fold an interleave of shifted, truncated deinterleave results into a
+/// deinterleave of the original vector at half the element width, directly
+/// selecting the high half of each source element.
+///
+/// ```
+/// %d = call { <vscale x 2 x i16>, ... }
+/// @llvm.vector.deinterleave4(<vscale x 8 x i16> %x)
+/// %d0 = extractvalue { <vscale x 2 x i16>, ... } %d, 0
+/// %d1 = extractvalue { <vscale x 2 x i16>, ... } %d, 1
+/// %d2 = extractvalue { <vscale x 2 x i16>, ... } %d, 2
+/// %d3 = extractvalue { <vscale x 2 x i16>, ... } %d, 3
+/// %s0 = lshr <vscale x 2 x i16> %d0, splat (i16 8)
+/// %s1 = lshr <vscale x 2 x i16> %d1, splat (i16 8)
+/// %s2 = lshr <vscale x 2 x i16> %d2, splat (i16 8)
+/// %s3 = lshr <vscale x 2 x i16> %d3, splat (i16 8)
+/// %t0 = trunc <vscale x 2 x i16> %s0 to <vscale x 2 x i8>
+/// %t1 = trunc <vscale x 2 x i16> %s1 to <vscale x 2 x i8>
+/// %t2 = trunc <vscale x 2 x i16> %s2 to <vscale x 2 x i8>
+/// %t3 = trunc <vscale x 2 x i16> %s3 to <vscale x 2 x i8>
+/// %r = call <vscale x 8 x i8> @llvm.vector.interleave4(%t0, %t1, %t2, %t3)
+/// ```
+/// becomes:
+///
+/// ```
+/// %bc = bitcast <vscale x 8 x i16> %x to <vscale x 16 x i8>
+/// %d = call { <vscale x 8 x i8>, <vscale x 8 x i8> }
+/// @llvm.vector.deinterleave2(<vscale x 16 x i8> %bc)
+/// %r = extractvalue { <vscale x 8 x i8>, <vscale x 8 x i8> } %d, 1
+/// ```
+bool VectorCombine::foldInterleaveOfHighHalfTruncs(Instruction &I) {
+ auto *Interleave = dyn_cast<IntrinsicInst>(&I);
+ if (!Interleave || Interleave->hasOperandBundles())
+ return false;
+
+ unsigned Factor = getInterleaveIntrinsicFactor(Interleave->getIntrinsicID());
+ if (!Factor || Interleave->arg_size() != Factor)
+ return false;
+
+ auto *InterleaveTy = dyn_cast<VectorType>(Interleave->getType());
+ if (!InterleaveTy || !InterleaveTy->getElementType()->isIntegerTy())
+ return false;
+
+ unsigned DstEltBits = InterleaveTy->getScalarSizeInBits();
+ Intrinsic::ID DeinterleaveID = Intrinsic::getDeinterleaveIntrinsicID(Factor);
+ IntrinsicInst *Deinterleave = nullptr;
+
+ for (unsigned Idx = 0; Idx != Factor; ++Idx) {
+ auto *Trunc = dyn_cast<TruncInst>(Interleave->getArgOperand(Idx));
+ if (!Trunc || !Trunc->hasOneUse())
+ return false;
+
+ auto *Shift = dyn_cast<BinaryOperator>(Trunc->getOperand(0));
+ if (!Shift || !Shift->hasOneUse() ||
+ (Shift->getOpcode() != Instruction::LShr &&
+ Shift->getOpcode() != Instruction::AShr) ||
+ !match(Shift->getOperand(1), m_SpecificInt(DstEltBits)))
+ return false;
+
+ // Each interleave operand must come from the corresponding deinterleave
+ // result.
+ auto *Extract = dyn_cast<ExtractValueInst>(Shift->getOperand(0));
+ if (!Extract || !Extract->hasOneUse() || Extract->getNumIndices() != 1 ||
+ *Extract->idx_begin() != Idx)
+ return false;
+
+ auto *DI = dyn_cast<IntrinsicInst>(Extract->getAggregateOperand());
+ if (!DI || DI->hasOperandBundles() ||
+ DI->getIntrinsicID() != DeinterleaveID)
+ return false;
+
+ if (!Deinterleave)
+ Deinterleave = DI;
+ else if (DI != Deinterleave)
+ return false;
+ }
+
+ // Require the entire matched tree to become dead, otherwise the replacement
+ // will add another shuffle sequence to the remaining operations.
+ if (!Deinterleave || !Deinterleave->hasNUses(Factor))
+ return false;
+
+ Value *Source = Deinterleave->getArgOperand(0);
+ auto *SourceTy = dyn_cast<VectorType>(Source->getType());
+ if (!SourceTy ||
+ SourceTy->getElementCount() != InterleaveTy->getElementCount() ||
+ !SourceTy->getElementType()->isIntegerTy() ||
+ SourceTy->getScalarSizeInBits() != 2 * DstEltBits)
+ return false;
+
+ auto *BitcastTy = VectorType::getDoubleElementsVectorType(InterleaveTy);
+ Value *Bitcast = Builder.CreateBitCast(Source, BitcastTy);
+ Value *NewDeinterleave = Builder.CreateIntrinsic(
+ Intrinsic::vector_deinterleave2, {BitcastTy}, {Bitcast});
+ // A vector bitcast orders each element's low half first on little-endian
+ // targets and its high half first on big-endian targets.
+ unsigned HighHalfIndex = DL->isLittleEndian() ? 1 : 0;
+ Value *HighHalves =
+ Builder.CreateExtractValue(NewDeinterleave, HighHalfIndex);
+ replaceValue(I, *HighHalves);
+ return true;
+}
+
/// If we're interleaving 2 constant splats, for instance `<vscale x 8 x i32>
/// <splat of 666>` and `<vscale x 8 x i32> <splat of 777>`, we can create a
/// larger splat `<vscale x 8 x i64> <splat of ((777 << 32) | 666)>` first
@@ -6476,6 +6579,8 @@ bool VectorCombine::run() {
return true;
if (scalarizeVPIntrinsic(I))
return true;
+ if (foldInterleaveOfHighHalfTruncs(I))
+ return true;
if (foldInterleaveIntrinsics(I))
return true;
if (foldBitcastOfVPLoad(I))
diff --git a/llvm/test/CodeGen/AArch64/sve-vectorcombine-interleave.ll b/llvm/test/CodeGen/AArch64/sve-vectorcombine-interleave.ll
new file mode 100644
index 0000000000000..731bb482bcb34
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-vectorcombine-interleave.ll
@@ -0,0 +1,65 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -passes=vector-combine -mattr=+sve %s -o - | llc -mattr=+sve -o - | FileCheck %s
+
+target triple = "aarch64"
+
+define void @high_half_trunc_interleave4(ptr %src, ptr %dst) {
+; CHECK-LABEL: high_half_trunc_interleave4:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ldr z0, [x0, #1, mul vl]
+; CHECK-NEXT: ldr z1, [x0]
+; CHECK-NEXT: uzp2 z0.b, z1.b, z0.b
+; CHECK-NEXT: str z0, [x1]
+; CHECK-NEXT: ret
+ %x = load <vscale x 16 x i16>, ptr %src, align 2
+ %d = call { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } @llvm.vector.deinterleave4.nxv16i16(<vscale x 16 x i16> %x)
+ %d0 = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } %d, 0
+ %d1 = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } %d, 1
+ %d2 = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } %d, 2
+ %d3 = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } %d, 3
+ %s0 = lshr <vscale x 4 x i16> %d0, splat (i16 8)
+ %s1 = lshr <vscale x 4 x i16> %d1, splat (i16 8)
+ %s2 = lshr <vscale x 4 x i16> %d2, splat (i16 8)
+ %s3 = lshr <vscale x 4 x i16> %d3, splat (i16 8)
+ %t0 = trunc <vscale x 4 x i16> %s0 to <vscale x 4 x i8>
+ %t1 = trunc <vscale x 4 x i16> %s1 to <vscale x 4 x i8>
+ %t2 = trunc <vscale x 4 x i16> %s2 to <vscale x 4 x i8>
+ %t3 = trunc <vscale x 4 x i16> %s3 to <vscale x 4 x i8>
+ %r = call <vscale x 16 x i8> @llvm.vector.interleave4.nxv16i8(<vscale x 4 x i8> %t0, <vscale x 4 x i8> %t1, <vscale x 4 x i8> %t2, <vscale x 4 x i8> %t3)
+ store <vscale x 16 x i8> %r, ptr %dst, align 1
+ ret void
+}
+
+; Keep the load and bitcast in separate blocks so VectorCombine
+; exposes a byte deinterleave2, which can be selected as a ld2b.
+define void @high_half_trunc_interleave4_byte_load(ptr %src, ptr %dst) {
+; CHECK-LABEL: high_half_trunc_interleave4_byte_load:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ptrue p0.b
+; CHECK-NEXT: ld2b { z0.b, z1.b }, p0/z, [x0]
+; CHECK-NEXT: str z1, [x1]
+; CHECK-NEXT: ret
+entry:
+ %bytes = load <vscale x 32 x i8>, ptr %src, align 2
+ br label %body
+
+body:
+ %x = bitcast <vscale x 32 x i8> %bytes to <vscale x 16 x i16>
+ %d = call { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } @llvm.vector.deinterleave4.nxv16i16(<vscale x 16 x i16> %x)
+ %d0 = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } %d, 0
+ %d1 = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } %d, 1
+ %d2 = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } %d, 2
+ %d3 = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } %d, 3
+ %s0 = lshr <vscale x 4 x i16> %d0, splat (i16 8)
+ %s1 = lshr <vscale x 4 x i16> %d1, splat (i16 8)
+ %s2 = lshr <vscale x 4 x i16> %d2, splat (i16 8)
+ %s3 = lshr <vscale x 4 x i16> %d3, splat (i16 8)
+ %t0 = trunc <vscale x 4 x i16> %s0 to <vscale x 4 x i8>
+ %t1 = trunc <vscale x 4 x i16> %s1 to <vscale x 4 x i8>
+ %t2 = trunc <vscale x 4 x i16> %s2 to <vscale x 4 x i8>
+ %t3 = trunc <vscale x 4 x i16> %s3 to <vscale x 4 x i8>
+ %r = call <vscale x 16 x i8> @llvm.vector.interleave4.nxv16i8(<vscale x 4 x i8> %t0, <vscale x 4 x i8> %t1, <vscale x 4 x i8> %t2, <vscale x 4 x i8> %t3)
+ store <vscale x 16 x i8> %r, ptr %dst, align 1
+ ret void
+}
+
diff --git a/llvm/test/Transforms/VectorCombine/fold-interleave.ll b/llvm/test/Transforms/VectorCombine/fold-interleave.ll
new file mode 100644
index 0000000000000..efe51333c024a
--- /dev/null
+++ b/llvm/test/Transforms/VectorCombine/fold-interleave.ll
@@ -0,0 +1,209 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -passes=vector-combine -S %s | FileCheck %s
+; RUN: opt -passes=vector-combine -data-layout=E -S %s | FileCheck %s --check-prefixes=BE
+
+define <vscale x 8 x i8> @lshr_trunc_interleave4(<vscale x 8 x i16> %x) {
+; CHECK-LABEL: define <vscale x 8 x i8> @lshr_trunc_interleave4(
+; CHECK-SAME: <vscale x 8 x i16> [[X:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast <vscale x 8 x i16> [[X]] to <vscale x 16 x i8>
+; CHECK-NEXT: [[TMP2:%.*]] = call { <vscale x 8 x i8>, <vscale x 8 x i8> } @llvm.vector.deinterleave2.nxv16i8(<vscale x 16 x i8> [[TMP1]])
+; CHECK-NEXT: [[R:%.*]] = extractvalue { <vscale x 8 x i8>, <vscale x 8 x i8> } [[TMP2]], 1
+; CHECK-NEXT: ret <vscale x 8 x i8> [[R]]
+;
+; BE-LABEL: define <vscale x 8 x i8> @lshr_trunc_interleave4(
+; BE-SAME: <vscale x 8 x i16> [[X:%.*]]) {
+; BE-NEXT: [[TMP1:%.*]] = bitcast <vscale x 8 x i16> [[X]] to <vscale x 16 x i8>
+; BE-NEXT: [[TMP2:%.*]] = call { <vscale x 8 x i8>, <vscale x 8 x i8> } @llvm.vector.deinterleave2.nxv16i8(<vscale x 16 x i8> [[TMP1]])
+; BE-NEXT: [[R:%.*]] = extractvalue { <vscale x 8 x i8>, <vscale x 8 x i8> } [[TMP2]], 0
+; BE-NEXT: ret <vscale x 8 x i8> [[R]]
+;
+ %d = call { <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16> } @llvm.vector.deinterleave4.nxv8i16(<vscale x 8 x i16> %x)
+ %d0 = extractvalue { <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16> } %d, 0
+ %d1 = extractvalue { <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16> } %d, 1
+ %d2 = extractvalue { <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16> } %d, 2
+ %d3 = extractvalue { <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16> } %d, 3
+ %s0 = lshr <vscale x 2 x i16> %d0, splat (i16 8)
+ %s1 = lshr <vscale x 2 x i16> %d1, splat (i16 8)
+ %s2 = lshr <vscale x 2 x i16> %d2, splat (i16 8)
+ %s3 = lshr <vscale x 2 x i16> %d3, splat (i16 8)
+ %t0 = trunc <vscale x 2 x i16> %s0 to <vscale x 2 x i8>
+ %t1 = trunc <vscale x 2 x i16> %s1 to <vscale x 2 x i8>
+ %t2 = trunc <vscale x 2 x i16> %s2 to <vscale x 2 x i8>
+ %t3 = trunc <vscale x 2 x i16> %s3 to <vscale x 2 x i8>
+ %r = call <vscale x 8 x i8> @llvm.vector.interleave4.nxv8i8(<vscale x 2 x i8> %t0, <vscale x 2 x i8> %t1, <vscale x 2 x i8> %t2, <vscale x 2 x i8> %t3)
+ ret <vscale x 8 x i8> %r
+}
+
+define <8 x i8> @fixed_lshr_trunc_interleave4(<8 x i16> %x) {
+; CHECK-LABEL: define <8 x i8> @fixed_lshr_trunc_interleave4(
+; CHECK-SAME: <8 x i16> [[X:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast <8 x i16> [[X]] to <16 x i8>
+; CHECK-NEXT: [[TMP2:%.*]] = call { <8 x i8>, <8 x i8> } @llvm.vector.deinterleave2.v16i8(<16 x i8> [[TMP1]])
+; CHECK-NEXT: [[R:%.*]] = extractvalue { <8 x i8>, <8 x i8> } [[TMP2]], 1
+; CHECK-NEXT: ret <8 x i8> [[R]]
+;
+; BE-LABEL: define <8 x i8> @fixed_lshr_trunc_interleave4(
+; BE-SAME: <8 x i16> [[X:%.*]]) {
+; BE-NEXT: [[TMP1:%.*]] = bitcast <8 x i16> [[X]] to <16 x i8>
+; BE-NEXT: [[TMP2:%.*]] = call { <8 x i8>, <8 x i8> } @llvm.vector.deinterleave2.v16i8(<16 x i8> [[TMP1]])
+; BE-NEXT: [[R:%.*]] = extractvalue { <8 x i8>, <8 x i8> } [[TMP2]], 0
+; BE-NEXT: ret <8 x i8> [[R]]
+;
+ %d = call { <2 x i16>, <2 x i16>, <2 x i16>, <2 x i16> } @llvm.vector.deinterleave4.v8i16(<8 x i16> %x)
+ %d0 = extractvalue { <2 x i16>, <2 x i16>, <2 x i16>, <2 x i16> } %d, 0
+ %d1 = extractvalue { <2 x i16>, <2 x i16>, <2 x i16>, <2 x i16> } %d, 1
+ %d2 = extractvalue { <2 x i16>, <2 x i16>, <2 x i16>, <2 x i16> } %d, 2
+ %d3 = extractvalue { <2 x i16>, <2 x i16>, <2 x i16>, <2 x i16> } %d, 3
+ %s0 = lshr <2 x i16> %d0, splat (i16 8)
+ %s1 = lshr <2 x i16> %d1, splat (i16 8)
+ %s2 = lshr <2 x i16> %d2, splat (i16 8)
+ %s3 = lshr <2 x i16> %d3, splat (i16 8)
+ %t0 = trunc <2 x i16> %s0 to <2 x i8>
+ %t1 = trunc <2 x i16> %s1 to <2 x i8>
+ %t2 = trunc <2 x i16> %s2 to <2 x i8>
+ %t3 = trunc <2 x i16> %s3 to <2 x i8>
+ %r = call <8 x i8> @llvm.vector.interleave4.v8i8(<2 x i8> %t0, <2 x i8> %t1, <2 x i8> %t2, <2 x i8> %t3)
+ ret <8 x i8> %r
+}
+
+define <vscale x 4 x i16> @mixed_shifts_trunc_interleave2(<vscale x 4 x i32> %x) {
+; CHECK-LABEL: define <vscale x 4 x i16> @mixed_shifts_trunc_interleave2(
+; CHECK-SAME: <vscale x 4 x i32> [[X:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast <vscale x 4 x i32> [[X]] to <vscale x 8 x i16>
+; CHECK-NEXT: [[TMP2:%.*]] = call { <vscale x 4 x i16>, <vscale x 4 x i16> } @llvm.vector.deinterleave2.nxv8i16(<vscale x 8 x i16> [[TMP1]])
+; CHECK-NEXT: [[R:%.*]] = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16> } [[TMP2]], 1
+; CHECK-NEXT: ret <vscale x 4 x i16> [[R]]
+;
+; BE-LABEL: define <vscale x 4 x i16> @mixed_shifts_trunc_interleave2(
+; BE-SAME: <vscale x 4 x i32> [[X:%.*]]) {
+; BE-NEXT: [[TMP1:%.*]] = bitcast <vscale x 4 x i32> [[X]] to <vscale x 8 x i16>
+; BE-NEXT: [[TMP2:%.*]] = call { <vscale x 4 x i16>, <vscale x 4 x i16> } @llvm.vector.deinterleave2.nxv8i16(<vscale x 8 x i16> [[TMP1]])
+; BE-NEXT: [[R:%.*]] = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16> } [[TMP2]], 0
+; BE-NEXT: ret <vscale x 4 x i16> [[R]]
+;
+ %d = call { <vscale x 2 x i32>, <vscale x 2 x i32> } @llvm.vector.deinterleave2.nxv4i32(<vscale x 4 x i32> %x)
+ %d0 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } %d, 0
+ %d1 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } %d, 1
+ %s0 = lshr exact <vscale x 2 x i32> %d0, splat (i32 16)
+ %s1 = ashr exact <vscale x 2 x i32> %d1, splat (i32 16)
+ %t0 = trunc nuw <vscale x 2 x i32> %s0 to <vscale x 2 x i16>
+ %t1 = trunc nuw <vscale x 2 x i32> %s1 to <vscale x 2 x i16>
+ %r = call <vscale x 4 x i16> @llvm.vector.interleave2.nxv4i16(<vscale x 2 x i16> %t0, <vscale x 2 x i16> %t1)
+ ret <vscale x 4 x i16> %r
+}
+
+; Negative test - the fold should only be applied if the shifts are equal.
+define <vscale x 4 x i16> @different_shift_amounts(<vscale x 4 x i32> %x) {
+; CHECK-LABEL: define <vscale x 4 x i16> @different_shift_amounts(
+; CHECK-SAME: <vscale x 4 x i32> [[X:%.*]]) {
+; CHECK-NEXT: [[D:%.*]] = call { <vscale x 2 x i32>, <vscale x 2 x i32> } @llvm.vector.deinterleave2.nxv4i32(<vscale x 4 x i32> [[X]])
+; CHECK-NEXT: [[D0:%.*]] = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } [[D]], 0
+; CHECK-NEXT: [[D1:%.*]] = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } [[D]], 1
+; CHECK-NEXT: [[S0:%.*]] = lshr <vscale x 2 x i32> [[D0]], splat (i32 16)
+; CHECK-NEXT: [[S1:%.*]] = lshr <vscale x 2 x i32> [[D1]], splat (i32 15)
+; CHECK-NEXT: [[T0:%.*]] = trunc <vscale x 2 x i32> [[S0]] to <vscale x 2 x i16>
+; CHECK-NEXT: [[T1:%.*]] = trunc <vscale x 2 x i32> [[S1]] to <vscale x 2 x i16>
+; CHECK-NEXT: [[R:%.*]] = call <vscale x 4 x i16> @llvm.vector.interleave2.nxv4i16(<vscale x 2 x i16> [[T0]], <vscale x 2 x i16> [[T1]])
+; CHECK-NEXT: ret <vscale x 4 x i16> [[R]]
+;
+; BE-LABEL: define <vscale x 4 x i16> @different_shift_amounts(
+; BE-SAME: <vscale x 4 x i32> [[X:%.*]]) {
+; BE-NEXT: [[D:%.*]] = call { <vscale x 2 x i32>, <vscale x 2 x i32> } @llvm.vector.deinterleave2.nxv4i32(<vscale x 4 x i32> [[X]])
+; BE-NEXT: [[D0:%.*]] = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } [[D]], 0
+; BE-NEXT: [[D1:%.*]] = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } [[D]], 1
+; BE-NEXT: [[S0:%.*]] = lshr <vscale x 2 x i32> [[D0]], splat (i32 16)
+; BE-NEXT: [[S1:%.*]] = lshr <vscale x 2 x i32> [[D1]], splat (i32 15)
+; BE-NEXT: [[T0:%.*]] = trunc <vscale x 2 x i32> [[S0]] to <vscale x 2 x i16>
+; BE-NEXT: [[T1:%.*]] = trunc <vscale x 2 x i32> [[S1]] to <vscale x 2 x i16>
+; BE-NEXT: [[R:%.*]] = call <vscale x 4 x i16> @llvm.vector.interleave2.nxv4i16(<vscale x 2 x i16> [[T0]], <vscale x 2 x i16> [[T1]])
+; BE-NEXT: ret <vscale x 4 x i16> [[R]]
+;
+ %d = call { <vscale x 2 x i32>, <vscale x 2 x i32> } @llvm.vector.deinterleave2.nxv4i32(<vscale x 4 x i32> %x)
+ %d0 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } %d, 0
+ %d1 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } %d, 1
+ %s0 = lshr <vscale x 2 x i32> %d0, splat (i32 16)
+ %s1 = lshr <vscale x 2 x i32> %d1, splat (i32 15)
+ %t0 = trunc <vscale x 2 x i32> %s0 to <vscale x 2 x i16>
+ %t1 = trunc <vscale x 2 x i32> %s1 to <vscale x 2 x i16>
+ %r = call <vscale x 4 x i16> @llvm.vector.interleave2.nxv4i16(<vscale x 2 x i16> %t0, <vscale x 2 x i16> %t1)
+ ret <vscale x 4 x i16> %r
+}
+
+; Negative test - the fold shouldn't be generated as the deitnerleaved fields are passed in the wrong order.
+define <vscale x 4 x i16> @reordered_fields(<vscale x 4 x i32> %x) {
+; CHECK-LABEL: define <vscale x 4 x i16> @reordered_fields(
+; CHECK-SAME: <vscale x 4 x i32> [[X:%.*]]) {
+; CHECK-NEXT: [[D:%.*]] = call { <vscale x 2 x i32>, <vscale x 2 x i32> } @llvm.vector.deinterleave2.nxv4i32(<vscale x 4 x i32> [[X]])
+; CHECK-NEXT: [[D0:%.*]] = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } [[D]], 0
+; CHECK-NEXT: [[D1:%.*]] = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } [[D]], 1
+; CHECK-NEXT: [[S0:%.*]] = lshr <vscale x 2 x i32> [[D0]], splat (i32 16)
+; CHECK-NEXT: [[S1:%.*]] = lshr <vscale x 2 x i32> [[D1]], splat (i32 16)
+; CHECK-NEXT: [[T0:%.*]] = trunc <vscale x 2 x i32> [[S0]] to <vscale x 2 x i16>
+; CHECK-NEXT: [[T1:%.*]] = trunc <vscale x 2 x i32> [[S1]] to <vscale x 2 x i16>
+; CHECK-NEXT: [[R:%.*]] = call <vscale x 4 x i16> @llvm.vector.interleave2.nxv4i16(<vscale x 2 x i16> [[T1]], <vscale x 2 x i16> [[T0]])
+; CHECK-NEXT: ret <vscale x 4 x i16> [[R]]
+;
+; BE-LABEL: define <vscale x 4 x i16> @reordered_fields(
+; BE-SAME: <vscale x 4 x i32> [[X:%.*]]) {
+; BE-NEXT: [[D:%.*]] = call { <vscale x 2 x i32>, <vscale x 2 x i32> } @llvm.vector.deinterleave2.nxv4i32(<vscale x 4 x i32> [[X]])
+; BE-NEXT: [[D0:%.*]] = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } [[D]], 0
+; BE-NEXT: [[D1:%.*]] = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } [[D]], 1
+; BE-NEXT: [[S0:%.*]] = lshr <vscale x 2 x i32> [[D0]], splat (i32 16)
+; BE-NEXT: [[S1:%.*]] = lshr <vscale x 2 x i32> [[D1]], splat (i32 16)
+; BE-NEXT: [[T0:%.*]] = trunc <vscale x 2 x i32> [[S0]] to <vscale x 2 x i16>
+; BE-NEXT: [[T1:%.*]] = trunc <vscale x 2 x i32> [[S1]] to <vscale x 2 x i16>
+; BE-NEXT: [[R:%.*]] = call <vscale x 4 x i16> @llvm.vector.interleave2.nxv4i16(<vscale x 2 x i16> [[T1]], <vscale x 2 x i16> [[T0]])
+; BE-NEXT: ret <vscale x 4 x i16> [[R]]
+;
+ %d = call { <vscale x 2 x i32>, <vscale x 2 x i32> } @llvm.vector.deinterleave2.nxv4i32(<vscale x 4 x i32> %x)
+ %d0 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } %d, 0
+ %d1 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } %d, 1
+ %s0 = lshr <vscale x 2 x i32> %d0, splat (i32 16)
+ %s1 = lshr <vscale x 2 x i32> %d1, splat (i32 16)
+ %t0 = trunc <vscale x 2 x i32> %s0 to <vscale x 2 x i16>
+ %t1 = trunc <vscale x 2 x i32> %s1 to <vscale x 2 x i16>
+ %r = call <vscale x 4 x i16> @llvm.vector.interleave2.nxv4i16(<vscale x 2 x i16> %t1, <vscale x 2 x i16> %t0)
+ ret <vscale x 4 x i16> %r
+}
+
+; Negative test - the fold shouldn't be generated as the trunc has more than one use.
+define <vscale x 4 x i16> @extra_trunc_use(
+; CHECK-LABEL: define <vscale x 4 x i16> @extra_trunc_use(
+; CHECK-SAME: <vscale x 4 x i32> [[X:%.*]], ptr [[DST:%.*]]) {
+; CHECK-NEXT: [[D:%.*]] = call { <vscale x 2 x i32>, <vscale x 2 x i32> } @llvm.vector.deinterleave2.nxv4i32(<vscale x 4 x i32> [[X]])
+; CHECK-NEXT: [[D0:%.*]] = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } [[D]], 0
+; CHECK-NEXT: [[D1:%.*]] = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } [[D]], 1
+; CHECK-NEXT: [[S0:%.*]] = lshr <vscale x 2 x i32> [[D0]], splat (i32 16)
+; CHECK-NEXT: [[S1:%.*]] = lshr <vscale x 2 x i32> [[D1]], splat (i32 16)
+; CHECK-NEXT: [[T0:%.*]] = trunc <vscale x 2 x i32> [[S0]] to <vscale x 2 x i16>
+; CHECK-NEXT: [[T1:%.*]] = trunc <vscale x 2 x i32> [[S1]] to <vscale x 2 x i16>
+; CHECK-NEXT: store <vscale x 2 x i16> [[T0]], ptr [[DST]], align 4
+; CHECK-NEXT: [[R:%.*]] = call <vscale x 4 x i16> @llvm.vector.interleave2.nxv4i16(<vscale x 2 x i16> [[T0]], <vscale x 2 x i16> [[T1]])
+; CHECK-NEXT: ret <vscale x 4 x i16> [[R]]
+;
+; BE-LABEL: define <vscale x 4 x i16> @extra_trunc_use(
+; BE-SAME: <vscale x 4 x i32> [[X:%.*]], ptr [[DST:%.*]]) {
+; BE-NEXT: [[D:%.*]] = call { <vscale x 2 x i32>, <vscale x 2 x i32> } @llvm.vector.deinterleave2.nxv4i32(<vscale x 4 x i32> [[X]])
+; BE-NEXT: [[D0:%.*]] = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } [[D]], 0
+; BE-NEXT: [[D1:%.*]] = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } [[D]], 1
+; BE-NEXT: [[S0:%.*]] = lshr <vscale x 2 x i32> [[D0]], splat (i32 16)
+; BE-NEXT: [[S1:%.*]] = lshr <vscale x 2 x i32> [[D1]], splat (i32 16)
+; BE-NEXT: [[T0:%.*]] = trunc <vscale x 2 x i32> [[S0]] to <vscale x 2 x i16>
+; BE-NEXT: [[T1:%.*]] = trunc <vscale x 2 x i32> [[S1]] to <vscale x 2 x i16>
+; BE-NEXT: store <vscale x 2 x i16> [[T0]], ptr [[DST]], align 4
+; BE-NEXT: [[R:%.*]] = call <vscale x 4 x i16> @llvm.vector.interleave2.nxv4i16(<vscale x 2 x i16> [[T0]], <vscale x 2 x i16> [[T1]])
+; BE-NEXT: ret <vscale x 4 x i16> [[R]]
+;
+ <vscale x 4 x i32> %x, ptr %dst) {
+ %d = call { <vscale x 2 x i32>, <vscale x 2 x i32> } @llvm.vector.deinterleave2.nxv4i32(<vscale x 4 x i32> %x)
+ %d0 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } %d, 0
+ %d1 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } %d, 1
+ %s0 = lshr <vscale x 2 x i32> %d0, splat (i32 16)
+ %s1 = lshr <vscale x 2 x i32> %d1, splat (i32 16)
+ %t0 = trunc <vscale x 2 x i32> %s0 to <vscale x 2 x i16>
+ %t1 = trunc <vscale x 2 x i32> %s1 to <vscale x 2 x i16>
+ store <vscale x 2 x i16> %t0, ptr %dst
+ %r = call <vscale x 4 x i16> @llvm.vector.interleave2.nxv4i16(<vscale x 2 x i16> %t0, <vscale x 2 x i16> %t1)
+ ret <vscale x 4 x i16> %r
+}
+
>From c01172ec6b55ae5e35b5b995d09f5628657b8c78 Mon Sep 17 00:00:00 2001
From: Jacob Crawley <jacob.crawley at arm.com>
Date: Wed, 22 Jul 2026 14:54:21 +0000
Subject: [PATCH 2/2] rm cost model increase
---
.../AArch64/AArch64TargetTransformInfo.cpp | 13 +---
.../AArch64/sve-interleave-low-vf-cost.ll | 60 -------------------
2 files changed, 2 insertions(+), 71 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index f462ef8b2c295..a86ef715aba16 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -5463,17 +5463,8 @@ InstructionCost AArch64TTIImpl::getInterleavedMemoryOpCost(
// operations. The tree has Log2(Factor) levels, with Factor UZP/ZIP
// operations at each level, giving a total shuffle cost of
// Factor * Log2(Factor).
- auto SubVecCost = getTypeLegalizationCost(SubVecTy);
- auto ResultCost = getTypeLegalizationCost(VecTy);
- llvm::InstructionCost LegalizationCost = SubVecCost.first;
-
- // FIXME: A temporary increase to the cost in cases where the input
- // element type is 4x the output type. Otherwise it produces an SVE tail
- // loop which is significantly larger than the NEON equivalent.
- if (Opcode == Instruction::Store && Factor == 4 &&
- SubVecCost.second.getScalarSizeInBits() ==
- (4 * ResultCost.second.getScalarSizeInBits()))
- LegalizationCost *= 4;
+ llvm::InstructionCost LegalizationCost =
+ getTypeLegalizationCost(SubVecTy).first;
return MemCost + (Factor * LegalizationCost) + (Factor * Log2_64(Factor));
}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleave-low-vf-cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleave-low-vf-cost.ll
index 2bd9a42404a5b..16dc321ed4056 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleave-low-vf-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleave-low-vf-cost.ll
@@ -70,64 +70,4 @@ exit:
ret void
}
-; Check that the increased low-VF interleaved-store cost prevents selection of
-; an SVE epilogue.
-
-; For VF vscale x 4:
-; load cost = 2 + (4 * 1) + (4 * 2) = 14
-; store cost = 1 + (4 * 4) + (4 * 2) = 25
-;
-; This makes the fixed VF 8 epilogue preferable to VF vscale x 4.
-;
-; CHECK-LABEL: LV: Checking a loop in 'deinterleave4_nxv4i16_load_interleave4_nxv4i8_store'
-; CHECK: Cost of 14 for VF vscale x 4: INTERLEAVE-GROUP with factor 4
-; CHECK: Cost of 25 for VF vscale x 4: INTERLEAVE-GROUP with factor 4
-; CHECK: LV: Selecting VF: vscale x 16
-; CHECK: LEV: Vectorizing epilogue loop with VF = 8
-define void @deinterleave4_nxv4i16_load_interleave4_nxv4i8_store(
- ptr readonly %src, ptr writeonly %out, i32 %n) #0 {
-entry:
- %empty = icmp eq i32 %n, 0
- br i1 %empty, label %exit, label %loop
-
-loop:
- %src.iv = phi ptr [ %src.next, %loop ], [ %src, %entry ]
- %out.iv = phi ptr [ %out.next, %loop ], [ %out, %entry ]
- %iv = phi i32 [ %iv.next, %loop ], [ %n, %entry ]
-
- %ptr.g = getelementptr inbounds i16, ptr %src.iv, i64 1
- %ptr.r = getelementptr inbounds i16, ptr %src.iv, i64 2
- %ptr.a = getelementptr inbounds i16, ptr %src.iv, i64 3
- %load.b = load i16, ptr %src.iv, align 2
- %load.g = load i16, ptr %ptr.g, align 2
- %load.r = load i16, ptr %ptr.r, align 2
- %load.a = load i16, ptr %ptr.a, align 2
-
- %shift.b = lshr i16 %load.b, 8
- %shift.g = lshr i16 %load.g, 8
- %shift.r = lshr i16 %load.r, 8
- %shift.a = lshr i16 %load.a, 8
- %trunc.b = trunc nuw i16 %shift.b to i8
- %trunc.g = trunc nuw i16 %shift.g to i8
- %trunc.r = trunc nuw i16 %shift.r to i8
- %trunc.a = trunc nuw i16 %shift.a to i8
-
- %out.g = getelementptr inbounds i8, ptr %out.iv, i64 1
- %out.r = getelementptr inbounds i8, ptr %out.iv, i64 2
- %out.a = getelementptr inbounds i8, ptr %out.iv, i64 3
- store i8 %trunc.b, ptr %out.iv, align 1
- store i8 %trunc.g, ptr %out.g, align 1
- store i8 %trunc.r, ptr %out.r, align 1
- store i8 %trunc.a, ptr %out.a, align 1
-
- %src.next = getelementptr inbounds i16, ptr %src.iv, i64 4
- %out.next = getelementptr inbounds i8, ptr %out.iv, i64 4
- %iv.next = add nsw i32 %iv, -1
- %done = icmp eq i32 %iv.next, 0
- br i1 %done, label %exit, label %loop
-
-exit:
- ret void
-}
-
attributes #0 = { "target-features"="+sve" }
More information about the llvm-commits
mailing list