[llvm] [VectorCombine] Fold Deinterleave/Interleave Pairs (PR #211022)
Jacob Crawley via llvm-commits
llvm-commits at lists.llvm.org
Thu Jul 30 07:28:21 PDT 2026
https://github.com/jacob-crawley updated https://github.com/llvm/llvm-project/pull/211022
>From b7a4e30e03c4285b5ed954035072f8d5bdaa9037 Mon Sep 17 00:00:00 2001
From: Jacob Crawley <jacob.crawley at arm.com>
Date: Tue, 21 Jul 2026 14:53:59 +0000
Subject: [PATCH 1/4] [VectorCombine] Fold interleave of high-half truncations
Fold an interleave of shifted and truncated deinterleave results into a
deinterleave of the original vector at half the element width.
This replaces a deinterlave/interleave sequence and shift/truncates
with a single half-width deinterleave, enabling improved SVE codegen on
AArch64 targets.
---
.../Transforms/Vectorize/VectorCombine.cpp | 105 +++++++++
.../AArch64/sve-vectorcombine-interleave.ll | 65 ++++++
.../VectorCombine/fold-interleave.ll | 209 ++++++++++++++++++
3 files changed, 379 insertions(+)
create mode 100644 llvm/test/CodeGen/AArch64/sve-vectorcombine-interleave.ll
create mode 100644 llvm/test/Transforms/VectorCombine/fold-interleave.ll
diff --git a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
index 741bd4460a5ab..759c826b6e7a7 100644
--- a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
+++ b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
@@ -155,6 +155,7 @@ class VectorCombine {
bool foldEquivalentReductionCmp(Instruction &I);
bool foldReduceAddCmpZero(Instruction &I);
bool foldSelectShuffle(Instruction &I, bool FromReduction = false);
+ bool foldInterleaveOfHighHalfTruncs(Instruction &I);
bool foldInterleaveIntrinsics(Instruction &I);
bool foldDeinterleaveIntrinsics(Instruction &I);
bool foldBitcastOfVPLoad(Instruction &I);
@@ -5882,6 +5883,108 @@ bool VectorCombine::foldInsExtVectorToShuffle(Instruction &I) {
return true;
}
+/// Fold an interleave of shifted, truncated deinterleave results into a
+/// deinterleave of the original vector at half the element width, directly
+/// selecting the high half of each source element.
+///
+/// ```
+/// %d = call { <vscale x 2 x i16>, ... }
+/// @llvm.vector.deinterleave4(<vscale x 8 x i16> %x)
+/// %d0 = extractvalue { <vscale x 2 x i16>, ... } %d, 0
+/// %d1 = extractvalue { <vscale x 2 x i16>, ... } %d, 1
+/// %d2 = extractvalue { <vscale x 2 x i16>, ... } %d, 2
+/// %d3 = extractvalue { <vscale x 2 x i16>, ... } %d, 3
+/// %s0 = lshr <vscale x 2 x i16> %d0, splat (i16 8)
+/// %s1 = lshr <vscale x 2 x i16> %d1, splat (i16 8)
+/// %s2 = lshr <vscale x 2 x i16> %d2, splat (i16 8)
+/// %s3 = lshr <vscale x 2 x i16> %d3, splat (i16 8)
+/// %t0 = trunc <vscale x 2 x i16> %s0 to <vscale x 2 x i8>
+/// %t1 = trunc <vscale x 2 x i16> %s1 to <vscale x 2 x i8>
+/// %t2 = trunc <vscale x 2 x i16> %s2 to <vscale x 2 x i8>
+/// %t3 = trunc <vscale x 2 x i16> %s3 to <vscale x 2 x i8>
+/// %r = call <vscale x 8 x i8> @llvm.vector.interleave4(%t0, %t1, %t2, %t3)
+/// ```
+/// becomes:
+///
+/// ```
+/// %bc = bitcast <vscale x 8 x i16> %x to <vscale x 16 x i8>
+/// %d = call { <vscale x 8 x i8>, <vscale x 8 x i8> }
+/// @llvm.vector.deinterleave2(<vscale x 16 x i8> %bc)
+/// %r = extractvalue { <vscale x 8 x i8>, <vscale x 8 x i8> } %d, 1
+/// ```
+bool VectorCombine::foldInterleaveOfHighHalfTruncs(Instruction &I) {
+ auto *Interleave = dyn_cast<IntrinsicInst>(&I);
+ if (!Interleave || Interleave->hasOperandBundles())
+ return false;
+
+ unsigned Factor = getInterleaveIntrinsicFactor(Interleave->getIntrinsicID());
+ if (!Factor || Interleave->arg_size() != Factor)
+ return false;
+
+ auto *InterleaveTy = dyn_cast<VectorType>(Interleave->getType());
+ if (!InterleaveTy || !InterleaveTy->getElementType()->isIntegerTy())
+ return false;
+
+ unsigned DstEltBits = InterleaveTy->getScalarSizeInBits();
+ Intrinsic::ID DeinterleaveID = Intrinsic::getDeinterleaveIntrinsicID(Factor);
+ IntrinsicInst *Deinterleave = nullptr;
+
+ for (unsigned Idx = 0; Idx != Factor; ++Idx) {
+ auto *Trunc = dyn_cast<TruncInst>(Interleave->getArgOperand(Idx));
+ if (!Trunc || !Trunc->hasOneUse())
+ return false;
+
+ auto *Shift = dyn_cast<BinaryOperator>(Trunc->getOperand(0));
+ if (!Shift || !Shift->hasOneUse() ||
+ (Shift->getOpcode() != Instruction::LShr &&
+ Shift->getOpcode() != Instruction::AShr) ||
+ !match(Shift->getOperand(1), m_SpecificInt(DstEltBits)))
+ return false;
+
+ // Each interleave operand must come from the corresponding deinterleave
+ // result.
+ auto *Extract = dyn_cast<ExtractValueInst>(Shift->getOperand(0));
+ if (!Extract || !Extract->hasOneUse() || Extract->getNumIndices() != 1 ||
+ *Extract->idx_begin() != Idx)
+ return false;
+
+ auto *DI = dyn_cast<IntrinsicInst>(Extract->getAggregateOperand());
+ if (!DI || DI->hasOperandBundles() ||
+ DI->getIntrinsicID() != DeinterleaveID)
+ return false;
+
+ if (!Deinterleave)
+ Deinterleave = DI;
+ else if (DI != Deinterleave)
+ return false;
+ }
+
+ // Require the entire matched tree to become dead, otherwise the replacement
+ // will add another shuffle sequence to the remaining operations.
+ if (!Deinterleave || !Deinterleave->hasNUses(Factor))
+ return false;
+
+ Value *Source = Deinterleave->getArgOperand(0);
+ auto *SourceTy = dyn_cast<VectorType>(Source->getType());
+ if (!SourceTy ||
+ SourceTy->getElementCount() != InterleaveTy->getElementCount() ||
+ !SourceTy->getElementType()->isIntegerTy() ||
+ SourceTy->getScalarSizeInBits() != 2 * DstEltBits)
+ return false;
+
+ auto *BitcastTy = VectorType::getDoubleElementsVectorType(InterleaveTy);
+ Value *Bitcast = Builder.CreateBitCast(Source, BitcastTy);
+ Value *NewDeinterleave = Builder.CreateIntrinsic(
+ Intrinsic::vector_deinterleave2, {BitcastTy}, {Bitcast});
+ // A vector bitcast orders each element's low half first on little-endian
+ // targets and its high half first on big-endian targets.
+ unsigned HighHalfIndex = DL->isLittleEndian() ? 1 : 0;
+ Value *HighHalves =
+ Builder.CreateExtractValue(NewDeinterleave, HighHalfIndex);
+ replaceValue(I, *HighHalves);
+ return true;
+}
+
/// If we're interleaving 2 constant splats, for instance `<vscale x 8 x i32>
/// <splat of 666>` and `<vscale x 8 x i32> <splat of 777>`, we can create a
/// larger splat `<vscale x 8 x i64> <splat of ((777 << 32) | 666)>` first
@@ -6476,6 +6579,8 @@ bool VectorCombine::run() {
return true;
if (scalarizeVPIntrinsic(I))
return true;
+ if (foldInterleaveOfHighHalfTruncs(I))
+ return true;
if (foldInterleaveIntrinsics(I))
return true;
if (foldBitcastOfVPLoad(I))
diff --git a/llvm/test/CodeGen/AArch64/sve-vectorcombine-interleave.ll b/llvm/test/CodeGen/AArch64/sve-vectorcombine-interleave.ll
new file mode 100644
index 0000000000000..731bb482bcb34
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-vectorcombine-interleave.ll
@@ -0,0 +1,65 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -passes=vector-combine -mattr=+sve %s -o - | llc -mattr=+sve -o - | FileCheck %s
+
+target triple = "aarch64"
+
+define void @high_half_trunc_interleave4(ptr %src, ptr %dst) {
+; CHECK-LABEL: high_half_trunc_interleave4:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ldr z0, [x0, #1, mul vl]
+; CHECK-NEXT: ldr z1, [x0]
+; CHECK-NEXT: uzp2 z0.b, z1.b, z0.b
+; CHECK-NEXT: str z0, [x1]
+; CHECK-NEXT: ret
+ %x = load <vscale x 16 x i16>, ptr %src, align 2
+ %d = call { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } @llvm.vector.deinterleave4.nxv16i16(<vscale x 16 x i16> %x)
+ %d0 = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } %d, 0
+ %d1 = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } %d, 1
+ %d2 = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } %d, 2
+ %d3 = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } %d, 3
+ %s0 = lshr <vscale x 4 x i16> %d0, splat (i16 8)
+ %s1 = lshr <vscale x 4 x i16> %d1, splat (i16 8)
+ %s2 = lshr <vscale x 4 x i16> %d2, splat (i16 8)
+ %s3 = lshr <vscale x 4 x i16> %d3, splat (i16 8)
+ %t0 = trunc <vscale x 4 x i16> %s0 to <vscale x 4 x i8>
+ %t1 = trunc <vscale x 4 x i16> %s1 to <vscale x 4 x i8>
+ %t2 = trunc <vscale x 4 x i16> %s2 to <vscale x 4 x i8>
+ %t3 = trunc <vscale x 4 x i16> %s3 to <vscale x 4 x i8>
+ %r = call <vscale x 16 x i8> @llvm.vector.interleave4.nxv16i8(<vscale x 4 x i8> %t0, <vscale x 4 x i8> %t1, <vscale x 4 x i8> %t2, <vscale x 4 x i8> %t3)
+ store <vscale x 16 x i8> %r, ptr %dst, align 1
+ ret void
+}
+
+; Keep the load and bitcast in separate blocks so VectorCombine
+; exposes a byte deinterleave2, which can be selected as a ld2b.
+define void @high_half_trunc_interleave4_byte_load(ptr %src, ptr %dst) {
+; CHECK-LABEL: high_half_trunc_interleave4_byte_load:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ptrue p0.b
+; CHECK-NEXT: ld2b { z0.b, z1.b }, p0/z, [x0]
+; CHECK-NEXT: str z1, [x1]
+; CHECK-NEXT: ret
+entry:
+ %bytes = load <vscale x 32 x i8>, ptr %src, align 2
+ br label %body
+
+body:
+ %x = bitcast <vscale x 32 x i8> %bytes to <vscale x 16 x i16>
+ %d = call { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } @llvm.vector.deinterleave4.nxv16i16(<vscale x 16 x i16> %x)
+ %d0 = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } %d, 0
+ %d1 = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } %d, 1
+ %d2 = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } %d, 2
+ %d3 = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } %d, 3
+ %s0 = lshr <vscale x 4 x i16> %d0, splat (i16 8)
+ %s1 = lshr <vscale x 4 x i16> %d1, splat (i16 8)
+ %s2 = lshr <vscale x 4 x i16> %d2, splat (i16 8)
+ %s3 = lshr <vscale x 4 x i16> %d3, splat (i16 8)
+ %t0 = trunc <vscale x 4 x i16> %s0 to <vscale x 4 x i8>
+ %t1 = trunc <vscale x 4 x i16> %s1 to <vscale x 4 x i8>
+ %t2 = trunc <vscale x 4 x i16> %s2 to <vscale x 4 x i8>
+ %t3 = trunc <vscale x 4 x i16> %s3 to <vscale x 4 x i8>
+ %r = call <vscale x 16 x i8> @llvm.vector.interleave4.nxv16i8(<vscale x 4 x i8> %t0, <vscale x 4 x i8> %t1, <vscale x 4 x i8> %t2, <vscale x 4 x i8> %t3)
+ store <vscale x 16 x i8> %r, ptr %dst, align 1
+ ret void
+}
+
diff --git a/llvm/test/Transforms/VectorCombine/fold-interleave.ll b/llvm/test/Transforms/VectorCombine/fold-interleave.ll
new file mode 100644
index 0000000000000..efe51333c024a
--- /dev/null
+++ b/llvm/test/Transforms/VectorCombine/fold-interleave.ll
@@ -0,0 +1,209 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -passes=vector-combine -S %s | FileCheck %s
+; RUN: opt -passes=vector-combine -data-layout=E -S %s | FileCheck %s --check-prefixes=BE
+
+define <vscale x 8 x i8> @lshr_trunc_interleave4(<vscale x 8 x i16> %x) {
+; CHECK-LABEL: define <vscale x 8 x i8> @lshr_trunc_interleave4(
+; CHECK-SAME: <vscale x 8 x i16> [[X:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast <vscale x 8 x i16> [[X]] to <vscale x 16 x i8>
+; CHECK-NEXT: [[TMP2:%.*]] = call { <vscale x 8 x i8>, <vscale x 8 x i8> } @llvm.vector.deinterleave2.nxv16i8(<vscale x 16 x i8> [[TMP1]])
+; CHECK-NEXT: [[R:%.*]] = extractvalue { <vscale x 8 x i8>, <vscale x 8 x i8> } [[TMP2]], 1
+; CHECK-NEXT: ret <vscale x 8 x i8> [[R]]
+;
+; BE-LABEL: define <vscale x 8 x i8> @lshr_trunc_interleave4(
+; BE-SAME: <vscale x 8 x i16> [[X:%.*]]) {
+; BE-NEXT: [[TMP1:%.*]] = bitcast <vscale x 8 x i16> [[X]] to <vscale x 16 x i8>
+; BE-NEXT: [[TMP2:%.*]] = call { <vscale x 8 x i8>, <vscale x 8 x i8> } @llvm.vector.deinterleave2.nxv16i8(<vscale x 16 x i8> [[TMP1]])
+; BE-NEXT: [[R:%.*]] = extractvalue { <vscale x 8 x i8>, <vscale x 8 x i8> } [[TMP2]], 0
+; BE-NEXT: ret <vscale x 8 x i8> [[R]]
+;
+ %d = call { <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16> } @llvm.vector.deinterleave4.nxv8i16(<vscale x 8 x i16> %x)
+ %d0 = extractvalue { <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16> } %d, 0
+ %d1 = extractvalue { <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16> } %d, 1
+ %d2 = extractvalue { <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16> } %d, 2
+ %d3 = extractvalue { <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16> } %d, 3
+ %s0 = lshr <vscale x 2 x i16> %d0, splat (i16 8)
+ %s1 = lshr <vscale x 2 x i16> %d1, splat (i16 8)
+ %s2 = lshr <vscale x 2 x i16> %d2, splat (i16 8)
+ %s3 = lshr <vscale x 2 x i16> %d3, splat (i16 8)
+ %t0 = trunc <vscale x 2 x i16> %s0 to <vscale x 2 x i8>
+ %t1 = trunc <vscale x 2 x i16> %s1 to <vscale x 2 x i8>
+ %t2 = trunc <vscale x 2 x i16> %s2 to <vscale x 2 x i8>
+ %t3 = trunc <vscale x 2 x i16> %s3 to <vscale x 2 x i8>
+ %r = call <vscale x 8 x i8> @llvm.vector.interleave4.nxv8i8(<vscale x 2 x i8> %t0, <vscale x 2 x i8> %t1, <vscale x 2 x i8> %t2, <vscale x 2 x i8> %t3)
+ ret <vscale x 8 x i8> %r
+}
+
+define <8 x i8> @fixed_lshr_trunc_interleave4(<8 x i16> %x) {
+; CHECK-LABEL: define <8 x i8> @fixed_lshr_trunc_interleave4(
+; CHECK-SAME: <8 x i16> [[X:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast <8 x i16> [[X]] to <16 x i8>
+; CHECK-NEXT: [[TMP2:%.*]] = call { <8 x i8>, <8 x i8> } @llvm.vector.deinterleave2.v16i8(<16 x i8> [[TMP1]])
+; CHECK-NEXT: [[R:%.*]] = extractvalue { <8 x i8>, <8 x i8> } [[TMP2]], 1
+; CHECK-NEXT: ret <8 x i8> [[R]]
+;
+; BE-LABEL: define <8 x i8> @fixed_lshr_trunc_interleave4(
+; BE-SAME: <8 x i16> [[X:%.*]]) {
+; BE-NEXT: [[TMP1:%.*]] = bitcast <8 x i16> [[X]] to <16 x i8>
+; BE-NEXT: [[TMP2:%.*]] = call { <8 x i8>, <8 x i8> } @llvm.vector.deinterleave2.v16i8(<16 x i8> [[TMP1]])
+; BE-NEXT: [[R:%.*]] = extractvalue { <8 x i8>, <8 x i8> } [[TMP2]], 0
+; BE-NEXT: ret <8 x i8> [[R]]
+;
+ %d = call { <2 x i16>, <2 x i16>, <2 x i16>, <2 x i16> } @llvm.vector.deinterleave4.v8i16(<8 x i16> %x)
+ %d0 = extractvalue { <2 x i16>, <2 x i16>, <2 x i16>, <2 x i16> } %d, 0
+ %d1 = extractvalue { <2 x i16>, <2 x i16>, <2 x i16>, <2 x i16> } %d, 1
+ %d2 = extractvalue { <2 x i16>, <2 x i16>, <2 x i16>, <2 x i16> } %d, 2
+ %d3 = extractvalue { <2 x i16>, <2 x i16>, <2 x i16>, <2 x i16> } %d, 3
+ %s0 = lshr <2 x i16> %d0, splat (i16 8)
+ %s1 = lshr <2 x i16> %d1, splat (i16 8)
+ %s2 = lshr <2 x i16> %d2, splat (i16 8)
+ %s3 = lshr <2 x i16> %d3, splat (i16 8)
+ %t0 = trunc <2 x i16> %s0 to <2 x i8>
+ %t1 = trunc <2 x i16> %s1 to <2 x i8>
+ %t2 = trunc <2 x i16> %s2 to <2 x i8>
+ %t3 = trunc <2 x i16> %s3 to <2 x i8>
+ %r = call <8 x i8> @llvm.vector.interleave4.v8i8(<2 x i8> %t0, <2 x i8> %t1, <2 x i8> %t2, <2 x i8> %t3)
+ ret <8 x i8> %r
+}
+
+define <vscale x 4 x i16> @mixed_shifts_trunc_interleave2(<vscale x 4 x i32> %x) {
+; CHECK-LABEL: define <vscale x 4 x i16> @mixed_shifts_trunc_interleave2(
+; CHECK-SAME: <vscale x 4 x i32> [[X:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast <vscale x 4 x i32> [[X]] to <vscale x 8 x i16>
+; CHECK-NEXT: [[TMP2:%.*]] = call { <vscale x 4 x i16>, <vscale x 4 x i16> } @llvm.vector.deinterleave2.nxv8i16(<vscale x 8 x i16> [[TMP1]])
+; CHECK-NEXT: [[R:%.*]] = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16> } [[TMP2]], 1
+; CHECK-NEXT: ret <vscale x 4 x i16> [[R]]
+;
+; BE-LABEL: define <vscale x 4 x i16> @mixed_shifts_trunc_interleave2(
+; BE-SAME: <vscale x 4 x i32> [[X:%.*]]) {
+; BE-NEXT: [[TMP1:%.*]] = bitcast <vscale x 4 x i32> [[X]] to <vscale x 8 x i16>
+; BE-NEXT: [[TMP2:%.*]] = call { <vscale x 4 x i16>, <vscale x 4 x i16> } @llvm.vector.deinterleave2.nxv8i16(<vscale x 8 x i16> [[TMP1]])
+; BE-NEXT: [[R:%.*]] = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16> } [[TMP2]], 0
+; BE-NEXT: ret <vscale x 4 x i16> [[R]]
+;
+ %d = call { <vscale x 2 x i32>, <vscale x 2 x i32> } @llvm.vector.deinterleave2.nxv4i32(<vscale x 4 x i32> %x)
+ %d0 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } %d, 0
+ %d1 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } %d, 1
+ %s0 = lshr exact <vscale x 2 x i32> %d0, splat (i32 16)
+ %s1 = ashr exact <vscale x 2 x i32> %d1, splat (i32 16)
+ %t0 = trunc nuw <vscale x 2 x i32> %s0 to <vscale x 2 x i16>
+ %t1 = trunc nuw <vscale x 2 x i32> %s1 to <vscale x 2 x i16>
+ %r = call <vscale x 4 x i16> @llvm.vector.interleave2.nxv4i16(<vscale x 2 x i16> %t0, <vscale x 2 x i16> %t1)
+ ret <vscale x 4 x i16> %r
+}
+
+; Negative test - the fold should only be applied if the shifts are equal.
+define <vscale x 4 x i16> @different_shift_amounts(<vscale x 4 x i32> %x) {
+; CHECK-LABEL: define <vscale x 4 x i16> @different_shift_amounts(
+; CHECK-SAME: <vscale x 4 x i32> [[X:%.*]]) {
+; CHECK-NEXT: [[D:%.*]] = call { <vscale x 2 x i32>, <vscale x 2 x i32> } @llvm.vector.deinterleave2.nxv4i32(<vscale x 4 x i32> [[X]])
+; CHECK-NEXT: [[D0:%.*]] = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } [[D]], 0
+; CHECK-NEXT: [[D1:%.*]] = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } [[D]], 1
+; CHECK-NEXT: [[S0:%.*]] = lshr <vscale x 2 x i32> [[D0]], splat (i32 16)
+; CHECK-NEXT: [[S1:%.*]] = lshr <vscale x 2 x i32> [[D1]], splat (i32 15)
+; CHECK-NEXT: [[T0:%.*]] = trunc <vscale x 2 x i32> [[S0]] to <vscale x 2 x i16>
+; CHECK-NEXT: [[T1:%.*]] = trunc <vscale x 2 x i32> [[S1]] to <vscale x 2 x i16>
+; CHECK-NEXT: [[R:%.*]] = call <vscale x 4 x i16> @llvm.vector.interleave2.nxv4i16(<vscale x 2 x i16> [[T0]], <vscale x 2 x i16> [[T1]])
+; CHECK-NEXT: ret <vscale x 4 x i16> [[R]]
+;
+; BE-LABEL: define <vscale x 4 x i16> @different_shift_amounts(
+; BE-SAME: <vscale x 4 x i32> [[X:%.*]]) {
+; BE-NEXT: [[D:%.*]] = call { <vscale x 2 x i32>, <vscale x 2 x i32> } @llvm.vector.deinterleave2.nxv4i32(<vscale x 4 x i32> [[X]])
+; BE-NEXT: [[D0:%.*]] = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } [[D]], 0
+; BE-NEXT: [[D1:%.*]] = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } [[D]], 1
+; BE-NEXT: [[S0:%.*]] = lshr <vscale x 2 x i32> [[D0]], splat (i32 16)
+; BE-NEXT: [[S1:%.*]] = lshr <vscale x 2 x i32> [[D1]], splat (i32 15)
+; BE-NEXT: [[T0:%.*]] = trunc <vscale x 2 x i32> [[S0]] to <vscale x 2 x i16>
+; BE-NEXT: [[T1:%.*]] = trunc <vscale x 2 x i32> [[S1]] to <vscale x 2 x i16>
+; BE-NEXT: [[R:%.*]] = call <vscale x 4 x i16> @llvm.vector.interleave2.nxv4i16(<vscale x 2 x i16> [[T0]], <vscale x 2 x i16> [[T1]])
+; BE-NEXT: ret <vscale x 4 x i16> [[R]]
+;
+ %d = call { <vscale x 2 x i32>, <vscale x 2 x i32> } @llvm.vector.deinterleave2.nxv4i32(<vscale x 4 x i32> %x)
+ %d0 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } %d, 0
+ %d1 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } %d, 1
+ %s0 = lshr <vscale x 2 x i32> %d0, splat (i32 16)
+ %s1 = lshr <vscale x 2 x i32> %d1, splat (i32 15)
+ %t0 = trunc <vscale x 2 x i32> %s0 to <vscale x 2 x i16>
+ %t1 = trunc <vscale x 2 x i32> %s1 to <vscale x 2 x i16>
+ %r = call <vscale x 4 x i16> @llvm.vector.interleave2.nxv4i16(<vscale x 2 x i16> %t0, <vscale x 2 x i16> %t1)
+ ret <vscale x 4 x i16> %r
+}
+
+; Negative test - the fold shouldn't be generated as the deitnerleaved fields are passed in the wrong order.
+define <vscale x 4 x i16> @reordered_fields(<vscale x 4 x i32> %x) {
+; CHECK-LABEL: define <vscale x 4 x i16> @reordered_fields(
+; CHECK-SAME: <vscale x 4 x i32> [[X:%.*]]) {
+; CHECK-NEXT: [[D:%.*]] = call { <vscale x 2 x i32>, <vscale x 2 x i32> } @llvm.vector.deinterleave2.nxv4i32(<vscale x 4 x i32> [[X]])
+; CHECK-NEXT: [[D0:%.*]] = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } [[D]], 0
+; CHECK-NEXT: [[D1:%.*]] = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } [[D]], 1
+; CHECK-NEXT: [[S0:%.*]] = lshr <vscale x 2 x i32> [[D0]], splat (i32 16)
+; CHECK-NEXT: [[S1:%.*]] = lshr <vscale x 2 x i32> [[D1]], splat (i32 16)
+; CHECK-NEXT: [[T0:%.*]] = trunc <vscale x 2 x i32> [[S0]] to <vscale x 2 x i16>
+; CHECK-NEXT: [[T1:%.*]] = trunc <vscale x 2 x i32> [[S1]] to <vscale x 2 x i16>
+; CHECK-NEXT: [[R:%.*]] = call <vscale x 4 x i16> @llvm.vector.interleave2.nxv4i16(<vscale x 2 x i16> [[T1]], <vscale x 2 x i16> [[T0]])
+; CHECK-NEXT: ret <vscale x 4 x i16> [[R]]
+;
+; BE-LABEL: define <vscale x 4 x i16> @reordered_fields(
+; BE-SAME: <vscale x 4 x i32> [[X:%.*]]) {
+; BE-NEXT: [[D:%.*]] = call { <vscale x 2 x i32>, <vscale x 2 x i32> } @llvm.vector.deinterleave2.nxv4i32(<vscale x 4 x i32> [[X]])
+; BE-NEXT: [[D0:%.*]] = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } [[D]], 0
+; BE-NEXT: [[D1:%.*]] = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } [[D]], 1
+; BE-NEXT: [[S0:%.*]] = lshr <vscale x 2 x i32> [[D0]], splat (i32 16)
+; BE-NEXT: [[S1:%.*]] = lshr <vscale x 2 x i32> [[D1]], splat (i32 16)
+; BE-NEXT: [[T0:%.*]] = trunc <vscale x 2 x i32> [[S0]] to <vscale x 2 x i16>
+; BE-NEXT: [[T1:%.*]] = trunc <vscale x 2 x i32> [[S1]] to <vscale x 2 x i16>
+; BE-NEXT: [[R:%.*]] = call <vscale x 4 x i16> @llvm.vector.interleave2.nxv4i16(<vscale x 2 x i16> [[T1]], <vscale x 2 x i16> [[T0]])
+; BE-NEXT: ret <vscale x 4 x i16> [[R]]
+;
+ %d = call { <vscale x 2 x i32>, <vscale x 2 x i32> } @llvm.vector.deinterleave2.nxv4i32(<vscale x 4 x i32> %x)
+ %d0 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } %d, 0
+ %d1 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } %d, 1
+ %s0 = lshr <vscale x 2 x i32> %d0, splat (i32 16)
+ %s1 = lshr <vscale x 2 x i32> %d1, splat (i32 16)
+ %t0 = trunc <vscale x 2 x i32> %s0 to <vscale x 2 x i16>
+ %t1 = trunc <vscale x 2 x i32> %s1 to <vscale x 2 x i16>
+ %r = call <vscale x 4 x i16> @llvm.vector.interleave2.nxv4i16(<vscale x 2 x i16> %t1, <vscale x 2 x i16> %t0)
+ ret <vscale x 4 x i16> %r
+}
+
+; Negative test - the fold shouldn't be generated as the trunc has more than one use.
+define <vscale x 4 x i16> @extra_trunc_use(
+; CHECK-LABEL: define <vscale x 4 x i16> @extra_trunc_use(
+; CHECK-SAME: <vscale x 4 x i32> [[X:%.*]], ptr [[DST:%.*]]) {
+; CHECK-NEXT: [[D:%.*]] = call { <vscale x 2 x i32>, <vscale x 2 x i32> } @llvm.vector.deinterleave2.nxv4i32(<vscale x 4 x i32> [[X]])
+; CHECK-NEXT: [[D0:%.*]] = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } [[D]], 0
+; CHECK-NEXT: [[D1:%.*]] = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } [[D]], 1
+; CHECK-NEXT: [[S0:%.*]] = lshr <vscale x 2 x i32> [[D0]], splat (i32 16)
+; CHECK-NEXT: [[S1:%.*]] = lshr <vscale x 2 x i32> [[D1]], splat (i32 16)
+; CHECK-NEXT: [[T0:%.*]] = trunc <vscale x 2 x i32> [[S0]] to <vscale x 2 x i16>
+; CHECK-NEXT: [[T1:%.*]] = trunc <vscale x 2 x i32> [[S1]] to <vscale x 2 x i16>
+; CHECK-NEXT: store <vscale x 2 x i16> [[T0]], ptr [[DST]], align 4
+; CHECK-NEXT: [[R:%.*]] = call <vscale x 4 x i16> @llvm.vector.interleave2.nxv4i16(<vscale x 2 x i16> [[T0]], <vscale x 2 x i16> [[T1]])
+; CHECK-NEXT: ret <vscale x 4 x i16> [[R]]
+;
+; BE-LABEL: define <vscale x 4 x i16> @extra_trunc_use(
+; BE-SAME: <vscale x 4 x i32> [[X:%.*]], ptr [[DST:%.*]]) {
+; BE-NEXT: [[D:%.*]] = call { <vscale x 2 x i32>, <vscale x 2 x i32> } @llvm.vector.deinterleave2.nxv4i32(<vscale x 4 x i32> [[X]])
+; BE-NEXT: [[D0:%.*]] = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } [[D]], 0
+; BE-NEXT: [[D1:%.*]] = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } [[D]], 1
+; BE-NEXT: [[S0:%.*]] = lshr <vscale x 2 x i32> [[D0]], splat (i32 16)
+; BE-NEXT: [[S1:%.*]] = lshr <vscale x 2 x i32> [[D1]], splat (i32 16)
+; BE-NEXT: [[T0:%.*]] = trunc <vscale x 2 x i32> [[S0]] to <vscale x 2 x i16>
+; BE-NEXT: [[T1:%.*]] = trunc <vscale x 2 x i32> [[S1]] to <vscale x 2 x i16>
+; BE-NEXT: store <vscale x 2 x i16> [[T0]], ptr [[DST]], align 4
+; BE-NEXT: [[R:%.*]] = call <vscale x 4 x i16> @llvm.vector.interleave2.nxv4i16(<vscale x 2 x i16> [[T0]], <vscale x 2 x i16> [[T1]])
+; BE-NEXT: ret <vscale x 4 x i16> [[R]]
+;
+ <vscale x 4 x i32> %x, ptr %dst) {
+ %d = call { <vscale x 2 x i32>, <vscale x 2 x i32> } @llvm.vector.deinterleave2.nxv4i32(<vscale x 4 x i32> %x)
+ %d0 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } %d, 0
+ %d1 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } %d, 1
+ %s0 = lshr <vscale x 2 x i32> %d0, splat (i32 16)
+ %s1 = lshr <vscale x 2 x i32> %d1, splat (i32 16)
+ %t0 = trunc <vscale x 2 x i32> %s0 to <vscale x 2 x i16>
+ %t1 = trunc <vscale x 2 x i32> %s1 to <vscale x 2 x i16>
+ store <vscale x 2 x i16> %t0, ptr %dst
+ %r = call <vscale x 4 x i16> @llvm.vector.interleave2.nxv4i16(<vscale x 2 x i16> %t0, <vscale x 2 x i16> %t1)
+ ret <vscale x 4 x i16> %r
+}
+
>From e0afad0945f658b52be79b25b19c35f413078f1a Mon Sep 17 00:00:00 2001
From: Jacob Crawley <jacob.crawley at arm.com>
Date: Tue, 28 Jul 2026 15:06:13 +0000
Subject: [PATCH 2/4] Generalise the Vector Combine to fold away
deinterleave/interleave pairs.
---
.../Transforms/Vectorize/VectorCombine.cpp | 291 ++++++---
.../AArch64/deinterleave-interleave-pairs.ll | 590 ++++++++++++++++++
.../AArch64/sve-vectorcombine-interleave.ll | 17 +-
.../VectorCombine/fold-interleave.ll | 209 -------
4 files changed, 808 insertions(+), 299 deletions(-)
create mode 100644 llvm/test/Transforms/VectorCombine/AArch64/deinterleave-interleave-pairs.ll
rename llvm/test/{CodeGen => Transforms/VectorCombine}/AArch64/sve-vectorcombine-interleave.ll (89%)
delete mode 100644 llvm/test/Transforms/VectorCombine/fold-interleave.ll
diff --git a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
index 759c826b6e7a7..d949ae1d6600b 100644
--- a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
+++ b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
@@ -155,7 +155,6 @@ class VectorCombine {
bool foldEquivalentReductionCmp(Instruction &I);
bool foldReduceAddCmpZero(Instruction &I);
bool foldSelectShuffle(Instruction &I, bool FromReduction = false);
- bool foldInterleaveOfHighHalfTruncs(Instruction &I);
bool foldInterleaveIntrinsics(Instruction &I);
bool foldDeinterleaveIntrinsics(Instruction &I);
bool foldBitcastOfVPLoad(Instruction &I);
@@ -163,6 +162,7 @@ class VectorCombine {
bool shrinkType(Instruction &I);
bool shrinkLoadForShuffles(Instruction &I);
bool shrinkPhiOfShuffles(Instruction &I);
+ bool foldDeinterleaveInterleavePair(Instruction &I);
void replaceValue(Instruction &Old, Value &New, bool Erase = true) {
LLVM_DEBUG(dbgs() << "VC: Replacing: " << Old << '\n');
@@ -5883,105 +5883,227 @@ bool VectorCombine::foldInsExtVectorToShuffle(Instruction &I) {
return true;
}
-/// Fold an interleave of shifted, truncated deinterleave results into a
-/// deinterleave of the original vector at half the element width, directly
-/// selecting the high half of each source element.
+/// Fold away a matched pair of vector.deinterleave/interleave intrinsics
+/// with a chain of elementwise operations on each between the
+/// deinterleave and interleave.
///
-/// ```
-/// %d = call { <vscale x 2 x i16>, ... }
-/// @llvm.vector.deinterleave4(<vscale x 8 x i16> %x)
-/// %d0 = extractvalue { <vscale x 2 x i16>, ... } %d, 0
-/// %d1 = extractvalue { <vscale x 2 x i16>, ... } %d, 1
-/// %d2 = extractvalue { <vscale x 2 x i16>, ... } %d, 2
-/// %d3 = extractvalue { <vscale x 2 x i16>, ... } %d, 3
-/// %s0 = lshr <vscale x 2 x i16> %d0, splat (i16 8)
-/// %s1 = lshr <vscale x 2 x i16> %d1, splat (i16 8)
-/// %s2 = lshr <vscale x 2 x i16> %d2, splat (i16 8)
-/// %s3 = lshr <vscale x 2 x i16> %d3, splat (i16 8)
-/// %t0 = trunc <vscale x 2 x i16> %s0 to <vscale x 2 x i8>
-/// %t1 = trunc <vscale x 2 x i16> %s1 to <vscale x 2 x i8>
-/// %t2 = trunc <vscale x 2 x i16> %s2 to <vscale x 2 x i8>
-/// %t3 = trunc <vscale x 2 x i16> %s3 to <vscale x 2 x i8>
-/// %r = call <vscale x 8 x i8> @llvm.vector.interleave4(%t0, %t1, %t2, %t3)
-/// ```
-/// becomes:
+/// For example:
+/// ```
+/// %d = call { <2 x i16>, <2 x i16> } @deinterleave2.v4i16(<4 x i16> %v)
+/// %f0 = extractvalue { <2 x i16>, <2 x i16> } %d, 0
+/// %f1 = extractvalue { <2 x i16>, <2 x i16> } %d, 1
///
-/// ```
-/// %bc = bitcast <vscale x 8 x i16> %x to <vscale x 16 x i8>
-/// %d = call { <vscale x 8 x i8>, <vscale x 8 x i8> }
-/// @llvm.vector.deinterleave2(<vscale x 16 x i8> %bc)
-/// %r = extractvalue { <vscale x 8 x i8>, <vscale x 8 x i8> } %d, 1
-/// ```
-bool VectorCombine::foldInterleaveOfHighHalfTruncs(Instruction &I) {
- auto *Interleave = dyn_cast<IntrinsicInst>(&I);
- if (!Interleave || Interleave->hasOperandBundles())
- return false;
+/// %u0 = add <2 x i16> %f0, splat (i16 3)
+/// %u1 = add <2 x i16> %f1, splat (i16 3)
+///
+/// %r = call <4 x i16> @interleave2.v4i16(<2 x i16> %u0, <2 x i16> %u1)
+/// ```
+/// Folds to:
+/// ```
+/// %r = add <4 x i16> %v, splat (i16 3)
+/// ```
+bool VectorCombine::foldDeinterleaveInterleavePair(Instruction &I) {
+ auto *Deinterleave = dyn_cast<IntrinsicInst>(&I);
+ if (!Deinterleave)
+ return false;
+
+ unsigned Factor =
+ getDeinterleaveIntrinsicFactor(Deinterleave->getIntrinsicID());
+ if (!Factor || Deinterleave->hasOperandBundles() ||
+ !Deinterleave->hasNUndroppableUses(Factor))
+ return false;
+
+ const Intrinsic::ID InterleaveIID =
+ Intrinsic::getInterleaveIntrinsicID(Factor);
+
+ // Collect one extract for each deinterleaved field.
+ SmallVector<Instruction *, 8> CurrentInsts(Factor, nullptr);
+ for (Use &U : Deinterleave->uses()) {
+ if (U.getUser()->isDroppable())
+ continue;
- unsigned Factor = getInterleaveIntrinsicFactor(Interleave->getIntrinsicID());
- if (!Factor || Interleave->arg_size() != Factor)
- return false;
+ auto *Extract = dyn_cast<ExtractValueInst>(U.getUser());
+ if (!Extract || Extract->getNumIndices() != 1)
+ return false;
- auto *InterleaveTy = dyn_cast<VectorType>(Interleave->getType());
- if (!InterleaveTy || !InterleaveTy->getElementType()->isIntegerTy())
- return false;
+ unsigned Index = *Extract->idx_begin();
+ if (Index >= Factor || CurrentInsts[Index])
+ return false;
- unsigned DstEltBits = InterleaveTy->getScalarSizeInBits();
- Intrinsic::ID DeinterleaveID = Intrinsic::getDeinterleaveIntrinsicID(Factor);
- IntrinsicInst *Deinterleave = nullptr;
+ CurrentInsts[Index] = Extract;
+ }
- for (unsigned Idx = 0; Idx != Factor; ++Idx) {
- auto *Trunc = dyn_cast<TruncInst>(Interleave->getArgOperand(Idx));
- if (!Trunc || !Trunc->hasOneUse())
- return false;
+ // Stores a chain steps operations with the preceding operand.
+ struct ElementwiseStep {
+ SmallVector<Instruction *, 8> Insts;
+ unsigned ChainOperand;
+ };
- auto *Shift = dyn_cast<BinaryOperator>(Trunc->getOperand(0));
- if (!Shift || !Shift->hasOneUse() ||
- (Shift->getOpcode() != Instruction::LShr &&
- Shift->getOpcode() != Instruction::AShr) ||
- !match(Shift->getOperand(1), m_SpecificInt(DstEltBits)))
- return false;
+ SmallVector<ElementwiseStep, 4> Steps;
+ IntrinsicInst *Interleave = nullptr;
+ unsigned NumVisited = 0;
- // Each interleave operand must come from the corresponding deinterleave
- // result.
- auto *Extract = dyn_cast<ExtractValueInst>(Shift->getOperand(0));
- if (!Extract || !Extract->hasOneUse() || Extract->getNumIndices() != 1 ||
- *Extract->idx_begin() != Idx)
+ auto getNumDataOperands = [](Instruction *Inst) -> unsigned {
+ if (auto *II = dyn_cast<IntrinsicInst>(Inst))
+ return II->arg_size();
+ return Inst->getNumOperands();
+ };
+
+ auto isSupportedElementwise = [&](Instruction *Inst) {
+ auto *ResultTy = dyn_cast<VectorType>(Inst->getType());
+ if (!ResultTy)
return false;
- auto *DI = dyn_cast<IntrinsicInst>(Extract->getAggregateOperand());
- if (!DI || DI->hasOperandBundles() ||
- DI->getIntrinsicID() != DeinterleaveID)
+ if (auto *II = dyn_cast<IntrinsicInst>(Inst)) {
+ if (II->hasOperandBundles() || II->isConvergent() ||
+ !isTriviallyVectorizable(II->getIntrinsicID()))
+ return false;
+ } else if (!isa<BinaryOperator, UnaryOperator, CastInst, CmpInst,
+ SelectInst, FreezeInst>(Inst)) {
return false;
+ }
+
+ // Reject operations such as element-count-changing bitcasts.
+ for (unsigned Op = 0, E = getNumDataOperands(Inst); Op != E; ++Op) {
+ auto *OperandTy = dyn_cast<VectorType>(Inst->getOperand(Op)->getType());
+ if (OperandTy &&
+ OperandTy->getElementCount() != ResultTy->getElementCount())
+ return false;
+ }
+
+ return true;
+ };
+
+ // Follow the chains until they reach the matching interleave.
+ while (NumVisited + Factor <= MaxInstrsToScan) {
+ NumVisited += Factor;
+
+ SmallVector<Instruction *, 8> NextInsts;
+ SmallVector<unsigned, 8> OperandNumbers;
+ NextInsts.reserve(Factor);
+ OperandNumbers.reserve(Factor);
+
+ for (Instruction *Current : CurrentInsts) {
+ Use *U = Current->getSingleUndroppableUse();
+ auto *Next = U ? dyn_cast<Instruction>(U->getUser()) : nullptr;
+ if (!Next)
+ return false;
+
+ NextInsts.push_back(Next);
+ OperandNumbers.push_back(U->getOperandNo());
+ }
+
+ // Check whether every chain has reached the same interleave.
+ if (auto *II = dyn_cast<IntrinsicInst>(NextInsts.front());
+ II && II->getIntrinsicID() == InterleaveIID) {
+ if (II->hasOperandBundles() || II->arg_size() != Factor)
+ return false;
- if (!Deinterleave)
- Deinterleave = DI;
- else if (DI != Deinterleave)
+ for (unsigned Index = 0; Index != Factor; ++Index)
+ if (NextInsts[Index] != II || OperandNumbers[Index] != Index)
+ return false;
+
+ Interleave = II;
+ break;
+ }
+
+ Instruction *FirstInst = NextInsts.front();
+ unsigned ChainOperand = OperandNumbers.front();
+
+ if (!isSupportedElementwise(FirstInst) ||
+ ChainOperand >= getNumDataOperands(FirstInst))
return false;
+
+ for (unsigned Index = 1; Index != Factor; ++Index) {
+ Instruction *Inst = NextInsts[Index];
+ if (OperandNumbers[Index] != ChainOperand ||
+ !FirstInst->isSameOperationAs(Inst))
+ return false;
+ }
+
+ // Non-chain operands must be either the same scalar or splats of that
+ // scalar.
+ auto getSplatOrScalar = [](Value *V) -> Value * {
+ return isa<VectorType>(V->getType()) ? getSplatValue(V) : V;
+ };
+
+ for (unsigned Op = 0, E = getNumDataOperands(FirstInst); Op != E; ++Op) {
+ if (Op == ChainOperand)
+ continue;
+
+ Value *CommonValue = getSplatOrScalar(FirstInst->getOperand(Op));
+ if (!CommonValue || any_of(drop_begin(NextInsts), [&](Instruction *Inst) {
+ return getSplatOrScalar(Inst->getOperand(Op)) != CommonValue;
+ }))
+ return false;
+ }
+
+ CurrentInsts.assign(NextInsts.begin(), NextInsts.end());
+
+ Steps.push_back(ElementwiseStep{std::move(NextInsts), ChainOperand});
}
- // Require the entire matched tree to become dead, otherwise the replacement
- // will add another shuffle sequence to the remaining operations.
- if (!Deinterleave || !Deinterleave->hasNUses(Factor))
+ if (!Interleave)
return false;
- Value *Source = Deinterleave->getArgOperand(0);
- auto *SourceTy = dyn_cast<VectorType>(Source->getType());
- if (!SourceTy ||
- SourceTy->getElementCount() != InterleaveTy->getElementCount() ||
- !SourceTy->getElementType()->isIntegerTy() ||
- SourceTy->getScalarSizeInBits() != 2 * DstEltBits)
- return false;
+ // Rebuild the matched elementwise chain at the original vector width.
+ Builder.SetInsertPoint(Interleave);
- auto *BitcastTy = VectorType::getDoubleElementsVectorType(InterleaveTy);
- Value *Bitcast = Builder.CreateBitCast(Source, BitcastTy);
- Value *NewDeinterleave = Builder.CreateIntrinsic(
- Intrinsic::vector_deinterleave2, {BitcastTy}, {Bitcast});
- // A vector bitcast orders each element's low half first on little-endian
- // targets and its high half first on big-endian targets.
- unsigned HighHalfIndex = DL->isLittleEndian() ? 1 : 0;
- Value *HighHalves =
- Builder.CreateExtractValue(NewDeinterleave, HighHalfIndex);
- replaceValue(I, *HighHalves);
+ Value *WideValue = Deinterleave->getArgOperand(0);
+
+ ElementCount WideEC =
+ cast<VectorType>(Deinterleave->getArgOperand(0)->getType())
+ ->getElementCount();
+ bool DropTruncFlags = false;
+ for (const ElementwiseStep &Step : Steps) {
+ Instruction *NarrowInst = Step.Insts.front();
+
+ unsigned NumOperands = getNumDataOperands(NarrowInst);
+ SmallVector<Value *, 4> NewOperands;
+ NewOperands.reserve(NumOperands);
+
+ for (unsigned Op = 0; Op != NumOperands; ++Op) {
+ Value *Operand = NarrowInst->getOperand(Op);
+
+ if (Op == Step.ChainOperand)
+ Operand = WideValue;
+ else if (isa<VectorType>(Operand->getType()))
+ Operand = Builder.CreateVectorSplat(WideEC, getSplatValue(Operand));
+ NewOperands.push_back(Operand);
+ }
+
+ auto *WideResultTy =
+ VectorType::get(NarrowInst->getType()->getScalarType(), WideEC);
+
+ Value *NewValue;
+ if (isa<BinaryOperator, UnaryOperator>(NarrowInst)) {
+ NewValue = Builder.CreateNAryOp(NarrowInst->getOpcode(), NewOperands);
+ } else if (auto *Cast = dyn_cast<CastInst>(NarrowInst)) {
+ NewValue =
+ Builder.CreateCast(Cast->getOpcode(), NewOperands[0], WideResultTy);
+ } else if (auto *Cmp = dyn_cast<CmpInst>(NarrowInst)) {
+ NewValue = Builder.CreateCmp(Cmp->getPredicate(), NewOperands[0],
+ NewOperands[1]);
+ } else if (isa<SelectInst>(NarrowInst)) {
+ NewValue =
+ Builder.CreateSelect(NewOperands[0], NewOperands[1], NewOperands[2]);
+ } else if (isa<FreezeInst>(NarrowInst)) {
+ NewValue = Builder.CreateFreeze(NewOperands[0]);
+ } else if (auto *II = dyn_cast<IntrinsicInst>(NarrowInst)) {
+ NewValue = Builder.CreateIntrinsic(WideResultTy, II->getIntrinsicID(),
+ NewOperands);
+ } else {
+ llvm_unreachable("Unsupported instruction");
+ }
+
+ SmallVector<Value *, 8> NarrowInsts(Step.Insts.begin(), Step.Insts.end());
+ propagateIRFlags(NewValue, NarrowInsts);
+
+ WideValue = NewValue;
+ }
+
+ assert(WideValue->getType() == Interleave->getType());
+ replaceValue(*Interleave, *WideValue);
return true;
}
@@ -6055,6 +6177,9 @@ bool VectorCombine::foldInterleaveIntrinsics(Instruction &I) {
/// %merge1 = bitcast <vscale x 16 x i16> %f1 to <vscale x 8 x i32>
/// ```
bool VectorCombine::foldDeinterleaveIntrinsics(Instruction &I) {
+ if (foldDeinterleaveInterleavePair(I))
+ return true;
+
// This pattern involves bitcast that is not compatible with big endian.
if (DL->isBigEndian())
return false;
@@ -6579,8 +6704,6 @@ bool VectorCombine::run() {
return true;
if (scalarizeVPIntrinsic(I))
return true;
- if (foldInterleaveOfHighHalfTruncs(I))
- return true;
if (foldInterleaveIntrinsics(I))
return true;
if (foldBitcastOfVPLoad(I))
diff --git a/llvm/test/Transforms/VectorCombine/AArch64/deinterleave-interleave-pairs.ll b/llvm/test/Transforms/VectorCombine/AArch64/deinterleave-interleave-pairs.ll
new file mode 100644
index 0000000000000..6bb7d5220d963
--- /dev/null
+++ b/llvm/test/Transforms/VectorCombine/AArch64/deinterleave-interleave-pairs.ll
@@ -0,0 +1,590 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -passes=vector-combine %s -S -o - | FileCheck %s
+
+target triple = "aarch64-unknown-linux-gnu"
+
+define <vscale x 12 x i16> @deinterleave3_interleave3_direct(<vscale x 12 x i16> %v) {
+; SCAN3-LABEL: define <vscale x 12 x i16> @deinterleave3_interleave3_direct(
+; SCAN3-SAME: <vscale x 12 x i16> [[V:%.*]]) {
+; SCAN3-NEXT: ret <vscale x 12 x i16> [[V]]
+; CHECK-LABEL: define <vscale x 12 x i16> @deinterleave3_interleave3_direct(
+; CHECK-SAME: <vscale x 12 x i16> [[V:%.*]]) {
+; CHECK-NEXT: ret <vscale x 12 x i16> [[V]]
+;
+ %d = call { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } @llvm.vector.deinterleave3.nxv12i16(<vscale x 12 x i16> %v)
+ %f0 = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } %d, 0
+ %f1 = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } %d, 1
+ %f2 = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } %d, 2
+ %r = call <vscale x 12 x i16> @llvm.vector.interleave3.nxv12i16(<vscale x 4 x i16> %f0, <vscale x 4 x i16> %f1, <vscale x 4 x i16> %f2)
+ ret <vscale x 12 x i16> %r
+}
+
+define <vscale x 16 x i8> @deinterleave4_lshr_trunc_interleave4(<vscale x 16 x i16> %v) {
+; CHECK-LABEL: define <vscale x 16 x i8> @deinterleave4_lshr_trunc_interleave4(
+; CHECK-SAME: <vscale x 16 x i16> [[V:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = lshr <vscale x 16 x i16> [[V]], splat (i16 8)
+; CHECK-NEXT: [[R:%.*]] = trunc nuw <vscale x 16 x i16> [[TMP1]] to <vscale x 16 x i8>
+; CHECK-NEXT: ret <vscale x 16 x i8> [[R]]
+;
+ %d = call { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } @llvm.vector.deinterleave4.nxv16i16(<vscale x 16 x i16> %v)
+ %f0 = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } %d, 0
+ %f1 = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } %d, 1
+ %f2 = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } %d, 2
+ %f3 = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } %d, 3
+ %s0 = lshr <vscale x 4 x i16> %f0, splat (i16 8)
+ %t0 = trunc nuw <vscale x 4 x i16> %s0 to <vscale x 4 x i8>
+ %s1 = lshr <vscale x 4 x i16> %f1, splat (i16 8)
+ %t1 = trunc nuw <vscale x 4 x i16> %s1 to <vscale x 4 x i8>
+ %s2 = lshr <vscale x 4 x i16> %f2, splat (i16 8)
+ %t2 = trunc nuw <vscale x 4 x i16> %s2 to <vscale x 4 x i8>
+ %s3 = lshr <vscale x 4 x i16> %f3, splat (i16 8)
+ %t3 = trunc nuw <vscale x 4 x i16> %s3 to <vscale x 4 x i8>
+ %r = call <vscale x 16 x i8> @llvm.vector.interleave4.nxv16i8(<vscale x 4 x i8> %t0, <vscale x 4 x i8> %t1, <vscale x 4 x i8> %t2, <vscale x 4 x i8> %t3)
+ ret <vscale x 16 x i8> %r
+}
+
+define <vscale x 16 x i16> @deinterleave4_five_step_chain_interleave4(<vscale x 16 x i32> %v) {
+; CHECK-LABEL: define <vscale x 16 x i16> @deinterleave4_five_step_chain_interleave4(
+; CHECK-SAME: <vscale x 16 x i32> [[V:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = and <vscale x 16 x i32> [[V]], splat (i32 65535)
+; CHECK-NEXT: [[TMP2:%.*]] = lshr <vscale x 16 x i32> [[TMP1]], splat (i32 3)
+; CHECK-NEXT: [[TMP3:%.*]] = xor <vscale x 16 x i32> [[TMP2]], splat (i32 90)
+; CHECK-NEXT: [[TMP4:%.*]] = add <vscale x 16 x i32> [[TMP3]], splat (i32 7)
+; CHECK-NEXT: [[R:%.*]] = trunc <vscale x 16 x i32> [[TMP4]] to <vscale x 16 x i16>
+; CHECK-NEXT: ret <vscale x 16 x i16> [[R]]
+;
+ %d = call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave4.nxv16i32(<vscale x 16 x i32> %v)
+ %f0 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } %d, 0
+ %f1 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } %d, 1
+ %f2 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } %d, 2
+ %f3 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } %d, 3
+ %a0 = and <vscale x 4 x i32> %f0, splat (i32 65535)
+ %a1 = and <vscale x 4 x i32> %f1, splat (i32 65535)
+ %a2 = and <vscale x 4 x i32> %f2, splat (i32 65535)
+ %a3 = and <vscale x 4 x i32> %f3, splat (i32 65535)
+ %b0 = lshr <vscale x 4 x i32> %a0, splat (i32 3)
+ %b1 = lshr <vscale x 4 x i32> %a1, splat (i32 3)
+ %b2 = lshr <vscale x 4 x i32> %a2, splat (i32 3)
+ %b3 = lshr <vscale x 4 x i32> %a3, splat (i32 3)
+ %c0 = xor <vscale x 4 x i32> %b0, splat (i32 90)
+ %c1 = xor <vscale x 4 x i32> %b1, splat (i32 90)
+ %c2 = xor <vscale x 4 x i32> %b2, splat (i32 90)
+ %c3 = xor <vscale x 4 x i32> %b3, splat (i32 90)
+ %d0 = add <vscale x 4 x i32> %c0, splat (i32 7)
+ %d1 = add <vscale x 4 x i32> %c1, splat (i32 7)
+ %d2 = add <vscale x 4 x i32> %c2, splat (i32 7)
+ %d3 = add <vscale x 4 x i32> %c3, splat (i32 7)
+ %e0 = trunc <vscale x 4 x i32> %d0 to <vscale x 4 x i16>
+ %e1 = trunc <vscale x 4 x i32> %d1 to <vscale x 4 x i16>
+ %e2 = trunc <vscale x 4 x i32> %d2 to <vscale x 4 x i16>
+ %e3 = trunc <vscale x 4 x i32> %d3 to <vscale x 4 x i16>
+ %r = call <vscale x 16 x i16> @llvm.vector.interleave4.nxv16i16(<vscale x 4 x i16> %e0, <vscale x 4 x i16> %e1, <vscale x 4 x i16> %e2, <vscale x 4 x i16> %e3)
+ ret <vscale x 16 x i16> %r
+}
+
+define <vscale x 14 x i16> @deinterleave7_sub_interleave7_lhs_splat(<vscale x 14 x i16> %v) {
+; CHECK-LABEL: define <vscale x 14 x i16> @deinterleave7_sub_interleave7_lhs_splat(
+; CHECK-SAME: <vscale x 14 x i16> [[V:%.*]]) {
+; CHECK-NEXT: [[R:%.*]] = sub <vscale x 14 x i16> splat (i16 1023), [[V]]
+; CHECK-NEXT: ret <vscale x 14 x i16> [[R]]
+;
+ %d = call { <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16> } @llvm.vector.deinterleave7.nxv14i16(<vscale x 14 x i16> %v)
+ %f0 = extractvalue { <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16> } %d, 0
+ %f1 = extractvalue { <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16> } %d, 1
+ %f2 = extractvalue { <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16> } %d, 2
+ %f3 = extractvalue { <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16> } %d, 3
+ %f4 = extractvalue { <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16> } %d, 4
+ %f5 = extractvalue { <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16> } %d, 5
+ %f6 = extractvalue { <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16> } %d, 6
+ %u0 = sub <vscale x 2 x i16> splat (i16 1023), %f0
+ %u1 = sub <vscale x 2 x i16> splat (i16 1023), %f1
+ %u2 = sub <vscale x 2 x i16> splat (i16 1023), %f2
+ %u3 = sub <vscale x 2 x i16> splat (i16 1023), %f3
+ %u4 = sub <vscale x 2 x i16> splat (i16 1023), %f4
+ %u5 = sub <vscale x 2 x i16> splat (i16 1023), %f5
+ %u6 = sub <vscale x 2 x i16> splat (i16 1023), %f6
+ %r = call <vscale x 14 x i16> @llvm.vector.interleave7.nxv14i16(<vscale x 2 x i16> %u0, <vscale x 2 x i16> %u1, <vscale x 2 x i16> %u2, <vscale x 2 x i16> %u3, <vscale x 2 x i16> %u4, <vscale x 2 x i16> %u5, <vscale x 2 x i16> %u6)
+ ret <vscale x 14 x i16> %r
+}
+
+define <vscale x 16 x i32> @deinterleave8_and_interleave8(<vscale x 16 x i32> %v) {
+; CHECK-LABEL: define <vscale x 16 x i32> @deinterleave8_and_interleave8(
+; CHECK-SAME: <vscale x 16 x i32> [[V:%.*]]) {
+; CHECK-NEXT: [[R:%.*]] = and <vscale x 16 x i32> [[V]], splat (i32 255)
+; CHECK-NEXT: ret <vscale x 16 x i32> [[R]]
+;
+ %d = call { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } @llvm.vector.deinterleave8.nxv16i32(<vscale x 16 x i32> %v)
+ %f0 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } %d, 0
+ %f1 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } %d, 1
+ %f2 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } %d, 2
+ %f3 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } %d, 3
+ %f4 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } %d, 4
+ %f5 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } %d, 5
+ %f6 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } %d, 6
+ %f7 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } %d, 7
+ %u0 = and <vscale x 2 x i32> %f0, splat (i32 255)
+ %u1 = and <vscale x 2 x i32> %f1, splat (i32 255)
+ %u2 = and <vscale x 2 x i32> %f2, splat (i32 255)
+ %u3 = and <vscale x 2 x i32> %f3, splat (i32 255)
+ %u4 = and <vscale x 2 x i32> %f4, splat (i32 255)
+ %u5 = and <vscale x 2 x i32> %f5, splat (i32 255)
+ %u6 = and <vscale x 2 x i32> %f6, splat (i32 255)
+ %u7 = and <vscale x 2 x i32> %f7, splat (i32 255)
+ %r = call <vscale x 16 x i32> @llvm.vector.interleave8.nxv16i32(<vscale x 2 x i32> %u0, <vscale x 2 x i32> %u1, <vscale x 2 x i32> %u2, <vscale x 2 x i32> %u3, <vscale x 2 x i32> %u4, <vscale x 2 x i32> %u5, <vscale x 2 x i32> %u6, <vscale x 2 x i32> %u7)
+ ret <vscale x 16 x i32> %r
+}
+
+define <vscale x 16 x i16> @deinterleave4_zext_interleave4(<vscale x 16 x i8> %v) {
+; CHECK-LABEL: define <vscale x 16 x i16> @deinterleave4_zext_interleave4(
+; CHECK-SAME: <vscale x 16 x i8> [[V:%.*]]) {
+; CHECK-NEXT: [[R:%.*]] = zext <vscale x 16 x i8> [[V]] to <vscale x 16 x i16>
+; CHECK-NEXT: ret <vscale x 16 x i16> [[R]]
+;
+ %d = call { <vscale x 4 x i8>, <vscale x 4 x i8>, <vscale x 4 x i8>, <vscale x 4 x i8> } @llvm.vector.deinterleave4.nxv16i8(<vscale x 16 x i8> %v)
+ %f0 = extractvalue { <vscale x 4 x i8>, <vscale x 4 x i8>, <vscale x 4 x i8>, <vscale x 4 x i8> } %d, 0
+ %f1 = extractvalue { <vscale x 4 x i8>, <vscale x 4 x i8>, <vscale x 4 x i8>, <vscale x 4 x i8> } %d, 1
+ %f2 = extractvalue { <vscale x 4 x i8>, <vscale x 4 x i8>, <vscale x 4 x i8>, <vscale x 4 x i8> } %d, 2
+ %f3 = extractvalue { <vscale x 4 x i8>, <vscale x 4 x i8>, <vscale x 4 x i8>, <vscale x 4 x i8> } %d, 3
+ %u0 = zext <vscale x 4 x i8> %f0 to <vscale x 4 x i16>
+ %u1 = zext <vscale x 4 x i8> %f1 to <vscale x 4 x i16>
+ %u2 = zext <vscale x 4 x i8> %f2 to <vscale x 4 x i16>
+ %u3 = zext <vscale x 4 x i8> %f3 to <vscale x 4 x i16>
+ %r = call <vscale x 16 x i16> @llvm.vector.interleave4.nxv16i16(<vscale x 4 x i16> %u0, <vscale x 4 x i16> %u1, <vscale x 4 x i16> %u2, <vscale x 4 x i16> %u3)
+ ret <vscale x 16 x i16> %r
+}
+
+define <16 x i8> @fixed_deinterleave4_lshr_trunc_interleave4(<16 x i16> %v) {
+; CHECK-LABEL: define <16 x i8> @fixed_deinterleave4_lshr_trunc_interleave4(
+; CHECK-SAME: <16 x i16> [[V:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = lshr <16 x i16> [[V]], splat (i16 8)
+; CHECK-NEXT: [[R:%.*]] = trunc <16 x i16> [[TMP1]] to <16 x i8>
+; CHECK-NEXT: ret <16 x i8> [[R]]
+;
+ %d = call { <4 x i16>, <4 x i16>, <4 x i16>, <4 x i16> } @llvm.vector.deinterleave4.v16i16(<16 x i16> %v)
+ %f0 = extractvalue { <4 x i16>, <4 x i16>, <4 x i16>, <4 x i16> } %d, 0
+ %f1 = extractvalue { <4 x i16>, <4 x i16>, <4 x i16>, <4 x i16> } %d, 1
+ %f2 = extractvalue { <4 x i16>, <4 x i16>, <4 x i16>, <4 x i16> } %d, 2
+ %f3 = extractvalue { <4 x i16>, <4 x i16>, <4 x i16>, <4 x i16> } %d, 3
+ %s0 = lshr <4 x i16> %f0, splat (i16 8)
+ %t0 = trunc <4 x i16> %s0 to <4 x i8>
+ %s1 = lshr <4 x i16> %f1, splat (i16 8)
+ %t1 = trunc <4 x i16> %s1 to <4 x i8>
+ %s2 = lshr <4 x i16> %f2, splat (i16 8)
+ %t2 = trunc <4 x i16> %s2 to <4 x i8>
+ %s3 = lshr <4 x i16> %f3, splat (i16 8)
+ %t3 = trunc <4 x i16> %s3 to <4 x i8>
+ %r = call <16 x i8> @llvm.vector.interleave4.v16i8(<4 x i8> %t0, <4 x i8> %t1, <4 x i8> %t2, <4 x i8> %t3)
+ ret <16 x i8> %r
+}
+
+define <vscale x 8 x i16> @deinterleave2_add_separate_splats_interleave2(<vscale x 8 x i16> %v, i16 %x) {
+; CHECK-LABEL: define <vscale x 8 x i16> @deinterleave2_add_separate_splats_interleave2(
+; CHECK-SAME: <vscale x 8 x i16> [[V:%.*]], i16 [[X:%.*]]) {
+; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 8 x i16> poison, i16 [[X]], i64 0
+; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector <vscale x 8 x i16> [[DOTSPLATINSERT]], <vscale x 8 x i16> poison, <vscale x 8 x i32> zeroinitializer
+; CHECK-NEXT: [[R:%.*]] = add <vscale x 8 x i16> [[V]], [[DOTSPLAT]]
+; CHECK-NEXT: ret <vscale x 8 x i16> [[R]]
+;
+ %d = call { <vscale x 4 x i16>, <vscale x 4 x i16> } @llvm.vector.deinterleave2.nxv8i16(<vscale x 8 x i16> %v)
+ %f0 = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16> } %d, 0
+ %f1 = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16> } %d, 1
+ %insert0 = insertelement <vscale x 4 x i16> poison, i16 %x, i64 0
+ %splat0 = shufflevector <vscale x 4 x i16> %insert0, <vscale x 4 x i16> poison, <vscale x 4 x i32> zeroinitializer
+ %insert1 = insertelement <vscale x 4 x i16> poison, i16 %x, i64 0
+ %splat1 = shufflevector <vscale x 4 x i16> %insert1, <vscale x 4 x i16> poison, <vscale x 4 x i32> zeroinitializer
+ %u0 = add <vscale x 4 x i16> %f0, %splat0
+ %u1 = add <vscale x 4 x i16> %f1, %splat1
+ %r = call <vscale x 8 x i16> @llvm.vector.interleave2.nxv8i16(<vscale x 4 x i16> %u0, <vscale x 4 x i16> %u1)
+ ret <vscale x 8 x i16> %r
+}
+
+define <vscale x 8 x double> @deinterleave2_fpext_interleave2(<vscale x 8 x float> %v) {
+; CHECK-LABEL: define <vscale x 8 x double> @deinterleave2_fpext_interleave2(
+; CHECK-SAME: <vscale x 8 x float> [[V:%.*]]) {
+; CHECK-NEXT: [[R:%.*]] = fpext <vscale x 8 x float> [[V]] to <vscale x 8 x double>
+; CHECK-NEXT: ret <vscale x 8 x double> [[R]]
+;
+ %d = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.vector.deinterleave2.nxv8f32(<vscale x 8 x float> %v)
+ %f0 = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } %d, 0
+ %f1 = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } %d, 1
+ %u0 = fpext <vscale x 4 x float> %f0 to <vscale x 4 x double>
+ %u1 = fpext <vscale x 4 x float> %f1 to <vscale x 4 x double>
+ %r = call <vscale x 8 x double> @llvm.vector.interleave2.nxv8f64(<vscale x 4 x double> %u0, <vscale x 4 x double> %u1)
+ ret <vscale x 8 x double> %r
+}
+
+define <vscale x 8 x float> @deinterleave2_fneg_interleave2(<vscale x 8 x float> %v) {
+; CHECK-LABEL: define <vscale x 8 x float> @deinterleave2_fneg_interleave2(
+; CHECK-SAME: <vscale x 8 x float> [[V:%.*]]) {
+; CHECK-NEXT: [[R:%.*]] = fneg <vscale x 8 x float> [[V]]
+; CHECK-NEXT: ret <vscale x 8 x float> [[R]]
+;
+ %d = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.vector.deinterleave2.nxv8f32(<vscale x 8 x float> %v)
+ %f0 = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } %d, 0
+ %f1 = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } %d, 1
+ %u0 = fneg <vscale x 4 x float> %f0
+ %u1 = fneg <vscale x 4 x float> %f1
+ %r = call <vscale x 8 x float> @llvm.vector.interleave2.nxv8f32(<vscale x 4 x float> %u0, <vscale x 4 x float> %u1)
+ ret <vscale x 8 x float> %r
+}
+
+define <vscale x 8 x float> @deinterleave2_bitcast_interleave2(<vscale x 8 x i32> %v) {
+; CHECK-LABEL: define <vscale x 8 x float> @deinterleave2_bitcast_interleave2(
+; CHECK-SAME: <vscale x 8 x i32> [[V:%.*]]) {
+; CHECK-NEXT: [[R:%.*]] = bitcast <vscale x 8 x i32> [[V]] to <vscale x 8 x float>
+; CHECK-NEXT: ret <vscale x 8 x float> [[R]]
+;
+ %d = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> %v)
+ %f0 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } %d, 0
+ %f1 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } %d, 1
+ %u0 = bitcast <vscale x 4 x i32> %f0 to <vscale x 4 x float>
+ %u1 = bitcast <vscale x 4 x i32> %f1 to <vscale x 4 x float>
+ %r = call <vscale x 8 x float> @llvm.vector.interleave2.nxv8f32(<vscale x 4 x float> %u0, <vscale x 4 x float> %u1)
+ ret <vscale x 8 x float> %r
+}
+
+define <vscale x 8 x i1> @deinterleave2_icmp_interleave2(<vscale x 8 x i16> %v) {
+; CHECK-LABEL: define <vscale x 8 x i1> @deinterleave2_icmp_interleave2(
+; CHECK-SAME: <vscale x 8 x i16> [[V:%.*]]) {
+; CHECK-NEXT: [[R:%.*]] = icmp eq <vscale x 8 x i16> [[V]], zeroinitializer
+; CHECK-NEXT: ret <vscale x 8 x i1> [[R]]
+;
+ %d = call { <vscale x 4 x i16>, <vscale x 4 x i16> } @llvm.vector.deinterleave2.nxv8i16(<vscale x 8 x i16> %v)
+ %f0 = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16> } %d, 0
+ %f1 = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16> } %d, 1
+ %u0 = icmp eq <vscale x 4 x i16> %f0, zeroinitializer
+ %u1 = icmp eq <vscale x 4 x i16> %f1, zeroinitializer
+ %r = call <vscale x 8 x i1> @llvm.vector.interleave2.nxv8i1(<vscale x 4 x i1> %u0, <vscale x 4 x i1> %u1)
+ ret <vscale x 8 x i1> %r
+}
+
+define <vscale x 8 x i16> @deinterleave2_select_interleave2(<vscale x 8 x i1> %v) {
+; CHECK-LABEL: define <vscale x 8 x i16> @deinterleave2_select_interleave2(
+; CHECK-SAME: <vscale x 8 x i1> [[V:%.*]]) {
+; CHECK-NEXT: [[R:%.*]] = select <vscale x 8 x i1> [[V]], <vscale x 8 x i16> zeroinitializer, <vscale x 8 x i16> splat (i16 1)
+; CHECK-NEXT: ret <vscale x 8 x i16> [[R]]
+;
+ %d = call { <vscale x 4 x i1>, <vscale x 4 x i1> } @llvm.vector.deinterleave2.nxv8i1(<vscale x 8 x i1> %v)
+ %f0 = extractvalue { <vscale x 4 x i1>, <vscale x 4 x i1> } %d, 0
+ %f1 = extractvalue { <vscale x 4 x i1>, <vscale x 4 x i1> } %d, 1
+ %u0 = select <vscale x 4 x i1> %f0, <vscale x 4 x i16> zeroinitializer, <vscale x 4 x i16> splat (i16 1)
+ %u1 = select <vscale x 4 x i1> %f1, <vscale x 4 x i16> zeroinitializer, <vscale x 4 x i16> splat (i16 1)
+ %r = call <vscale x 8 x i16> @llvm.vector.interleave2.nxv8i16(<vscale x 4 x i16> %u0, <vscale x 4 x i16> %u1)
+ ret <vscale x 8 x i16> %r
+}
+
+define <vscale x 8 x i16> @deinterleave2_freeze_interleave2(<vscale x 8 x i16> %v) {
+; CHECK-LABEL: define <vscale x 8 x i16> @deinterleave2_freeze_interleave2(
+; CHECK-SAME: <vscale x 8 x i16> [[V:%.*]]) {
+; CHECK-NEXT: [[R:%.*]] = freeze <vscale x 8 x i16> [[V]]
+; CHECK-NEXT: ret <vscale x 8 x i16> [[R]]
+;
+ %d = call { <vscale x 4 x i16>, <vscale x 4 x i16> } @llvm.vector.deinterleave2.nxv8i16(<vscale x 8 x i16> %v)
+ %f0 = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16> } %d, 0
+ %f1 = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16> } %d, 1
+ %u0 = freeze <vscale x 4 x i16> %f0
+ %u1 = freeze <vscale x 4 x i16> %f1
+ %r = call <vscale x 8 x i16> @llvm.vector.interleave2.nxv8i16(<vscale x 4 x i16> %u0, <vscale x 4 x i16> %u1)
+ ret <vscale x 8 x i16> %r
+}
+
+define <vscale x 8 x i16> @deinterleave2_abs_interleave2(<vscale x 8 x i16> %v) {
+; CHECK-LABEL: define <vscale x 8 x i16> @deinterleave2_abs_interleave2(
+; CHECK-SAME: <vscale x 8 x i16> [[V:%.*]]) {
+; CHECK-NEXT: [[R:%.*]] = call <vscale x 8 x i16> @llvm.abs.nxv8i16(<vscale x 8 x i16> [[V]], i1 false)
+; CHECK-NEXT: ret <vscale x 8 x i16> [[R]]
+;
+ %d = call { <vscale x 4 x i16>, <vscale x 4 x i16> } @llvm.vector.deinterleave2.nxv8i16(<vscale x 8 x i16> %v)
+ %f0 = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16> } %d, 0
+ %f1 = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16> } %d, 1
+ %u0 = call <vscale x 4 x i16> @llvm.abs.nxv4i16(<vscale x 4 x i16> %f0, i1 false)
+ %u1 = call <vscale x 4 x i16> @llvm.abs.nxv4i16(<vscale x 4 x i16> %f1, i1 false)
+ %r = call <vscale x 8 x i16> @llvm.vector.interleave2.nxv8i16(<vscale x 4 x i16> %u0, <vscale x 4 x i16> %u1)
+ ret <vscale x 8 x i16> %r
+}
+
+define <vscale x 8 x i16> @deinterleave2_intersect_flags_interleave2(<vscale x 8 x i16> %v) {
+; CHECK-LABEL: define <vscale x 8 x i16> @deinterleave2_intersect_flags_interleave2(
+; CHECK-SAME: <vscale x 8 x i16> [[V:%.*]]) {
+; CHECK-NEXT: [[R:%.*]] = add <vscale x 8 x i16> [[V]], splat (i16 1)
+; CHECK-NEXT: ret <vscale x 8 x i16> [[R]]
+;
+ %d = call { <vscale x 4 x i16>, <vscale x 4 x i16> } @llvm.vector.deinterleave2.nxv8i16(<vscale x 8 x i16> %v)
+ %f0 = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16> } %d, 0
+ %f1 = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16> } %d, 1
+ %u0 = add nuw <vscale x 4 x i16> %f0, splat (i16 1)
+ %u1 = add <vscale x 4 x i16> %f1, splat (i16 1)
+ %r = call <vscale x 8 x i16> @llvm.vector.interleave2.nxv8i16(<vscale x 4 x i16> %u0, <vscale x 4 x i16> %u1)
+ ret <vscale x 8 x i16> %r
+}
+
+define <vscale x 16 x i16> @control_flow_sdiv(<vscale x 16 x i16> %v, i1 %cond, i16 %splat_value) {
+; CHECK-LABEL: define <vscale x 16 x i16> @control_flow_sdiv(
+; CHECK-SAME: <vscale x 16 x i16> [[V:%.*]], i1 [[COND:%.*]], i16 [[SPLAT_VALUE:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK: [[THEN]]:
+; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 16 x i16> poison, i16 [[SPLAT_VALUE]], i64 0
+; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector <vscale x 16 x i16> [[DOTSPLATINSERT]], <vscale x 16 x i16> poison, <vscale x 16 x i32> zeroinitializer
+; CHECK-NEXT: [[R:%.*]] = sdiv <vscale x 16 x i16> [[V]], [[DOTSPLAT]]
+; CHECK-NEXT: ret <vscale x 16 x i16> [[R]]
+; CHECK: [[ELSE]]:
+; CHECK-NEXT: ret <vscale x 16 x i16> zeroinitializer
+;
+entry:
+ %d = call { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } @llvm.vector.deinterleave4.nxv16i16(<vscale x 16 x i16> %v)
+ %f0 = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } %d, 0
+ %f1 = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } %d, 1
+ %f2 = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } %d, 2
+ %f3 = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } %d, 3
+ %insert = insertelement <vscale x 4 x i16> poison, i16 %splat_value, i64 0
+ %splat = shufflevector <vscale x 4 x i16> %insert, <vscale x 4 x i16> poison, <vscale x 4 x i32> zeroinitializer
+ br i1 %cond, label %then, label %else
+
+then:
+ %a0 = sdiv <vscale x 4 x i16> %f0, %splat
+ %a1 = sdiv <vscale x 4 x i16> %f1, %splat
+ %a2 = sdiv <vscale x 4 x i16> %f2, %splat
+ %a3 = sdiv <vscale x 4 x i16> %f3, %splat
+ %r = call <vscale x 16 x i16> @llvm.vector.interleave4.nxv16i16(<vscale x 4 x i16> %a0, <vscale x 4 x i16> %a1, <vscale x 4 x i16> %a2, <vscale x 4 x i16> %a3)
+ ret <vscale x 16 x i16> %r
+
+else:
+ ret <vscale x 16 x i16> zeroinitializer
+}
+
+; Negative test: operand bundles on the deinterleave must be preserved.
+define <vscale x 8 x i16> @negative_deinterleave2_operand_bundle(<vscale x 8 x i16> %v) {
+; CHECK-LABEL: define <vscale x 8 x i16> @negative_deinterleave2_operand_bundle(
+; CHECK-SAME: <vscale x 8 x i16> [[V:%.*]]) {
+; CHECK-NEXT: [[D:%.*]] = call { <vscale x 4 x i16>, <vscale x 4 x i16> } @llvm.vector.deinterleave2.nxv8i16(<vscale x 8 x i16> [[V]]) [ "deopt"(i32 0) ]
+; CHECK-NEXT: [[F0:%.*]] = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16> } [[D]], 0
+; CHECK-NEXT: [[F1:%.*]] = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16> } [[D]], 1
+; CHECK-NEXT: [[R:%.*]] = call <vscale x 8 x i16> @llvm.vector.interleave2.nxv8i16(<vscale x 4 x i16> [[F0]], <vscale x 4 x i16> [[F1]])
+; CHECK-NEXT: ret <vscale x 8 x i16> [[R]]
+;
+ %d = call { <vscale x 4 x i16>, <vscale x 4 x i16> } @llvm.vector.deinterleave2.nxv8i16(<vscale x 8 x i16> %v) [ "deopt"(i32 0) ]
+ %f0 = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16> } %d, 0
+ %f1 = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16> } %d, 1
+ %r = call <vscale x 8 x i16> @llvm.vector.interleave2.nxv8i16(<vscale x 4 x i16> %f0, <vscale x 4 x i16> %f1)
+ ret <vscale x 8 x i16> %r
+}
+
+; Negative test: one chain uses a different shift amount, so the fold must not happen.
+define <vscale x 16 x i8> @negative_deinterleave4_mismatched_shift_amount(<vscale x 16 x i16> %v) {
+; CHECK-LABEL: define <vscale x 16 x i8> @negative_deinterleave4_mismatched_shift_amount(
+; CHECK-SAME: <vscale x 16 x i16> [[V:%.*]]) {
+; CHECK-NEXT: [[D:%.*]] = call { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } @llvm.vector.deinterleave4.nxv16i16(<vscale x 16 x i16> [[V]])
+; CHECK-NEXT: [[F0:%.*]] = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } [[D]], 0
+; CHECK-NEXT: [[F1:%.*]] = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } [[D]], 1
+; CHECK-NEXT: [[F2:%.*]] = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } [[D]], 2
+; CHECK-NEXT: [[F3:%.*]] = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } [[D]], 3
+; CHECK-NEXT: [[S0:%.*]] = lshr <vscale x 4 x i16> [[F0]], splat (i16 8)
+; CHECK-NEXT: [[T0:%.*]] = trunc nuw <vscale x 4 x i16> [[S0]] to <vscale x 4 x i8>
+; CHECK-NEXT: [[S1:%.*]] = lshr <vscale x 4 x i16> [[F1]], splat (i16 8)
+; CHECK-NEXT: [[T1:%.*]] = trunc nuw <vscale x 4 x i16> [[S1]] to <vscale x 4 x i8>
+; CHECK-NEXT: [[S2:%.*]] = lshr <vscale x 4 x i16> [[F2]], splat (i16 7)
+; CHECK-NEXT: [[T2:%.*]] = trunc nuw <vscale x 4 x i16> [[S2]] to <vscale x 4 x i8>
+; CHECK-NEXT: [[S3:%.*]] = lshr <vscale x 4 x i16> [[F3]], splat (i16 8)
+; CHECK-NEXT: [[T3:%.*]] = trunc nuw <vscale x 4 x i16> [[S3]] to <vscale x 4 x i8>
+; CHECK-NEXT: [[R:%.*]] = call <vscale x 16 x i8> @llvm.vector.interleave4.nxv16i8(<vscale x 4 x i8> [[T0]], <vscale x 4 x i8> [[T1]], <vscale x 4 x i8> [[T2]], <vscale x 4 x i8> [[T3]])
+; CHECK-NEXT: ret <vscale x 16 x i8> [[R]]
+;
+ %d = call { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } @llvm.vector.deinterleave4.nxv16i16(<vscale x 16 x i16> %v)
+ %f0 = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } %d, 0
+ %f1 = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } %d, 1
+ %f2 = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } %d, 2
+ %f3 = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } %d, 3
+ %s0 = lshr <vscale x 4 x i16> %f0, splat (i16 8)
+ %t0 = trunc nuw <vscale x 4 x i16> %s0 to <vscale x 4 x i8>
+ %s1 = lshr <vscale x 4 x i16> %f1, splat (i16 8)
+ %t1 = trunc nuw <vscale x 4 x i16> %s1 to <vscale x 4 x i8>
+ %s2 = lshr <vscale x 4 x i16> %f2, splat (i16 7)
+ %t2 = trunc nuw <vscale x 4 x i16> %s2 to <vscale x 4 x i8>
+ %s3 = lshr <vscale x 4 x i16> %f3, splat (i16 8)
+ %t3 = trunc nuw <vscale x 4 x i16> %s3 to <vscale x 4 x i8>
+ %r = call <vscale x 16 x i8> @llvm.vector.interleave4.nxv16i8(<vscale x 4 x i8> %t0, <vscale x 4 x i8> %t1, <vscale x 4 x i8> %t2, <vscale x 4 x i8> %t3)
+ ret <vscale x 16 x i8> %r
+}
+
+; Negative test - the fold shouldn't be generated as the deitnerleaved fields are passed in the wrong order.
+define <vscale x 4 x i16> @negative_deinterleave2_reordered_fields(<vscale x 4 x i32> %x) {
+; CHECK-LABEL: define <vscale x 4 x i16> @negative_deinterleave2_reordered_fields(
+; CHECK-SAME: <vscale x 4 x i32> [[X:%.*]]) {
+; CHECK-NEXT: [[D:%.*]] = call { <vscale x 2 x i32>, <vscale x 2 x i32> } @llvm.vector.deinterleave2.nxv4i32(<vscale x 4 x i32> [[X]])
+; CHECK-NEXT: [[D0:%.*]] = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } [[D]], 0
+; CHECK-NEXT: [[D1:%.*]] = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } [[D]], 1
+; CHECK-NEXT: [[S0:%.*]] = lshr <vscale x 2 x i32> [[D0]], splat (i32 16)
+; CHECK-NEXT: [[S1:%.*]] = lshr <vscale x 2 x i32> [[D1]], splat (i32 16)
+; CHECK-NEXT: [[T0:%.*]] = trunc <vscale x 2 x i32> [[S0]] to <vscale x 2 x i16>
+; CHECK-NEXT: [[T1:%.*]] = trunc <vscale x 2 x i32> [[S1]] to <vscale x 2 x i16>
+; CHECK-NEXT: [[R:%.*]] = call <vscale x 4 x i16> @llvm.vector.interleave2.nxv4i16(<vscale x 2 x i16> [[T1]], <vscale x 2 x i16> [[T0]])
+; CHECK-NEXT: ret <vscale x 4 x i16> [[R]]
+;
+ %d = call { <vscale x 2 x i32>, <vscale x 2 x i32> } @llvm.vector.deinterleave2.nxv4i32(<vscale x 4 x i32> %x)
+ %d0 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } %d, 0
+ %d1 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } %d, 1
+ %s0 = lshr <vscale x 2 x i32> %d0, splat (i32 16)
+ %s1 = lshr <vscale x 2 x i32> %d1, splat (i32 16)
+ %t0 = trunc <vscale x 2 x i32> %s0 to <vscale x 2 x i16>
+ %t1 = trunc <vscale x 2 x i32> %s1 to <vscale x 2 x i16>
+ %r = call <vscale x 4 x i16> @llvm.vector.interleave2.nxv4i16(<vscale x 2 x i16> %t1, <vscale x 2 x i16> %t0)
+ ret <vscale x 4 x i16> %r
+}
+
+; Negative test: reusing one extracted field and skipping another means the fold must not happen.
+define <vscale x 16 x i16> @negative_deinterleave4_duplicate_extract_operand(<vscale x 16 x i16> %v) {
+; CHECK-LABEL: define <vscale x 16 x i16> @negative_deinterleave4_duplicate_extract_operand(
+; CHECK-SAME: <vscale x 16 x i16> [[V:%.*]]) {
+; CHECK-NEXT: [[D:%.*]] = call { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } @llvm.vector.deinterleave4.nxv16i16(<vscale x 16 x i16> [[V]])
+; CHECK-NEXT: [[F0:%.*]] = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } [[D]], 0
+; CHECK-NEXT: [[F1:%.*]] = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } [[D]], 1
+; CHECK-NEXT: [[F2:%.*]] = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } [[D]], 2
+; CHECK-NEXT: [[F0_DUP:%.*]] = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } [[D]], 0
+; CHECK-NEXT: [[R:%.*]] = call <vscale x 16 x i16> @llvm.vector.interleave4.nxv16i16(<vscale x 4 x i16> [[F0]], <vscale x 4 x i16> [[F1]], <vscale x 4 x i16> [[F2]], <vscale x 4 x i16> [[F0_DUP]])
+; CHECK-NEXT: ret <vscale x 16 x i16> [[R]]
+;
+ %d = call { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } @llvm.vector.deinterleave4.nxv16i16(<vscale x 16 x i16> %v)
+ %f0 = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } %d, 0
+ %f1 = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } %d, 1
+ %f2 = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } %d, 2
+ %f0.dup = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } %d, 0
+ %r = call <vscale x 16 x i16> @llvm.vector.interleave4.nxv16i16(<vscale x 4 x i16> %f0, <vscale x 4 x i16> %f1, <vscale x 4 x i16> %f2, <vscale x 4 x i16> %f0.dup)
+ ret <vscale x 16 x i16> %r
+}
+
+; Negative test: changing the extracted-value operand position in one chain means the fold must not happen.
+define <vscale x 16 x i16> @negative_deinterleave4_mismatched_operand_position(<vscale x 16 x i16> %v) {
+; CHECK-LABEL: define <vscale x 16 x i16> @negative_deinterleave4_mismatched_operand_position(
+; CHECK-SAME: <vscale x 16 x i16> [[V:%.*]]) {
+; CHECK-NEXT: [[D:%.*]] = call { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } @llvm.vector.deinterleave4.nxv16i16(<vscale x 16 x i16> [[V]])
+; CHECK-NEXT: [[F0:%.*]] = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } [[D]], 0
+; CHECK-NEXT: [[F1:%.*]] = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } [[D]], 1
+; CHECK-NEXT: [[F2:%.*]] = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } [[D]], 2
+; CHECK-NEXT: [[F3:%.*]] = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } [[D]], 3
+; CHECK-NEXT: [[U0:%.*]] = sub <vscale x 4 x i16> [[F0]], splat (i16 5)
+; CHECK-NEXT: [[U1:%.*]] = sub <vscale x 4 x i16> splat (i16 5), [[F1]]
+; CHECK-NEXT: [[U2:%.*]] = sub <vscale x 4 x i16> splat (i16 5), [[F2]]
+; CHECK-NEXT: [[U3:%.*]] = sub <vscale x 4 x i16> splat (i16 5), [[F3]]
+; CHECK-NEXT: [[R:%.*]] = call <vscale x 16 x i16> @llvm.vector.interleave4.nxv16i16(<vscale x 4 x i16> [[U0]], <vscale x 4 x i16> [[U1]], <vscale x 4 x i16> [[U2]], <vscale x 4 x i16> [[U3]])
+; CHECK-NEXT: ret <vscale x 16 x i16> [[R]]
+;
+ %d = call { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } @llvm.vector.deinterleave4.nxv16i16(<vscale x 16 x i16> %v)
+ %f0 = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } %d, 0
+ %f1 = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } %d, 1
+ %f2 = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } %d, 2
+ %f3 = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } %d, 3
+ %u0 = sub <vscale x 4 x i16> %f0, splat (i16 5)
+ %u1 = sub <vscale x 4 x i16> splat (i16 5), %f1
+ %u2 = sub <vscale x 4 x i16> splat (i16 5), %f2
+ %u3 = sub <vscale x 4 x i16> splat (i16 5), %f3
+ %r = call <vscale x 16 x i16> @llvm.vector.interleave4.nxv16i16(<vscale x 4 x i16> %u0, <vscale x 4 x i16> %u1, <vscale x 4 x i16> %u2, <vscale x 4 x i16> %u3)
+ ret <vscale x 16 x i16> %r
+}
+
+; Negative test: compare predicates are part of the operation and must match.
+define <vscale x 8 x i1> @negative_deinterleave2_mismatched_predicates(<vscale x 8 x i16> %v) {
+; CHECK-LABEL: define <vscale x 8 x i1> @negative_deinterleave2_mismatched_predicates(
+; CHECK-SAME: <vscale x 8 x i16> [[V:%.*]]) {
+; CHECK-NEXT: [[D:%.*]] = call { <vscale x 4 x i16>, <vscale x 4 x i16> } @llvm.vector.deinterleave2.nxv8i16(<vscale x 8 x i16> [[V]])
+; CHECK-NEXT: [[F0:%.*]] = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16> } [[D]], 0
+; CHECK-NEXT: [[F1:%.*]] = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16> } [[D]], 1
+; CHECK-NEXT: [[U0:%.*]] = icmp eq <vscale x 4 x i16> [[F0]], zeroinitializer
+; CHECK-NEXT: [[U1:%.*]] = icmp ne <vscale x 4 x i16> [[F1]], zeroinitializer
+; CHECK-NEXT: [[R:%.*]] = call <vscale x 8 x i1> @llvm.vector.interleave2.nxv8i1(<vscale x 4 x i1> [[U0]], <vscale x 4 x i1> [[U1]])
+; CHECK-NEXT: ret <vscale x 8 x i1> [[R]]
+;
+ %d = call { <vscale x 4 x i16>, <vscale x 4 x i16> } @llvm.vector.deinterleave2.nxv8i16(<vscale x 8 x i16> %v)
+ %f0 = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16> } %d, 0
+ %f1 = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16> } %d, 1
+ %u0 = icmp eq <vscale x 4 x i16> %f0, zeroinitializer
+ %u1 = icmp ne <vscale x 4 x i16> %f1, zeroinitializer
+ %r = call <vscale x 8 x i1> @llvm.vector.interleave2.nxv8i1(<vscale x 4 x i1> %u0, <vscale x 4 x i1> %u1)
+ ret <vscale x 8 x i1> %r
+}
+
+; Negative test: non-data intrinsic operands must match across every chain.
+define <vscale x 8 x i16> @negative_deinterleave2_mismatched_intrinsic_scalar(<vscale x 8 x i16> %v) {
+; CHECK-LABEL: define <vscale x 8 x i16> @negative_deinterleave2_mismatched_intrinsic_scalar(
+; CHECK-SAME: <vscale x 8 x i16> [[V:%.*]]) {
+; CHECK-NEXT: [[D:%.*]] = call { <vscale x 4 x i16>, <vscale x 4 x i16> } @llvm.vector.deinterleave2.nxv8i16(<vscale x 8 x i16> [[V]])
+; CHECK-NEXT: [[F0:%.*]] = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16> } [[D]], 0
+; CHECK-NEXT: [[F1:%.*]] = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16> } [[D]], 1
+; CHECK-NEXT: [[U0:%.*]] = call <vscale x 4 x i16> @llvm.abs.nxv4i16(<vscale x 4 x i16> [[F0]], i1 false)
+; CHECK-NEXT: [[U1:%.*]] = call <vscale x 4 x i16> @llvm.abs.nxv4i16(<vscale x 4 x i16> [[F1]], i1 true)
+; CHECK-NEXT: [[R:%.*]] = call <vscale x 8 x i16> @llvm.vector.interleave2.nxv8i16(<vscale x 4 x i16> [[U0]], <vscale x 4 x i16> [[U1]])
+; CHECK-NEXT: ret <vscale x 8 x i16> [[R]]
+;
+ %d = call { <vscale x 4 x i16>, <vscale x 4 x i16> } @llvm.vector.deinterleave2.nxv8i16(<vscale x 8 x i16> %v)
+ %f0 = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16> } %d, 0
+ %f1 = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16> } %d, 1
+ %u0 = call <vscale x 4 x i16> @llvm.abs.nxv4i16(<vscale x 4 x i16> %f0, i1 false)
+ %u1 = call <vscale x 4 x i16> @llvm.abs.nxv4i16(<vscale x 4 x i16> %f1, i1 true)
+ %r = call <vscale x 8 x i16> @llvm.vector.interleave2.nxv8i16(<vscale x 4 x i16> %u0, <vscale x 4 x i16> %u1)
+ ret <vscale x 8 x i16> %r
+}
+
+; Negative test: a bitcast that changes the element count is not elementwise.
+define <vscale x 16 x i8> @negative_deinterleave2_element_count_changing_bitcast(<vscale x 8 x i16> %v) {
+; CHECK-LABEL: define <vscale x 16 x i8> @negative_deinterleave2_element_count_changing_bitcast(
+; CHECK-SAME: <vscale x 8 x i16> [[V:%.*]]) {
+; CHECK-NEXT: [[D:%.*]] = call { <vscale x 4 x i16>, <vscale x 4 x i16> } @llvm.vector.deinterleave2.nxv8i16(<vscale x 8 x i16> [[V]])
+; CHECK-NEXT: [[F0:%.*]] = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16> } [[D]], 0
+; CHECK-NEXT: [[F1:%.*]] = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16> } [[D]], 1
+; CHECK-NEXT: [[U0:%.*]] = bitcast <vscale x 4 x i16> [[F0]] to <vscale x 8 x i8>
+; CHECK-NEXT: [[U1:%.*]] = bitcast <vscale x 4 x i16> [[F1]] to <vscale x 8 x i8>
+; CHECK-NEXT: [[R:%.*]] = call <vscale x 16 x i8> @llvm.vector.interleave2.nxv16i8(<vscale x 8 x i8> [[U0]], <vscale x 8 x i8> [[U1]])
+; CHECK-NEXT: ret <vscale x 16 x i8> [[R]]
+;
+ %d = call { <vscale x 4 x i16>, <vscale x 4 x i16> } @llvm.vector.deinterleave2.nxv8i16(<vscale x 8 x i16> %v)
+ %f0 = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16> } %d, 0
+ %f1 = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16> } %d, 1
+ %u0 = bitcast <vscale x 4 x i16> %f0 to <vscale x 8 x i8>
+ %u1 = bitcast <vscale x 4 x i16> %f1 to <vscale x 8 x i8>
+ %r = call <vscale x 16 x i8> @llvm.vector.interleave2.nxv16i8(<vscale x 8 x i8> %u0, <vscale x 8 x i8> %u1)
+ ret <vscale x 16 x i8> %r
+}
+
+; Negative test: arbitrary vector operands cannot be widened like splats.
+define <vscale x 8 x i16> @negative_deinterleave2_non_splat_vector_operand(<vscale x 8 x i16> %v, <vscale x 4 x i16> %x0, <vscale x 4 x i16> %x1) {
+; CHECK-LABEL: define <vscale x 8 x i16> @negative_deinterleave2_non_splat_vector_operand(
+; CHECK-SAME: <vscale x 8 x i16> [[V:%.*]], <vscale x 4 x i16> [[X0:%.*]], <vscale x 4 x i16> [[X1:%.*]]) {
+; CHECK-NEXT: [[D:%.*]] = call { <vscale x 4 x i16>, <vscale x 4 x i16> } @llvm.vector.deinterleave2.nxv8i16(<vscale x 8 x i16> [[V]])
+; CHECK-NEXT: [[F0:%.*]] = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16> } [[D]], 0
+; CHECK-NEXT: [[F1:%.*]] = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16> } [[D]], 1
+; CHECK-NEXT: [[U0:%.*]] = add <vscale x 4 x i16> [[F0]], [[X0]]
+; CHECK-NEXT: [[U1:%.*]] = add <vscale x 4 x i16> [[F1]], [[X1]]
+; CHECK-NEXT: [[R:%.*]] = call <vscale x 8 x i16> @llvm.vector.interleave2.nxv8i16(<vscale x 4 x i16> [[U0]], <vscale x 4 x i16> [[U1]])
+; CHECK-NEXT: ret <vscale x 8 x i16> [[R]]
+;
+ %d = call { <vscale x 4 x i16>, <vscale x 4 x i16> } @llvm.vector.deinterleave2.nxv8i16(<vscale x 8 x i16> %v)
+ %f0 = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16> } %d, 0
+ %f1 = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16> } %d, 1
+ %u0 = add <vscale x 4 x i16> %f0, %x0
+ %u1 = add <vscale x 4 x i16> %f1, %x1
+ %r = call <vscale x 8 x i16> @llvm.vector.interleave2.nxv8i16(<vscale x 4 x i16> %u0, <vscale x 4 x i16> %u1)
+ ret <vscale x 8 x i16> %r
+}
+
+; Negative test: the fold must not remove an intermediate value with another use.
+define <vscale x 8 x i16> @negative_deinterleave2_extra_intermediate_use(<vscale x 8 x i16> %v, ptr %p) {
+; CHECK-LABEL: define <vscale x 8 x i16> @negative_deinterleave2_extra_intermediate_use(
+; CHECK-SAME: <vscale x 8 x i16> [[V:%.*]], ptr [[P:%.*]]) {
+; CHECK-NEXT: [[D:%.*]] = call { <vscale x 4 x i16>, <vscale x 4 x i16> } @llvm.vector.deinterleave2.nxv8i16(<vscale x 8 x i16> [[V]])
+; CHECK-NEXT: [[F0:%.*]] = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16> } [[D]], 0
+; CHECK-NEXT: [[F1:%.*]] = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16> } [[D]], 1
+; CHECK-NEXT: [[U0:%.*]] = add <vscale x 4 x i16> [[F0]], splat (i16 1)
+; CHECK-NEXT: [[U1:%.*]] = add <vscale x 4 x i16> [[F1]], splat (i16 1)
+; CHECK-NEXT: store <vscale x 4 x i16> [[U0]], ptr [[P]], align 8
+; CHECK-NEXT: [[R:%.*]] = call <vscale x 8 x i16> @llvm.vector.interleave2.nxv8i16(<vscale x 4 x i16> [[U0]], <vscale x 4 x i16> [[U1]])
+; CHECK-NEXT: ret <vscale x 8 x i16> [[R]]
+;
+ %d = call { <vscale x 4 x i16>, <vscale x 4 x i16> } @llvm.vector.deinterleave2.nxv8i16(<vscale x 8 x i16> %v)
+ %f0 = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16> } %d, 0
+ %f1 = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16> } %d, 1
+ %u0 = add <vscale x 4 x i16> %f0, splat (i16 1)
+ %u1 = add <vscale x 4 x i16> %f1, splat (i16 1)
+ store <vscale x 4 x i16> %u0, ptr %p
+ %r = call <vscale x 8 x i16> @llvm.vector.interleave2.nxv8i16(<vscale x 4 x i16> %u0, <vscale x 4 x i16> %u1)
+ ret <vscale x 8 x i16> %r
+}
diff --git a/llvm/test/CodeGen/AArch64/sve-vectorcombine-interleave.ll b/llvm/test/Transforms/VectorCombine/AArch64/sve-vectorcombine-interleave.ll
similarity index 89%
rename from llvm/test/CodeGen/AArch64/sve-vectorcombine-interleave.ll
rename to llvm/test/Transforms/VectorCombine/AArch64/sve-vectorcombine-interleave.ll
index 731bb482bcb34..11537a5291cbc 100644
--- a/llvm/test/CodeGen/AArch64/sve-vectorcombine-interleave.ll
+++ b/llvm/test/Transforms/VectorCombine/AArch64/sve-vectorcombine-interleave.ll
@@ -6,9 +6,11 @@ target triple = "aarch64"
define void @high_half_trunc_interleave4(ptr %src, ptr %dst) {
; CHECK-LABEL: high_half_trunc_interleave4:
; CHECK: // %bb.0:
-; CHECK-NEXT: ldr z0, [x0, #1, mul vl]
-; CHECK-NEXT: ldr z1, [x0]
-; CHECK-NEXT: uzp2 z0.b, z1.b, z0.b
+; CHECK-NEXT: ldr z0, [x0]
+; CHECK-NEXT: ldr z1, [x0, #1, mul vl]
+; CHECK-NEXT: lsr z1.h, z1.h, #8
+; CHECK-NEXT: lsr z0.h, z0.h, #8
+; CHECK-NEXT: uzp1 z0.b, z0.b, z1.b
; CHECK-NEXT: str z0, [x1]
; CHECK-NEXT: ret
%x = load <vscale x 16 x i16>, ptr %src, align 2
@@ -35,9 +37,12 @@ define void @high_half_trunc_interleave4(ptr %src, ptr %dst) {
define void @high_half_trunc_interleave4_byte_load(ptr %src, ptr %dst) {
; CHECK-LABEL: high_half_trunc_interleave4_byte_load:
; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: ptrue p0.b
-; CHECK-NEXT: ld2b { z0.b, z1.b }, p0/z, [x0]
-; CHECK-NEXT: str z1, [x1]
+; CHECK-NEXT: ldr z0, [x0]
+; CHECK-NEXT: ldr z1, [x0, #1, mul vl]
+; CHECK-NEXT: lsr z1.h, z1.h, #8
+; CHECK-NEXT: lsr z0.h, z0.h, #8
+; CHECK-NEXT: uzp1 z0.b, z0.b, z1.b
+; CHECK-NEXT: str z0, [x1]
; CHECK-NEXT: ret
entry:
%bytes = load <vscale x 32 x i8>, ptr %src, align 2
diff --git a/llvm/test/Transforms/VectorCombine/fold-interleave.ll b/llvm/test/Transforms/VectorCombine/fold-interleave.ll
deleted file mode 100644
index efe51333c024a..0000000000000
--- a/llvm/test/Transforms/VectorCombine/fold-interleave.ll
+++ /dev/null
@@ -1,209 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
-; RUN: opt -passes=vector-combine -S %s | FileCheck %s
-; RUN: opt -passes=vector-combine -data-layout=E -S %s | FileCheck %s --check-prefixes=BE
-
-define <vscale x 8 x i8> @lshr_trunc_interleave4(<vscale x 8 x i16> %x) {
-; CHECK-LABEL: define <vscale x 8 x i8> @lshr_trunc_interleave4(
-; CHECK-SAME: <vscale x 8 x i16> [[X:%.*]]) {
-; CHECK-NEXT: [[TMP1:%.*]] = bitcast <vscale x 8 x i16> [[X]] to <vscale x 16 x i8>
-; CHECK-NEXT: [[TMP2:%.*]] = call { <vscale x 8 x i8>, <vscale x 8 x i8> } @llvm.vector.deinterleave2.nxv16i8(<vscale x 16 x i8> [[TMP1]])
-; CHECK-NEXT: [[R:%.*]] = extractvalue { <vscale x 8 x i8>, <vscale x 8 x i8> } [[TMP2]], 1
-; CHECK-NEXT: ret <vscale x 8 x i8> [[R]]
-;
-; BE-LABEL: define <vscale x 8 x i8> @lshr_trunc_interleave4(
-; BE-SAME: <vscale x 8 x i16> [[X:%.*]]) {
-; BE-NEXT: [[TMP1:%.*]] = bitcast <vscale x 8 x i16> [[X]] to <vscale x 16 x i8>
-; BE-NEXT: [[TMP2:%.*]] = call { <vscale x 8 x i8>, <vscale x 8 x i8> } @llvm.vector.deinterleave2.nxv16i8(<vscale x 16 x i8> [[TMP1]])
-; BE-NEXT: [[R:%.*]] = extractvalue { <vscale x 8 x i8>, <vscale x 8 x i8> } [[TMP2]], 0
-; BE-NEXT: ret <vscale x 8 x i8> [[R]]
-;
- %d = call { <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16> } @llvm.vector.deinterleave4.nxv8i16(<vscale x 8 x i16> %x)
- %d0 = extractvalue { <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16> } %d, 0
- %d1 = extractvalue { <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16> } %d, 1
- %d2 = extractvalue { <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16> } %d, 2
- %d3 = extractvalue { <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16> } %d, 3
- %s0 = lshr <vscale x 2 x i16> %d0, splat (i16 8)
- %s1 = lshr <vscale x 2 x i16> %d1, splat (i16 8)
- %s2 = lshr <vscale x 2 x i16> %d2, splat (i16 8)
- %s3 = lshr <vscale x 2 x i16> %d3, splat (i16 8)
- %t0 = trunc <vscale x 2 x i16> %s0 to <vscale x 2 x i8>
- %t1 = trunc <vscale x 2 x i16> %s1 to <vscale x 2 x i8>
- %t2 = trunc <vscale x 2 x i16> %s2 to <vscale x 2 x i8>
- %t3 = trunc <vscale x 2 x i16> %s3 to <vscale x 2 x i8>
- %r = call <vscale x 8 x i8> @llvm.vector.interleave4.nxv8i8(<vscale x 2 x i8> %t0, <vscale x 2 x i8> %t1, <vscale x 2 x i8> %t2, <vscale x 2 x i8> %t3)
- ret <vscale x 8 x i8> %r
-}
-
-define <8 x i8> @fixed_lshr_trunc_interleave4(<8 x i16> %x) {
-; CHECK-LABEL: define <8 x i8> @fixed_lshr_trunc_interleave4(
-; CHECK-SAME: <8 x i16> [[X:%.*]]) {
-; CHECK-NEXT: [[TMP1:%.*]] = bitcast <8 x i16> [[X]] to <16 x i8>
-; CHECK-NEXT: [[TMP2:%.*]] = call { <8 x i8>, <8 x i8> } @llvm.vector.deinterleave2.v16i8(<16 x i8> [[TMP1]])
-; CHECK-NEXT: [[R:%.*]] = extractvalue { <8 x i8>, <8 x i8> } [[TMP2]], 1
-; CHECK-NEXT: ret <8 x i8> [[R]]
-;
-; BE-LABEL: define <8 x i8> @fixed_lshr_trunc_interleave4(
-; BE-SAME: <8 x i16> [[X:%.*]]) {
-; BE-NEXT: [[TMP1:%.*]] = bitcast <8 x i16> [[X]] to <16 x i8>
-; BE-NEXT: [[TMP2:%.*]] = call { <8 x i8>, <8 x i8> } @llvm.vector.deinterleave2.v16i8(<16 x i8> [[TMP1]])
-; BE-NEXT: [[R:%.*]] = extractvalue { <8 x i8>, <8 x i8> } [[TMP2]], 0
-; BE-NEXT: ret <8 x i8> [[R]]
-;
- %d = call { <2 x i16>, <2 x i16>, <2 x i16>, <2 x i16> } @llvm.vector.deinterleave4.v8i16(<8 x i16> %x)
- %d0 = extractvalue { <2 x i16>, <2 x i16>, <2 x i16>, <2 x i16> } %d, 0
- %d1 = extractvalue { <2 x i16>, <2 x i16>, <2 x i16>, <2 x i16> } %d, 1
- %d2 = extractvalue { <2 x i16>, <2 x i16>, <2 x i16>, <2 x i16> } %d, 2
- %d3 = extractvalue { <2 x i16>, <2 x i16>, <2 x i16>, <2 x i16> } %d, 3
- %s0 = lshr <2 x i16> %d0, splat (i16 8)
- %s1 = lshr <2 x i16> %d1, splat (i16 8)
- %s2 = lshr <2 x i16> %d2, splat (i16 8)
- %s3 = lshr <2 x i16> %d3, splat (i16 8)
- %t0 = trunc <2 x i16> %s0 to <2 x i8>
- %t1 = trunc <2 x i16> %s1 to <2 x i8>
- %t2 = trunc <2 x i16> %s2 to <2 x i8>
- %t3 = trunc <2 x i16> %s3 to <2 x i8>
- %r = call <8 x i8> @llvm.vector.interleave4.v8i8(<2 x i8> %t0, <2 x i8> %t1, <2 x i8> %t2, <2 x i8> %t3)
- ret <8 x i8> %r
-}
-
-define <vscale x 4 x i16> @mixed_shifts_trunc_interleave2(<vscale x 4 x i32> %x) {
-; CHECK-LABEL: define <vscale x 4 x i16> @mixed_shifts_trunc_interleave2(
-; CHECK-SAME: <vscale x 4 x i32> [[X:%.*]]) {
-; CHECK-NEXT: [[TMP1:%.*]] = bitcast <vscale x 4 x i32> [[X]] to <vscale x 8 x i16>
-; CHECK-NEXT: [[TMP2:%.*]] = call { <vscale x 4 x i16>, <vscale x 4 x i16> } @llvm.vector.deinterleave2.nxv8i16(<vscale x 8 x i16> [[TMP1]])
-; CHECK-NEXT: [[R:%.*]] = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16> } [[TMP2]], 1
-; CHECK-NEXT: ret <vscale x 4 x i16> [[R]]
-;
-; BE-LABEL: define <vscale x 4 x i16> @mixed_shifts_trunc_interleave2(
-; BE-SAME: <vscale x 4 x i32> [[X:%.*]]) {
-; BE-NEXT: [[TMP1:%.*]] = bitcast <vscale x 4 x i32> [[X]] to <vscale x 8 x i16>
-; BE-NEXT: [[TMP2:%.*]] = call { <vscale x 4 x i16>, <vscale x 4 x i16> } @llvm.vector.deinterleave2.nxv8i16(<vscale x 8 x i16> [[TMP1]])
-; BE-NEXT: [[R:%.*]] = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16> } [[TMP2]], 0
-; BE-NEXT: ret <vscale x 4 x i16> [[R]]
-;
- %d = call { <vscale x 2 x i32>, <vscale x 2 x i32> } @llvm.vector.deinterleave2.nxv4i32(<vscale x 4 x i32> %x)
- %d0 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } %d, 0
- %d1 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } %d, 1
- %s0 = lshr exact <vscale x 2 x i32> %d0, splat (i32 16)
- %s1 = ashr exact <vscale x 2 x i32> %d1, splat (i32 16)
- %t0 = trunc nuw <vscale x 2 x i32> %s0 to <vscale x 2 x i16>
- %t1 = trunc nuw <vscale x 2 x i32> %s1 to <vscale x 2 x i16>
- %r = call <vscale x 4 x i16> @llvm.vector.interleave2.nxv4i16(<vscale x 2 x i16> %t0, <vscale x 2 x i16> %t1)
- ret <vscale x 4 x i16> %r
-}
-
-; Negative test - the fold should only be applied if the shifts are equal.
-define <vscale x 4 x i16> @different_shift_amounts(<vscale x 4 x i32> %x) {
-; CHECK-LABEL: define <vscale x 4 x i16> @different_shift_amounts(
-; CHECK-SAME: <vscale x 4 x i32> [[X:%.*]]) {
-; CHECK-NEXT: [[D:%.*]] = call { <vscale x 2 x i32>, <vscale x 2 x i32> } @llvm.vector.deinterleave2.nxv4i32(<vscale x 4 x i32> [[X]])
-; CHECK-NEXT: [[D0:%.*]] = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } [[D]], 0
-; CHECK-NEXT: [[D1:%.*]] = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } [[D]], 1
-; CHECK-NEXT: [[S0:%.*]] = lshr <vscale x 2 x i32> [[D0]], splat (i32 16)
-; CHECK-NEXT: [[S1:%.*]] = lshr <vscale x 2 x i32> [[D1]], splat (i32 15)
-; CHECK-NEXT: [[T0:%.*]] = trunc <vscale x 2 x i32> [[S0]] to <vscale x 2 x i16>
-; CHECK-NEXT: [[T1:%.*]] = trunc <vscale x 2 x i32> [[S1]] to <vscale x 2 x i16>
-; CHECK-NEXT: [[R:%.*]] = call <vscale x 4 x i16> @llvm.vector.interleave2.nxv4i16(<vscale x 2 x i16> [[T0]], <vscale x 2 x i16> [[T1]])
-; CHECK-NEXT: ret <vscale x 4 x i16> [[R]]
-;
-; BE-LABEL: define <vscale x 4 x i16> @different_shift_amounts(
-; BE-SAME: <vscale x 4 x i32> [[X:%.*]]) {
-; BE-NEXT: [[D:%.*]] = call { <vscale x 2 x i32>, <vscale x 2 x i32> } @llvm.vector.deinterleave2.nxv4i32(<vscale x 4 x i32> [[X]])
-; BE-NEXT: [[D0:%.*]] = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } [[D]], 0
-; BE-NEXT: [[D1:%.*]] = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } [[D]], 1
-; BE-NEXT: [[S0:%.*]] = lshr <vscale x 2 x i32> [[D0]], splat (i32 16)
-; BE-NEXT: [[S1:%.*]] = lshr <vscale x 2 x i32> [[D1]], splat (i32 15)
-; BE-NEXT: [[T0:%.*]] = trunc <vscale x 2 x i32> [[S0]] to <vscale x 2 x i16>
-; BE-NEXT: [[T1:%.*]] = trunc <vscale x 2 x i32> [[S1]] to <vscale x 2 x i16>
-; BE-NEXT: [[R:%.*]] = call <vscale x 4 x i16> @llvm.vector.interleave2.nxv4i16(<vscale x 2 x i16> [[T0]], <vscale x 2 x i16> [[T1]])
-; BE-NEXT: ret <vscale x 4 x i16> [[R]]
-;
- %d = call { <vscale x 2 x i32>, <vscale x 2 x i32> } @llvm.vector.deinterleave2.nxv4i32(<vscale x 4 x i32> %x)
- %d0 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } %d, 0
- %d1 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } %d, 1
- %s0 = lshr <vscale x 2 x i32> %d0, splat (i32 16)
- %s1 = lshr <vscale x 2 x i32> %d1, splat (i32 15)
- %t0 = trunc <vscale x 2 x i32> %s0 to <vscale x 2 x i16>
- %t1 = trunc <vscale x 2 x i32> %s1 to <vscale x 2 x i16>
- %r = call <vscale x 4 x i16> @llvm.vector.interleave2.nxv4i16(<vscale x 2 x i16> %t0, <vscale x 2 x i16> %t1)
- ret <vscale x 4 x i16> %r
-}
-
-; Negative test - the fold shouldn't be generated as the deitnerleaved fields are passed in the wrong order.
-define <vscale x 4 x i16> @reordered_fields(<vscale x 4 x i32> %x) {
-; CHECK-LABEL: define <vscale x 4 x i16> @reordered_fields(
-; CHECK-SAME: <vscale x 4 x i32> [[X:%.*]]) {
-; CHECK-NEXT: [[D:%.*]] = call { <vscale x 2 x i32>, <vscale x 2 x i32> } @llvm.vector.deinterleave2.nxv4i32(<vscale x 4 x i32> [[X]])
-; CHECK-NEXT: [[D0:%.*]] = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } [[D]], 0
-; CHECK-NEXT: [[D1:%.*]] = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } [[D]], 1
-; CHECK-NEXT: [[S0:%.*]] = lshr <vscale x 2 x i32> [[D0]], splat (i32 16)
-; CHECK-NEXT: [[S1:%.*]] = lshr <vscale x 2 x i32> [[D1]], splat (i32 16)
-; CHECK-NEXT: [[T0:%.*]] = trunc <vscale x 2 x i32> [[S0]] to <vscale x 2 x i16>
-; CHECK-NEXT: [[T1:%.*]] = trunc <vscale x 2 x i32> [[S1]] to <vscale x 2 x i16>
-; CHECK-NEXT: [[R:%.*]] = call <vscale x 4 x i16> @llvm.vector.interleave2.nxv4i16(<vscale x 2 x i16> [[T1]], <vscale x 2 x i16> [[T0]])
-; CHECK-NEXT: ret <vscale x 4 x i16> [[R]]
-;
-; BE-LABEL: define <vscale x 4 x i16> @reordered_fields(
-; BE-SAME: <vscale x 4 x i32> [[X:%.*]]) {
-; BE-NEXT: [[D:%.*]] = call { <vscale x 2 x i32>, <vscale x 2 x i32> } @llvm.vector.deinterleave2.nxv4i32(<vscale x 4 x i32> [[X]])
-; BE-NEXT: [[D0:%.*]] = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } [[D]], 0
-; BE-NEXT: [[D1:%.*]] = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } [[D]], 1
-; BE-NEXT: [[S0:%.*]] = lshr <vscale x 2 x i32> [[D0]], splat (i32 16)
-; BE-NEXT: [[S1:%.*]] = lshr <vscale x 2 x i32> [[D1]], splat (i32 16)
-; BE-NEXT: [[T0:%.*]] = trunc <vscale x 2 x i32> [[S0]] to <vscale x 2 x i16>
-; BE-NEXT: [[T1:%.*]] = trunc <vscale x 2 x i32> [[S1]] to <vscale x 2 x i16>
-; BE-NEXT: [[R:%.*]] = call <vscale x 4 x i16> @llvm.vector.interleave2.nxv4i16(<vscale x 2 x i16> [[T1]], <vscale x 2 x i16> [[T0]])
-; BE-NEXT: ret <vscale x 4 x i16> [[R]]
-;
- %d = call { <vscale x 2 x i32>, <vscale x 2 x i32> } @llvm.vector.deinterleave2.nxv4i32(<vscale x 4 x i32> %x)
- %d0 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } %d, 0
- %d1 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } %d, 1
- %s0 = lshr <vscale x 2 x i32> %d0, splat (i32 16)
- %s1 = lshr <vscale x 2 x i32> %d1, splat (i32 16)
- %t0 = trunc <vscale x 2 x i32> %s0 to <vscale x 2 x i16>
- %t1 = trunc <vscale x 2 x i32> %s1 to <vscale x 2 x i16>
- %r = call <vscale x 4 x i16> @llvm.vector.interleave2.nxv4i16(<vscale x 2 x i16> %t1, <vscale x 2 x i16> %t0)
- ret <vscale x 4 x i16> %r
-}
-
-; Negative test - the fold shouldn't be generated as the trunc has more than one use.
-define <vscale x 4 x i16> @extra_trunc_use(
-; CHECK-LABEL: define <vscale x 4 x i16> @extra_trunc_use(
-; CHECK-SAME: <vscale x 4 x i32> [[X:%.*]], ptr [[DST:%.*]]) {
-; CHECK-NEXT: [[D:%.*]] = call { <vscale x 2 x i32>, <vscale x 2 x i32> } @llvm.vector.deinterleave2.nxv4i32(<vscale x 4 x i32> [[X]])
-; CHECK-NEXT: [[D0:%.*]] = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } [[D]], 0
-; CHECK-NEXT: [[D1:%.*]] = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } [[D]], 1
-; CHECK-NEXT: [[S0:%.*]] = lshr <vscale x 2 x i32> [[D0]], splat (i32 16)
-; CHECK-NEXT: [[S1:%.*]] = lshr <vscale x 2 x i32> [[D1]], splat (i32 16)
-; CHECK-NEXT: [[T0:%.*]] = trunc <vscale x 2 x i32> [[S0]] to <vscale x 2 x i16>
-; CHECK-NEXT: [[T1:%.*]] = trunc <vscale x 2 x i32> [[S1]] to <vscale x 2 x i16>
-; CHECK-NEXT: store <vscale x 2 x i16> [[T0]], ptr [[DST]], align 4
-; CHECK-NEXT: [[R:%.*]] = call <vscale x 4 x i16> @llvm.vector.interleave2.nxv4i16(<vscale x 2 x i16> [[T0]], <vscale x 2 x i16> [[T1]])
-; CHECK-NEXT: ret <vscale x 4 x i16> [[R]]
-;
-; BE-LABEL: define <vscale x 4 x i16> @extra_trunc_use(
-; BE-SAME: <vscale x 4 x i32> [[X:%.*]], ptr [[DST:%.*]]) {
-; BE-NEXT: [[D:%.*]] = call { <vscale x 2 x i32>, <vscale x 2 x i32> } @llvm.vector.deinterleave2.nxv4i32(<vscale x 4 x i32> [[X]])
-; BE-NEXT: [[D0:%.*]] = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } [[D]], 0
-; BE-NEXT: [[D1:%.*]] = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } [[D]], 1
-; BE-NEXT: [[S0:%.*]] = lshr <vscale x 2 x i32> [[D0]], splat (i32 16)
-; BE-NEXT: [[S1:%.*]] = lshr <vscale x 2 x i32> [[D1]], splat (i32 16)
-; BE-NEXT: [[T0:%.*]] = trunc <vscale x 2 x i32> [[S0]] to <vscale x 2 x i16>
-; BE-NEXT: [[T1:%.*]] = trunc <vscale x 2 x i32> [[S1]] to <vscale x 2 x i16>
-; BE-NEXT: store <vscale x 2 x i16> [[T0]], ptr [[DST]], align 4
-; BE-NEXT: [[R:%.*]] = call <vscale x 4 x i16> @llvm.vector.interleave2.nxv4i16(<vscale x 2 x i16> [[T0]], <vscale x 2 x i16> [[T1]])
-; BE-NEXT: ret <vscale x 4 x i16> [[R]]
-;
- <vscale x 4 x i32> %x, ptr %dst) {
- %d = call { <vscale x 2 x i32>, <vscale x 2 x i32> } @llvm.vector.deinterleave2.nxv4i32(<vscale x 4 x i32> %x)
- %d0 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } %d, 0
- %d1 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } %d, 1
- %s0 = lshr <vscale x 2 x i32> %d0, splat (i32 16)
- %s1 = lshr <vscale x 2 x i32> %d1, splat (i32 16)
- %t0 = trunc <vscale x 2 x i32> %s0 to <vscale x 2 x i16>
- %t1 = trunc <vscale x 2 x i32> %s1 to <vscale x 2 x i16>
- store <vscale x 2 x i16> %t0, ptr %dst
- %r = call <vscale x 4 x i16> @llvm.vector.interleave2.nxv4i16(<vscale x 2 x i16> %t0, <vscale x 2 x i16> %t1)
- ret <vscale x 4 x i16> %r
-}
-
>From 12cdae57230d52aa450b3c3babfa7b7e491f32c3 Mon Sep 17 00:00:00 2001
From: Jacob Crawley <jacob.crawley at arm.com>
Date: Thu, 30 Jul 2026 13:42:06 +0000
Subject: [PATCH 3/4] Rm unused variable
---
llvm/lib/Transforms/Vectorize/VectorCombine.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
index d949ae1d6600b..4f00d48add34a 100644
--- a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
+++ b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
@@ -6054,7 +6054,7 @@ bool VectorCombine::foldDeinterleaveInterleavePair(Instruction &I) {
ElementCount WideEC =
cast<VectorType>(Deinterleave->getArgOperand(0)->getType())
->getElementCount();
- bool DropTruncFlags = false;
+
for (const ElementwiseStep &Step : Steps) {
Instruction *NarrowInst = Step.Insts.front();
>From 4bddb1d734017e55e017d45cfc06b7299d4cd0f6 Mon Sep 17 00:00:00 2001
From: Jacob Crawley <jacob.crawley at arm.com>
Date: Thu, 30 Jul 2026 14:26:21 +0000
Subject: [PATCH 4/4] move tests
---
.../AArch64/sve-vectorcombine-interleave.ll | 70 -------------------
.../deinterleave-interleave-pairs.ll | 0
2 files changed, 70 deletions(-)
delete mode 100644 llvm/test/Transforms/VectorCombine/AArch64/sve-vectorcombine-interleave.ll
rename llvm/test/Transforms/VectorCombine/{AArch64 => }/deinterleave-interleave-pairs.ll (100%)
diff --git a/llvm/test/Transforms/VectorCombine/AArch64/sve-vectorcombine-interleave.ll b/llvm/test/Transforms/VectorCombine/AArch64/sve-vectorcombine-interleave.ll
deleted file mode 100644
index 11537a5291cbc..0000000000000
--- a/llvm/test/Transforms/VectorCombine/AArch64/sve-vectorcombine-interleave.ll
+++ /dev/null
@@ -1,70 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
-; RUN: opt -passes=vector-combine -mattr=+sve %s -o - | llc -mattr=+sve -o - | FileCheck %s
-
-target triple = "aarch64"
-
-define void @high_half_trunc_interleave4(ptr %src, ptr %dst) {
-; CHECK-LABEL: high_half_trunc_interleave4:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ldr z0, [x0]
-; CHECK-NEXT: ldr z1, [x0, #1, mul vl]
-; CHECK-NEXT: lsr z1.h, z1.h, #8
-; CHECK-NEXT: lsr z0.h, z0.h, #8
-; CHECK-NEXT: uzp1 z0.b, z0.b, z1.b
-; CHECK-NEXT: str z0, [x1]
-; CHECK-NEXT: ret
- %x = load <vscale x 16 x i16>, ptr %src, align 2
- %d = call { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } @llvm.vector.deinterleave4.nxv16i16(<vscale x 16 x i16> %x)
- %d0 = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } %d, 0
- %d1 = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } %d, 1
- %d2 = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } %d, 2
- %d3 = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } %d, 3
- %s0 = lshr <vscale x 4 x i16> %d0, splat (i16 8)
- %s1 = lshr <vscale x 4 x i16> %d1, splat (i16 8)
- %s2 = lshr <vscale x 4 x i16> %d2, splat (i16 8)
- %s3 = lshr <vscale x 4 x i16> %d3, splat (i16 8)
- %t0 = trunc <vscale x 4 x i16> %s0 to <vscale x 4 x i8>
- %t1 = trunc <vscale x 4 x i16> %s1 to <vscale x 4 x i8>
- %t2 = trunc <vscale x 4 x i16> %s2 to <vscale x 4 x i8>
- %t3 = trunc <vscale x 4 x i16> %s3 to <vscale x 4 x i8>
- %r = call <vscale x 16 x i8> @llvm.vector.interleave4.nxv16i8(<vscale x 4 x i8> %t0, <vscale x 4 x i8> %t1, <vscale x 4 x i8> %t2, <vscale x 4 x i8> %t3)
- store <vscale x 16 x i8> %r, ptr %dst, align 1
- ret void
-}
-
-; Keep the load and bitcast in separate blocks so VectorCombine
-; exposes a byte deinterleave2, which can be selected as a ld2b.
-define void @high_half_trunc_interleave4_byte_load(ptr %src, ptr %dst) {
-; CHECK-LABEL: high_half_trunc_interleave4_byte_load:
-; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: ldr z0, [x0]
-; CHECK-NEXT: ldr z1, [x0, #1, mul vl]
-; CHECK-NEXT: lsr z1.h, z1.h, #8
-; CHECK-NEXT: lsr z0.h, z0.h, #8
-; CHECK-NEXT: uzp1 z0.b, z0.b, z1.b
-; CHECK-NEXT: str z0, [x1]
-; CHECK-NEXT: ret
-entry:
- %bytes = load <vscale x 32 x i8>, ptr %src, align 2
- br label %body
-
-body:
- %x = bitcast <vscale x 32 x i8> %bytes to <vscale x 16 x i16>
- %d = call { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } @llvm.vector.deinterleave4.nxv16i16(<vscale x 16 x i16> %x)
- %d0 = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } %d, 0
- %d1 = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } %d, 1
- %d2 = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } %d, 2
- %d3 = extractvalue { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } %d, 3
- %s0 = lshr <vscale x 4 x i16> %d0, splat (i16 8)
- %s1 = lshr <vscale x 4 x i16> %d1, splat (i16 8)
- %s2 = lshr <vscale x 4 x i16> %d2, splat (i16 8)
- %s3 = lshr <vscale x 4 x i16> %d3, splat (i16 8)
- %t0 = trunc <vscale x 4 x i16> %s0 to <vscale x 4 x i8>
- %t1 = trunc <vscale x 4 x i16> %s1 to <vscale x 4 x i8>
- %t2 = trunc <vscale x 4 x i16> %s2 to <vscale x 4 x i8>
- %t3 = trunc <vscale x 4 x i16> %s3 to <vscale x 4 x i8>
- %r = call <vscale x 16 x i8> @llvm.vector.interleave4.nxv16i8(<vscale x 4 x i8> %t0, <vscale x 4 x i8> %t1, <vscale x 4 x i8> %t2, <vscale x 4 x i8> %t3)
- store <vscale x 16 x i8> %r, ptr %dst, align 1
- ret void
-}
-
diff --git a/llvm/test/Transforms/VectorCombine/AArch64/deinterleave-interleave-pairs.ll b/llvm/test/Transforms/VectorCombine/deinterleave-interleave-pairs.ll
similarity index 100%
rename from llvm/test/Transforms/VectorCombine/AArch64/deinterleave-interleave-pairs.ll
rename to llvm/test/Transforms/VectorCombine/deinterleave-interleave-pairs.ll
More information about the llvm-commits
mailing list