[llvm] [AArch64] Add widening for vector interleave (PR #214313)
Kamlesh Kumar via llvm-commits
llvm-commits at lists.llvm.org
Wed Aug 5 11:55:00 PDT 2026
https://github.com/kamleshbhalui created https://github.com/llvm/llvm-project/pull/214313
Splitted widening change from here https://github.com/llvm/llvm-project/pull/210494
>From 411d0312feac3795578bb28d2619b8ed7290da7a Mon Sep 17 00:00:00 2001
From: Kamlesh Kumar <kamlesh.kumar at arm.com>
Date: Wed, 5 Aug 2026 18:42:29 +0000
Subject: [PATCH] [AArch64] Add widening for vector interleave
---
llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h | 3 +-
.../SelectionDAG/LegalizeVectorTypes.cpp | 38 +++
.../fixed-vector-interleave-widen-no-neon.ll | 216 ++++++++++++++++++
3 files changed, 256 insertions(+), 1 deletion(-)
create mode 100644 llvm/test/CodeGen/AArch64/fixed-vector-interleave-widen-no-neon.ll
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
index 7e458d6d82860..41ef7ac785379 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
@@ -1080,7 +1080,8 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
SDValue WidenVecRes_VECTOR_REVERSE(SDNode *N);
SDValue WidenVecRes_GET_ACTIVE_LANE_MASK(SDNode *N);
void WidenVecRes_VECTOR_DEINTERLEAVE(SDNode *N);
-
+ void WidenVecRes_VECTOR_INTERLEAVE(SDNode *N);
+
SDValue WidenVecRes_Ternary(SDNode *N);
SDValue WidenVecRes_Binary(SDNode *N);
SDValue WidenVecRes_MaskedBinary(SDNode *N);
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
index 3ed0d22efb80e..f511c51f044da 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
@@ -5328,6 +5328,9 @@ void DAGTypeLegalizer::WidenVectorResult(SDNode *N, unsigned ResNo) {
case ISD::GET_ACTIVE_LANE_MASK:
Res = WidenVecRes_GET_ACTIVE_LANE_MASK(N);
break;
+ case ISD::VECTOR_INTERLEAVE:
+ WidenVecRes_VECTOR_INTERLEAVE(N);
+ break;
case ISD::VECTOR_DEINTERLEAVE:
WidenVecRes_VECTOR_DEINTERLEAVE(N);
break;
@@ -7485,6 +7488,41 @@ SDValue DAGTypeLegalizer::WidenVecRes_GET_ACTIVE_LANE_MASK(SDNode *N) {
return DAG.getNode(ISD::GET_ACTIVE_LANE_MASK, SDLoc(N), NVT, N->ops());
}
+void DAGTypeLegalizer::WidenVecRes_VECTOR_INTERLEAVE(SDNode *N) {
+ EVT VT = N->getValueType(0);
+ EVT EltVT = VT.getVectorElementType();
+ ElementCount OrigEC = VT.getVectorElementCount();
+ unsigned Factor = N->getNumOperands();
+ SDLoc DL(N);
+
+ EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
+ ElementCount WidenEC = WidenVT.getVectorElementCount();
+
+ SmallVector<SDValue, 8> WidenOps(Factor);
+ for (unsigned Idx = 0U; Idx < Factor; ++Idx)
+ WidenOps[Idx] = GetWidenedVector(N->getOperand(Idx));
+
+ SmallVector<EVT, 8> WidenVTs(Factor, WidenVT);
+ SDValue Interleaved =
+ DAG.getNode(ISD::VECTOR_INTERLEAVE, DL, WidenVTs, WidenOps);
+
+ EVT PackedWidenVT = EVT::getVectorVT(*DAG.getContext(), EltVT,
+ WidenEC.multiplyCoefficientBy(Factor));
+ SmallVector<SDValue, 8> Slices(Factor);
+ for (unsigned Idx = 0; Idx != Factor; ++Idx)
+ Slices[Idx] = Interleaved.getValue(Idx);
+
+ SDValue Packed = DAG.getNode(ISD::CONCAT_VECTORS, DL, PackedWidenVT, Slices);
+
+ for (unsigned Idx = 0U; Idx < Factor; ++Idx) {
+ SDValue Narrow = DAG.getExtractSubvector(
+ DL, VT, Packed, OrigEC.multiplyCoefficientBy(Idx).getKnownMinValue());
+ SDValue Wide =
+ DAG.getInsertSubvector(DL, DAG.getUNDEF(WidenVT), Narrow, /*Idx=*/0U);
+ SetWidenedVector(SDValue(N, Idx), Wide);
+ }
+}
+
void DAGTypeLegalizer::WidenVecRes_VECTOR_DEINTERLEAVE(SDNode *N) {
EVT VT = N->getValueType(0);
EVT EltVT = VT.getVectorElementType();
diff --git a/llvm/test/CodeGen/AArch64/fixed-vector-interleave-widen-no-neon.ll b/llvm/test/CodeGen/AArch64/fixed-vector-interleave-widen-no-neon.ll
new file mode 100644
index 0000000000000..9584a51cf78df
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/fixed-vector-interleave-widen-no-neon.ll
@@ -0,0 +1,216 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+;RUN:llc -mtriple=aarch64-none-linux-gnu -mattr=-neon %s -o - | FileCheck %s
+
+define void @interleave3_v3f32(ptr %dst, ptr %a, ptr %b, ptr %c) {
+; CHECK-LABEL: interleave3_v3f32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ldr w9, [x2, #8]
+; CHECK-NEXT: ldr w10, [x1, #8]
+; CHECK-NEXT: ldr x8, [x1]
+; CHECK-NEXT: ldr x11, [x2]
+; CHECK-NEXT: ldr s0, [x3, #8]
+; CHECK-NEXT: ldr x13, [x3]
+; CHECK-NEXT: orr x9, x10, x9, lsl #32
+; CHECK-NEXT: mov x12, x8
+; CHECK-NEXT: bfi x12, x11, #32, #32
+; CHECK-NEXT: bfxil x11, x8, #32, #32
+; CHECK-NEXT: str x9, [sp, #-48]!
+; CHECK-NEXT: .cfi_def_cfa_offset 48
+; CHECK-NEXT: lsr x9, x8, #32
+; CHECK-NEXT: str x11, [sp, #16]
+; CHECK-NEXT: lsr x11, x12, #32
+; CHECK-NEXT: str x12, [sp, #32]
+; CHECK-NEXT: and x12, x13, #0xffffffff00000000
+; CHECK-NEXT: bfi x13, x9, #32, #32
+; CHECK-NEXT: ldr w9, [sp, #4]
+; CHECK-NEXT: bfi x8, x11, #32, #32
+; CHECK-NEXT: ldr w11, [sp, #20]
+; CHECK-NEXT: str s0, [x0, #32]
+; CHECK-NEXT: orr x9, x10, x9, lsl #32
+; CHECK-NEXT: stp x8, x13, [x0]
+; CHECK-NEXT: orr x10, x11, x12
+; CHECK-NEXT: stp x10, x9, [x0, #16]
+; CHECK-NEXT: add sp, sp, #48
+; CHECK-NEXT: ret
+ %vec0 = load <3 x float>, ptr %a, align 4
+ %vec1 = load <3 x float>, ptr %b, align 4
+ %vec2 = load <3 x float>, ptr %c, align 4
+ %interleaved = call <9 x float> @llvm.vector.interleave3.v3f32(
+ <3 x float> %vec0, <3 x float> %vec1, <3 x float> %vec2)
+ store <9 x float> %interleaved, ptr %dst, align 4
+ ret void
+}
+
+define void @interleave3_v5f32(ptr %dst, ptr %a, ptr %b, ptr %c) {
+; CHECK-LABEL: interleave3_v5f32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ldp x14, x13, [x2]
+; CHECK-NEXT: ldr w12, [x2, #16]
+; CHECK-NEXT: ldp x11, x15, [x3]
+; CHECK-NEXT: ldr w10, [x1, #16]
+; CHECK-NEXT: ldr s0, [x3, #16]
+; CHECK-NEXT: ldp x8, x9, [x1]
+; CHECK-NEXT: mov x17, x13
+; CHECK-NEXT: orr x12, x10, x12, lsl #32
+; CHECK-NEXT: bfi x17, x15, #32, #32
+; CHECK-NEXT: bfxil x15, x13, #32, #32
+; CHECK-NEXT: mov x16, x8
+; CHECK-NEXT: bfi x16, x14, #32, #32
+; CHECK-NEXT: stp x15, x12, [sp, #-96]!
+; CHECK-NEXT: .cfi_def_cfa_offset 96
+; CHECK-NEXT: extr x12, x9, x11, #32
+; CHECK-NEXT: mov x15, x8
+; CHECK-NEXT: and x9, x9, #0xffffffff00000000
+; CHECK-NEXT: bfxil x15, x11, #0, #32
+; CHECK-NEXT: stp x12, x17, [sp, #32]
+; CHECK-NEXT: lsr x12, x11, #32
+; CHECK-NEXT: stp x16, x15, [sp, #64]
+; CHECK-NEXT: ldr w15, [sp, #36]
+; CHECK-NEXT: lsr x16, x17, #32
+; CHECK-NEXT: extr x12, x12, x14, #32
+; CHECK-NEXT: ldr w17, [sp, #4]
+; CHECK-NEXT: ldr w14, [sp, #12]
+; CHECK-NEXT: orr x15, x15, x13, lsl #32
+; CHECK-NEXT: ldr w18, [sp, #68]
+; CHECK-NEXT: ldr w1, [sp, #76]
+; CHECK-NEXT: orr x9, x16, x9
+; CHECK-NEXT: str s0, [x0, #56]
+; CHECK-NEXT: stp x12, x15, [x0, #16]
+; CHECK-NEXT: extr x12, x17, x13, #32
+; CHECK-NEXT: bfi x11, x1, #32, #32
+; CHECK-NEXT: bfi x8, x18, #32, #32
+; CHECK-NEXT: stp x9, x12, [x0, #32]
+; CHECK-NEXT: orr x9, x10, x14, lsl #32
+; CHECK-NEXT: stp x8, x11, [x0]
+; CHECK-NEXT: str x9, [x0, #48]
+; CHECK-NEXT: add sp, sp, #96
+; CHECK-NEXT: ret
+ %vec0 = load <5 x float>, ptr %a, align 4
+ %vec1 = load <5 x float>, ptr %b, align 4
+ %vec2 = load <5 x float>, ptr %c, align 4
+ %interleaved = call <15 x float> @llvm.vector.interleave3.v5f32(
+ <5 x float> %vec0, <5 x float> %vec1, <5 x float> %vec2)
+ store <15 x float> %interleaved, ptr %dst, align 4
+ ret void
+}
+
+define void @interleave4_v3f32(ptr %dst, ptr %a, ptr %b, ptr %c, ptr %d) {
+; CHECK-LABEL: interleave4_v3f32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ldr w8, [x3, #8]
+; CHECK-NEXT: ldr w10, [x2, #8]
+; CHECK-NEXT: ldr x9, [x1]
+; CHECK-NEXT: ldr x12, [x4]
+; CHECK-NEXT: ldr x11, [x2]
+; CHECK-NEXT: ldr x13, [x3]
+; CHECK-NEXT: orr x8, x10, x8, lsl #32
+; CHECK-NEXT: mov x14, x9
+; CHECK-NEXT: ldr w15, [x1, #8]
+; CHECK-NEXT: mov x16, x12
+; CHECK-NEXT: bfi x14, x11, #32, #32
+; CHECK-NEXT: bfxil x16, x13, #32, #32
+; CHECK-NEXT: str x8, [sp, #-64]!
+; CHECK-NEXT: .cfi_def_cfa_offset 64
+; CHECK-NEXT: mov x8, x9
+; CHECK-NEXT: str x14, [sp, #48]
+; CHECK-NEXT: lsr x14, x16, #32
+; CHECK-NEXT: bfxil x8, x12, #0, #32
+; CHECK-NEXT: str x16, [sp, #16]
+; CHECK-NEXT: and x11, x11, #0xffffffff00000000
+; CHECK-NEXT: extr x14, x14, x13, #32
+; CHECK-NEXT: bfi x13, x12, #32, #32
+; CHECK-NEXT: ldr w16, [sp, #4]
+; CHECK-NEXT: str x8, [sp, #32]
+; CHECK-NEXT: lsr x8, x8, #32
+; CHECK-NEXT: ldr w12, [x4, #8]
+; CHECK-NEXT: orr x8, x8, x11
+; CHECK-NEXT: ldr w11, [sp, #52]
+; CHECK-NEXT: stp x13, x8, [x0, #8]
+; CHECK-NEXT: orr x8, x15, x10, lsl #32
+; CHECK-NEXT: orr x10, x16, x12, lsl #32
+; CHECK-NEXT: bfi x9, x11, #32, #32
+; CHECK-NEXT: stp x14, x8, [x0, #24]
+; CHECK-NEXT: str x9, [x0]
+; CHECK-NEXT: str x10, [x0, #40]
+; CHECK-NEXT: add sp, sp, #64
+; CHECK-NEXT: ret
+ %vec0 = load <3 x float>, ptr %a, align 4
+ %vec1 = load <3 x float>, ptr %b, align 4
+ %vec2 = load <3 x float>, ptr %c, align 4
+ %vec3 = load <3 x float>, ptr %d, align 4
+ %interleaved = call <12 x float> @llvm.vector.interleave4.v3f32(
+ <3 x float> %vec0, <3 x float> %vec1, <3 x float> %vec2, <3 x float> %vec3)
+ store <12 x float> %interleaved, ptr %dst, align 4
+ ret void
+}
+
+define void @interleave4_v5f32(ptr %dst, ptr %a, ptr %b, ptr %c, ptr %d) {
+; CHECK-LABEL: interleave4_v5f32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: sub sp, sp, #128
+; CHECK-NEXT: .cfi_def_cfa_offset 128
+; CHECK-NEXT: ldr w9, [x3, #16]
+; CHECK-NEXT: ldr w8, [x2, #16]
+; CHECK-NEXT: ldp x17, x12, [x2]
+; CHECK-NEXT: orr x16, x8, x9, lsl #32
+; CHECK-NEXT: ldp x13, x9, [x3]
+; CHECK-NEXT: ldp x11, x10, [x1]
+; CHECK-NEXT: ldp x14, x15, [x4]
+; CHECK-NEXT: str x16, [sp, #8]
+; CHECK-NEXT: ldr w16, [x1, #16]
+; CHECK-NEXT: mov x1, x12
+; CHECK-NEXT: mov x2, x9
+; CHECK-NEXT: bfxil x1, x10, #32, #32
+; CHECK-NEXT: mov x18, x11
+; CHECK-NEXT: bfi x2, x15, #32, #32
+; CHECK-NEXT: bfi x18, x17, #32, #32
+; CHECK-NEXT: extr x16, x16, x15, #32
+; CHECK-NEXT: lsr x15, x15, #32
+; CHECK-NEXT: stp x2, x1, [sp, #32]
+; CHECK-NEXT: mov x1, x13
+; CHECK-NEXT: bfi x1, x14, #32, #32
+; CHECK-NEXT: str x18, [sp, #96]
+; CHECK-NEXT: mov x18, x13
+; CHECK-NEXT: str x16, [sp]
+; CHECK-NEXT: extr x16, x10, x14, #32
+; CHECK-NEXT: bfxil x18, x17, #32, #32
+; CHECK-NEXT: str x1, [sp, #104]
+; CHECK-NEXT: lsr x17, x17, #32
+; CHECK-NEXT: and x14, x14, #0xffffffff00000000
+; CHECK-NEXT: ldr w5, [sp, #108]
+; CHECK-NEXT: stp x18, x16, [sp, #64]
+; CHECK-NEXT: ldr w18, [sp, #12]
+; CHECK-NEXT: ldr w3, [sp, #68]
+; CHECK-NEXT: extr x16, x17, x11, #32
+; CHECK-NEXT: ldr w17, [x4, #16]
+; CHECK-NEXT: bfi x13, x5, #32, #32
+; CHECK-NEXT: ldr w5, [sp, #36]
+; CHECK-NEXT: ldr w1, [sp, #4]
+; CHECK-NEXT: ldr w2, [sp, #76]
+; CHECK-NEXT: ldr w4, [sp, #100]
+; CHECK-NEXT: orr x14, x3, x14
+; CHECK-NEXT: ldr w3, [sp, #44]
+; CHECK-NEXT: extr x15, x15, x9, #32
+; CHECK-NEXT: bfi x9, x5, #32, #32
+; CHECK-NEXT: bfi x11, x4, #32, #32
+; CHECK-NEXT: orr x12, x2, x12, lsl #32
+; CHECK-NEXT: orr x8, x1, x8, lsl #32
+; CHECK-NEXT: extr x10, x3, x10, #32
+; CHECK-NEXT: str x9, [x0, #40]
+; CHECK-NEXT: orr x9, x18, x17, lsl #32
+; CHECK-NEXT: stp x13, x16, [x0, #8]
+; CHECK-NEXT: stp x10, x15, [x0, #48]
+; CHECK-NEXT: stp x14, x12, [x0, #24]
+; CHECK-NEXT: str x11, [x0]
+; CHECK-NEXT: stp x8, x9, [x0, #64]
+; CHECK-NEXT: add sp, sp, #128
+; CHECK-NEXT: ret
+ %vec0 = load <5 x float>, ptr %a, align 4
+ %vec1 = load <5 x float>, ptr %b, align 4
+ %vec2 = load <5 x float>, ptr %c, align 4
+ %vec3 = load <5 x float>, ptr %d, align 4
+ %interleaved = call <20 x float> @llvm.vector.interleave4.v5f32(
+ <5 x float> %vec0, <5 x float> %vec1, <5 x float> %vec2, <5 x float> %vec3)
+ store <20 x float> %interleaved, ptr %dst, align 4
+ ret void
+}
More information about the llvm-commits
mailing list