[llvm] [CodeGen] Expand unsupported interleave to shuffles (PR #210494)
via llvm-commits
llvm-commits at lists.llvm.org
Sat Jul 18 00:13:02 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-x86
@llvm/pr-subscribers-backend-aarch64
Author: Kamlesh Kumar (kamleshbhalui)
<details>
<summary>Changes</summary>
---
Patch is 1.12 MiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/210494.diff
24 Files Affected:
- (modified) llvm/include/llvm/CodeGen/TargetLowering.h (+7)
- (modified) llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp (-51)
- (modified) llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp (+159)
- (modified) llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp (-22)
- (modified) llvm/lib/Target/AArch64/AArch64ISelLowering.h (+3)
- (modified) llvm/lib/Target/RISCV/RISCVISelLowering.h (+3)
- (modified) llvm/test/CodeGen/AArch64/complex-deinterleaving-reductions.ll (+28-22)
- (modified) llvm/test/CodeGen/AArch64/complex-deinterleaving-splat.ll (+25-20)
- (modified) llvm/test/CodeGen/AArch64/fixed-vector-deinterleave.ll (+70-59)
- (added) llvm/test/CodeGen/AArch64/fixed-vector-interleave-odd-factor.ll (+131)
- (modified) llvm/test/CodeGen/AArch64/fixed-vector-interleave.ll (+46-62)
- (modified) llvm/test/CodeGen/AArch64/fixed_masked_deinterleaved_loads.ll (+11-17)
- (modified) llvm/test/CodeGen/AArch64/fixed_masked_interleaved_stores.ll (+9-15)
- (modified) llvm/test/CodeGen/AArch64/sve-vector-deinterleave.ll (+16-16)
- (modified) llvm/test/CodeGen/AArch64/sve-vector-interleave.ll (+16-16)
- (modified) llvm/test/CodeGen/RISCV/rvv/fixed-vectors-deinterleave-load.ll (+22-13)
- (modified) llvm/test/CodeGen/RISCV/rvv/fixed-vectors-interleave-store.ll (+19-12)
- (modified) llvm/test/CodeGen/RISCV/rvv/vector-deinterleave-fixed.ll (+757-392)
- (modified) llvm/test/CodeGen/RISCV/rvv/vector-deinterleave-load.ll (+76-23)
- (modified) llvm/test/CodeGen/RISCV/rvv/vector-deinterleave.ll (+240-305)
- (modified) llvm/test/CodeGen/RISCV/rvv/vector-interleave-fixed.ll (+7253-1813)
- (modified) llvm/test/CodeGen/RISCV/rvv/vector-interleave.ll (+6184-5397)
- (modified) llvm/test/CodeGen/RISCV/rvv/vp-vector-interleaved-access.ll (+20-8)
- (added) llvm/test/CodeGen/X86/vector-interleave-intrinsics.ll (+576)
``````````diff
diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h
index 9a6dd7735421e..9f8319dc6c867 100644
--- a/llvm/include/llvm/CodeGen/TargetLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetLowering.h
@@ -3324,6 +3324,13 @@ class LLVM_ABI TargetLoweringBase {
/// Default to be the minimum interleave factor: 2.
virtual unsigned getMaxSupportedInterleaveFactor() const { return 2; }
+ /// Return true if the target supports interleave intrinsic for the specified
+ /// factor and vector type.
+ virtual bool isInterleaveIntrinsicSupported(unsigned /*Factor*/,
+ EVT /*VecTy*/) const {
+ return false;
+ }
+
/// Lower an interleaved load to target specific intrinsics. Return
/// true on success.
///
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
index 65b8212755e6e..f7ec28db02e4c 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
@@ -3783,57 +3783,6 @@ bool SelectionDAGLegalize::ExpandNode(SDNode *Node) {
Results.push_back(TLI.expandVectorSplice(Node, DAG));
break;
}
- case ISD::VECTOR_DEINTERLEAVE: {
- unsigned Factor = Node->getNumOperands();
- if (Factor <= 2 || Factor % 2 != 0)
- break;
- SmallVector<SDValue, 8> Ops(Node->ops());
- EVT VecVT = Node->getValueType(0);
- SmallVector<EVT> HalfVTs(Factor / 2, VecVT);
- // Deinterleave at Factor/2 so each result contains two factors interleaved:
- // a0b0 c0d0 a1b1 c1d1 -> [a0c0 b0d0] [a1c1 b1d1]
- SDValue L = DAG.getNode(ISD::VECTOR_DEINTERLEAVE, dl, HalfVTs,
- ArrayRef(Ops).take_front(Factor / 2));
- SDValue R = DAG.getNode(ISD::VECTOR_DEINTERLEAVE, dl, HalfVTs,
- ArrayRef(Ops).take_back(Factor / 2));
- Results.resize(Factor);
- // Deinterleave the 2 factors out:
- // [a0c0 a1c1] [b0d0 b1d1] -> a0a1 b0b1 c0c1 d0d1
- for (unsigned I = 0; I < Factor / 2; I++) {
- SDValue Deinterleave =
- DAG.getNode(ISD::VECTOR_DEINTERLEAVE, dl, {VecVT, VecVT},
- {L.getValue(I), R.getValue(I)});
- Results[I] = Deinterleave.getValue(0);
- Results[I + Factor / 2] = Deinterleave.getValue(1);
- }
- break;
- }
- case ISD::VECTOR_INTERLEAVE: {
- unsigned Factor = Node->getNumOperands();
- if (Factor <= 2 || Factor % 2 != 0)
- break;
- EVT VecVT = Node->getValueType(0);
- SmallVector<EVT> HalfVTs(Factor / 2, VecVT);
- SmallVector<SDValue, 8> LOps, ROps;
- // Interleave so we have 2 factors per result:
- // a0a1 b0b1 c0c1 d0d1 -> [a0c0 b0d0] [a1c1 b1d1]
- for (unsigned I = 0; I < Factor / 2; I++) {
- SDValue Interleave =
- DAG.getNode(ISD::VECTOR_INTERLEAVE, dl, {VecVT, VecVT},
- {Node->getOperand(I), Node->getOperand(I + Factor / 2)});
- LOps.push_back(Interleave.getValue(0));
- ROps.push_back(Interleave.getValue(1));
- }
- // Interleave at Factor/2:
- // [a0c0 b0d0] [a1c1 b1d1] -> a0b0 c0d0 a1b1 c1d1
- SDValue L = DAG.getNode(ISD::VECTOR_INTERLEAVE, dl, HalfVTs, LOps);
- SDValue R = DAG.getNode(ISD::VECTOR_INTERLEAVE, dl, HalfVTs, ROps);
- for (unsigned I = 0; I < Factor / 2; I++)
- Results.push_back(L.getValue(I));
- for (unsigned I = 0; I < Factor / 2; I++)
- Results.push_back(R.getValue(I));
- break;
- }
case ISD::EXTRACT_ELEMENT: {
EVT OpTy = Node->getOperand(0).getValueType();
if (Node->getConstantOperandVal(1)) {
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
index f4cb60736ef7a..27df2bdd8d7cc 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
@@ -95,6 +95,20 @@ class VectorLegalizer {
/// operations to legalize them.
void Expand(SDNode *Node, SmallVectorImpl<SDValue> &Results);
+ /// Expand a VECTOR_DEINTERLEAVE.
+ void ExpandVectorDeinterleave(SDNode *Node,
+ SmallVectorImpl<SDValue> &Results);
+ /// Expand a VECTOR_INTERLEAVE.
+ void ExpandVectorInterleave(SDNode *Node, SmallVectorImpl<SDValue> &Results);
+
+ /// Expand a fixed-length VECTOR_INTERLEAVE.
+ void ExpandVectorInterleaveToShuffles(SDNode *Node,
+ SmallVectorImpl<SDValue> &Results);
+
+ /// Expand a fixed-length VECTOR_DEINTERLEAVE.
+ void ExpandVectorDeinterleaveToShuffles(SDNode *Node,
+ SmallVectorImpl<SDValue> &Results);
+
/// Implements expansion for FP_TO_UINT; falls back to UnrollVectorOp if
/// FP_TO_SINT isn't legal.
void ExpandFP_TO_UINT(SDNode *Node, SmallVectorImpl<SDValue> &Results);
@@ -491,6 +505,15 @@ SDValue VectorLegalizer::LegalizeOp(SDValue Op) {
case ISD::MASKED_SREM:
Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
break;
+ case ISD::VECTOR_INTERLEAVE:
+ case ISD::VECTOR_DEINTERLEAVE: {
+ EVT VT = Node->getValueType(0);
+ unsigned Factor = Node->getNumOperands();
+ Action = TLI.getOperationAction(Node->getOpcode(), VT);
+ if (!TLI.isInterleaveIntrinsicSupported(Factor, VT))
+ Action = TargetLowering::Expand;
+ break;
+ }
case ISD::SMULFIX:
case ISD::SMULFIXSAT:
case ISD::UMULFIX:
@@ -966,6 +989,136 @@ SDValue VectorLegalizer::ExpandStore(SDNode *N) {
SDValue TF = TLI.scalarizeVectorStore(ST, DAG);
return TF;
}
+void VectorLegalizer::ExpandVectorDeinterleave(
+ SDNode *Node, SmallVectorImpl<SDValue> &Results) {
+ assert(Node->getOpcode() == ISD::VECTOR_DEINTERLEAVE && "Unexpected opcode");
+
+ unsigned Factor = Node->getNumOperands();
+ assert(Factor == Node->getNumValues() &&
+ "Interleave operands and results must match");
+
+ EVT VecVT = Node->getValueType(0);
+ if (!TLI.isInterleaveIntrinsicSupported(Factor, VecVT)) {
+ ExpandVectorDeinterleaveToShuffles(Node, Results);
+ return;
+ }
+
+ // Break higher even factors into lower factors. The generated nodes are
+ // recursively legalized, allowing targets to custom lower any factor they
+ // support.
+ if (Factor > 2 && Factor % 2 == 0) {
+ SmallVector<EVT> HalfVTs(Factor / 2, VecVT);
+ SDLoc DL(Node);
+
+ SmallVector<SDValue, 8> Ops(Node->ops());
+ // Deinterleave at Factor/2 so each result contains two factors
+ // interleaved:
+ // a0b0 c0d0 a1b1 c1d1 -> [a0c0 b0d0] [a1c1 b1d1]
+ SDValue L = DAG.getNode(ISD::VECTOR_DEINTERLEAVE, DL, HalfVTs,
+ ArrayRef(Ops).take_front(Factor / 2));
+ SDValue R = DAG.getNode(ISD::VECTOR_DEINTERLEAVE, DL, HalfVTs,
+ ArrayRef(Ops).take_back(Factor / 2));
+ Results.resize(Factor);
+ // Deinterleave the 2 factors out:
+ // [a0c0 a1c1] [b0d0 b1d1] -> a0a1 b0b1 c0c1 d0d1
+ for (unsigned I = 0; I < Factor / 2; I++) {
+ SDValue Deinterleave =
+ DAG.getNode(ISD::VECTOR_DEINTERLEAVE, DL, {VecVT, VecVT},
+ {L.getValue(I), R.getValue(I)});
+ Results[I] = Deinterleave.getValue(0);
+ Results[I + Factor / 2] = Deinterleave.getValue(1);
+ }
+ return;
+ }
+}
+
+void VectorLegalizer::ExpandVectorInterleave(
+ SDNode *Node, SmallVectorImpl<SDValue> &Results) {
+ assert(Node->getOpcode() == ISD::VECTOR_INTERLEAVE && "Unexpected opcode");
+ unsigned Factor = Node->getNumOperands();
+ EVT VecVT = Node->getValueType(0);
+
+ if (!TLI.isInterleaveIntrinsicSupported(Factor, VecVT)) {
+ ExpandVectorInterleaveToShuffles(Node, Results);
+ return;
+ }
+
+ // Break higher even factors into lower factors. The generated nodes are
+ // recursively legalized, allowing targets to custom lower any factor they
+ // support.
+ if (Factor > 2 && Factor % 2 == 0) {
+ SmallVector<EVT> HalfVTs(Factor / 2, VecVT);
+ SDLoc DL(Node);
+ SmallVector<SDValue, 8> LOps, ROps;
+ // Interleave so we have 2 factors per result:
+ // a0a1 b0b1 c0c1 d0d1 -> [a0c0 b0d0] [a1c1 b1d1]
+ for (unsigned I = 0; I < Factor / 2; I++) {
+ SDValue Interleave =
+ DAG.getNode(ISD::VECTOR_INTERLEAVE, DL, {VecVT, VecVT},
+ {Node->getOperand(I), Node->getOperand(I + Factor / 2)});
+ LOps.push_back(Interleave.getValue(0));
+ ROps.push_back(Interleave.getValue(1));
+ }
+ // Interleave at Factor/2:
+ // [a0c0 b0d0] [a1c1 b1d1] -> a0b0 c0d0 a1b1 c1d1
+ SDValue L = DAG.getNode(ISD::VECTOR_INTERLEAVE, DL, HalfVTs, LOps);
+ SDValue R = DAG.getNode(ISD::VECTOR_INTERLEAVE, DL, HalfVTs, ROps);
+ for (unsigned I = 0; I < Factor / 2; I++)
+ Results.push_back(L.getValue(I));
+ for (unsigned I = 0; I < Factor / 2; I++)
+ Results.push_back(R.getValue(I));
+ return;
+ }
+}
+
+void VectorLegalizer::ExpandVectorInterleaveToShuffles(
+ SDNode *Node, SmallVectorImpl<SDValue> &Results) {
+ assert(Node->getOpcode() == ISD::VECTOR_INTERLEAVE && "Unexpected opcode");
+ EVT VT = Node->getValueType(0);
+ assert(VT.isFixedLengthVector() && "Expected a fixed-length vector");
+ unsigned Factor = Node->getNumOperands();
+ unsigned NumElts = VT.getVectorNumElements();
+ EVT WideVT = EVT::getVectorVT(*DAG.getContext(), VT.getVectorElementType(),
+ NumElts * Factor);
+ SDLoc DL(Node);
+
+ SmallVector<SDValue, 8> InVecs(Node->op_values());
+ SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, DL, WideVT, InVecs);
+ SDValue Interleaved =
+ DAG.getVectorShuffle(WideVT, DL, Concat, DAG.getUNDEF(WideVT),
+ createInterleaveMask(NumElts, Factor));
+
+ for (unsigned I = 0; I != Factor; ++I)
+ Results.push_back(DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Interleaved,
+ DAG.getVectorIdxConstant(I * NumElts, DL)));
+}
+
+void VectorLegalizer::ExpandVectorDeinterleaveToShuffles(
+ SDNode *Node, SmallVectorImpl<SDValue> &Results) {
+ assert(Node->getOpcode() == ISD::VECTOR_DEINTERLEAVE && "Unexpected opcode");
+ EVT VT = Node->getValueType(0);
+ assert(VT.isFixedLengthVector() && "Expected a fixed-length vector");
+ unsigned Factor = Node->getNumOperands();
+ unsigned NumElts = VT.getVectorNumElements();
+ EVT WideVT = EVT::getVectorVT(*DAG.getContext(), VT.getVectorElementType(),
+ NumElts * Factor);
+ SDLoc DL(Node);
+
+ SmallVector<int, 16> Mask;
+ for (unsigned I = 0; I != Factor; ++I) {
+ SmallVector<int, 16> StrideMask = createStrideMask(I, Factor, NumElts);
+ Mask.append(StrideMask.begin(), StrideMask.end());
+ }
+
+ SmallVector<SDValue, 8> InVecs(Node->op_values());
+ SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, DL, WideVT, InVecs);
+ SDValue Deinterleaved =
+ DAG.getVectorShuffle(WideVT, DL, Concat, DAG.getUNDEF(WideVT), Mask);
+
+ for (unsigned I = 0; I != Factor; ++I)
+ Results.push_back(DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Deinterleaved,
+ DAG.getVectorIdxConstant(I * NumElts, DL)));
+}
void VectorLegalizer::Expand(SDNode *Node, SmallVectorImpl<SDValue> &Results) {
switch (Node->getOpcode()) {
@@ -982,6 +1135,12 @@ void VectorLegalizer::Expand(SDNode *Node, SmallVectorImpl<SDValue> &Results) {
for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i)
Results.push_back(Node->getOperand(i));
return;
+ case ISD::VECTOR_INTERLEAVE:
+ ExpandVectorInterleave(Node, Results);
+ return;
+ case ISD::VECTOR_DEINTERLEAVE:
+ ExpandVectorDeinterleave(Node, Results);
+ return;
case ISD::SIGN_EXTEND_INREG:
if (SDValue Expanded = ExpandSEXTINREG(Node)) {
Results.push_back(Expanded);
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index 0f6bd53cafdd8..db661c4e09330 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -13065,18 +13065,6 @@ void SelectionDAGBuilder::visitVectorDeinterleave(const CallInst &I,
DAG.getVectorIdxConstant(OutNumElts * i, DL));
}
- // Use VECTOR_SHUFFLE for fixed-length vectors with factor of 2 to benefit
- // from existing legalisation and combines.
- if (OutVT.isFixedLengthVector() && Factor == 2) {
- SDValue Even = DAG.getVectorShuffle(OutVT, DL, SubVecs[0], SubVecs[1],
- createStrideMask(0, 2, OutNumElts));
- SDValue Odd = DAG.getVectorShuffle(OutVT, DL, SubVecs[0], SubVecs[1],
- createStrideMask(1, 2, OutNumElts));
- SDValue Res = DAG.getMergeValues({Even, Odd}, getCurSDLoc());
- setValue(&I, Res);
- return;
- }
-
SDValue Res = DAG.getNode(ISD::VECTOR_DEINTERLEAVE, DL,
DAG.getVTList(ValueVTs), SubVecs);
setValue(&I, Res);
@@ -13096,16 +13084,6 @@ void SelectionDAGBuilder::visitVectorInterleave(const CallInst &I,
"Expected VTs to be the same");
}
- // Use VECTOR_SHUFFLE for fixed-length vectors with factor of 2 to benefit
- // from existing legalisation and combines.
- if (OutVT.isFixedLengthVector() && Factor == 2) {
- unsigned NumElts = InVT.getVectorMinNumElements();
- SDValue V = DAG.getNode(ISD::CONCAT_VECTORS, DL, OutVT, InVecs);
- setValue(&I, DAG.getVectorShuffle(OutVT, DL, V, DAG.getUNDEF(OutVT),
- createInterleaveMask(NumElts, 2)));
- return;
- }
-
SmallVector<EVT, 8> ValueVTs(Factor, InVT);
SDValue Res =
DAG.getNode(ISD::VECTOR_INTERLEAVE, DL, DAG.getVTList(ValueVTs), InVecs);
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.h b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
index 6e395e004f519..05cc49ae834c0 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.h
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
@@ -231,6 +231,9 @@ class AArch64TargetLowering : public TargetLowering {
bool hasPairedLoad(EVT LoadedType, Align &RequiredAlignment) const override;
unsigned getMaxSupportedInterleaveFactor() const override { return 4; }
+ bool isInterleaveIntrinsicSupported(unsigned Factor, EVT VT) const override {
+ return VT.isScalableVector() || Factor == 3 || Factor % 2 == 0;
+ }
bool lowerInterleavedLoad(Instruction *Load, Value *Mask,
ArrayRef<ShuffleVectorInst *> Shuffles,
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.h b/llvm/lib/Target/RISCV/RISCVISelLowering.h
index f8a99d38e2691..6ae942a09e911 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.h
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.h
@@ -436,6 +436,9 @@ class RISCVTargetLowering : public TargetLowering {
bool isLegalFirstFaultLoad(EVT DataType, Align Alignment) const;
unsigned getMaxSupportedInterleaveFactor() const override { return 8; }
+ bool isInterleaveIntrinsicSupported(unsigned Factor, EVT VT) const override {
+ return Factor >= 2 && Factor <= 8;
+ }
bool fallBackToDAGISel(const Instruction &Inst) const override;
diff --git a/llvm/test/CodeGen/AArch64/complex-deinterleaving-reductions.ll b/llvm/test/CodeGen/AArch64/complex-deinterleaving-reductions.ll
index 355adec955e4b..3e3e3389c146d 100644
--- a/llvm/test/CodeGen/AArch64/complex-deinterleaving-reductions.ll
+++ b/llvm/test/CodeGen/AArch64/complex-deinterleaving-reductions.ll
@@ -31,8 +31,8 @@ define dso_local %"struct.std::complex" @complex_mul_v2f64(ptr %a, ptr %b) {
; CHECK-NEXT: fcmla v0.2d, v4.2d, v2.2d, #90
; CHECK-NEXT: b.ne .LBB0_1
; CHECK-NEXT: // %bb.2: // %middle.block
-; CHECK-NEXT: zip2 v2.2d, v1.2d, v0.2d
-; CHECK-NEXT: zip1 v0.2d, v1.2d, v0.2d
+; CHECK-NEXT: uzp2 v2.2d, v1.2d, v0.2d
+; CHECK-NEXT: uzp1 v0.2d, v1.2d, v0.2d
; CHECK-NEXT: faddp d0, v0.2d
; CHECK-NEXT: faddp d1, v2.2d
; CHECK-NEXT: ret
@@ -80,10 +80,13 @@ middle.block: ; preds = %vector.body
define %"struct.std::complex" @complex_mul_nonzero_init_v2f64(ptr %a, ptr %b) {
; CHECK-LABEL: complex_mul_nonzero_init_v2f64:
; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: movi v0.2d, #0000000000000000
; CHECK-NEXT: adrp x8, .LCPI1_0
+; CHECK-NEXT: adrp x9, .LCPI1_1
; CHECK-NEXT: ldr q1, [x8, :lo12:.LCPI1_0]
+; CHECK-NEXT: ldr q2, [x9, :lo12:.LCPI1_1]
; CHECK-NEXT: mov x8, xzr
+; CHECK-NEXT: zip2 v0.2d, v2.2d, v1.2d
+; CHECK-NEXT: zip1 v1.2d, v2.2d, v1.2d
; CHECK-NEXT: .LBB1_1: // %vector.body
; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
; CHECK-NEXT: add x9, x0, x8
@@ -98,8 +101,8 @@ define %"struct.std::complex" @complex_mul_nonzero_init_v2f64(ptr %a, ptr %b) {
; CHECK-NEXT: fcmla v0.2d, v4.2d, v2.2d, #90
; CHECK-NEXT: b.ne .LBB1_1
; CHECK-NEXT: // %bb.2: // %middle.block
-; CHECK-NEXT: zip2 v2.2d, v1.2d, v0.2d
-; CHECK-NEXT: zip1 v0.2d, v1.2d, v0.2d
+; CHECK-NEXT: uzp2 v2.2d, v1.2d, v0.2d
+; CHECK-NEXT: uzp1 v0.2d, v1.2d, v0.2d
; CHECK-NEXT: faddp d0, v0.2d
; CHECK-NEXT: faddp d1, v2.2d
; CHECK-NEXT: ret
@@ -143,14 +146,17 @@ middle.block: ; preds = %vector.body
define %"struct.std::complex" @complex_mul_v2f64_unrolled(ptr %a, ptr %b) {
; CHECK-LABEL: complex_mul_v2f64_unrolled:
; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: movi v0.2d, #0000000000000000
-; CHECK-NEXT: movi v1.2d, #0000000000000000
; CHECK-NEXT: adrp x8, .LCPI2_0
+; CHECK-NEXT: adrp x9, .LCPI2_1
; CHECK-NEXT: movi v3.2d, #0000000000000000
-; CHECK-NEXT: ldr q2, [x8, :lo12:.LCPI2_0]
+; CHECK-NEXT: ldr q1, [x8, :lo12:.LCPI2_0]
+; CHECK-NEXT: ldr q2, [x9, :lo12:.LCPI2_1]
; CHECK-NEXT: add x8, x0, #32
; CHECK-NEXT: add x9, x1, #32
; CHECK-NEXT: mov x10, #-100 // =0xffffffffffffff9c
+; CHECK-NEXT: zip2 v0.2d, v2.2d, v1.2d
+; CHECK-NEXT: zip1 v1.2d, v2.2d, v1.2d
+; CHECK-NEXT: movi v2.2d, #0000000000000000
; CHECK-NEXT: .LBB2_1: // %vector.body
; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
; CHECK-NEXT: ldp q5, q4, [x8, #-32]
@@ -158,24 +164,24 @@ define %"struct.std::complex" @complex_mul_v2f64_unrolled(ptr %a, ptr %b) {
; CHECK-NEXT: ldp q7, q6, [x9, #-32]
; CHECK-NEXT: ldp q17, q16, [x8], #64
; CHECK-NEXT: ldp q19, q18, [x9], #64
-; CHECK-NEXT: fcmla v2.2d, v7.2d, v5.2d, #0
-; CHECK-NEXT: fcmla v1.2d, v6.2d, v4.2d, #0
-; CHECK-NEXT: fcmla v0.2d, v19.2d, v17.2d, #0
-; CHECK-NEXT: fcmla v3.2d, v18.2d, v16.2d, #0
-; CHECK-NEXT: fcmla v2.2d, v7.2d, v5.2d, #90
-; CHECK-NEXT: fcmla v1.2d, v6.2d, v4.2d, #90
-; CHECK-NEXT: fcmla v0.2d, v19.2d, v17.2d, #90
-; CHECK-NEXT: fcmla v3.2d, v18.2d, v16.2d, #90
+; CHECK-NEXT: fcmla v1.2d, v7.2d, v5.2d, #0
+; CHECK-NEXT: fcmla v0.2d, v6.2d, v4.2d, #0
+; CHECK-NEXT: fcmla v3.2d, v19.2d, v17.2d, #0
+; CHECK-NEXT: fcmla v2.2d, v18.2d, v16.2d, #0
+; CHECK-NEXT: fcmla v1.2d, v7.2d, v5.2d, #90
+; CHECK-NEXT: fcmla v0.2d, v6.2d, v4.2d, #90
+; CHECK-NEXT: fcmla v3.2d, v19.2d, v17.2d, #90
+; CHECK-NEXT: fcmla v2.2d, v18.2d, v16.2d, #90
; CHECK-NEXT: b.ne .LBB2_1
; CHECK-NEXT: // %bb.2: // %middle.block
-; CHECK-NEXT: zip2 v4.2d, v0.2d, v3.2d
-; CHECK-NEXT: zip1 v0.2d, v0.2d, v3.2d
-; CHECK-NEXT: zip2 v3.2d, v2.2d, v1.2d
-; CHECK-NEXT: zip1 v1.2d, v2.2d, v1.2d
-; CHECK-NEXT: fadd v0.2d, v0.2d, v1.2d
+; CHECK-NEXT: uzp2 v4.2d, v3.2d, v2.2d
+; CHECK-NEXT: uzp1 v2.2d, v3.2d, v2.2d
+; CHECK-NEXT: uzp2 v3.2d, v1.2d, v0.2d
+; CHECK-NEXT: uzp1 v0.2d, v1.2d, v0.2d
; CHECK-NEXT: fadd v1.2d, v4.2d, v3.2d
-; CHECK-NEXT: faddp d0, v0.2d
+; CHECK-NEXT: fadd v0.2d, v2.2d, v0.2d
; CHECK-NEXT: faddp d1, v1.2d
+; CHECK-NEXT: faddp d0, v0.2d
; CHECK-NEXT: ret
entry:
%scevgep = getelementptr i8, ptr %a, i64 32
diff --git a/llvm/test/CodeGen/AArch64/complex-deinterleaving-splat.ll b/llvm/test/CodeGen/AArch64/complex-deinterleaving-splat.ll
index ad9240b0922bd..f8c7a25f47185 100644
--- a/llvm/test/CodeGen/AArch64/complex-deinterleaving-splat.ll
+++ b/llvm/test/CodeGen/AArch64/complex-deinterleaving-splat.ll
@@ -9,21 +9,23 @@ target triple = "aarch64"
define <4 x double> @complex_mul_const(<4 x double> %a, <4 x double> %b) {
; CHECK-LABEL: complex_mul_const:
; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: movi v6.2d, #0000000000000000
-; CHECK-NEXT: movi v...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/210494
More information about the llvm-commits
mailing list