[llvm] [CodeGen] Add Action types for vector_(de)interleave ops (PR #220859)
Kamlesh Kumar via llvm-commits
llvm-commits at lists.llvm.org
Thu Sep 3 08:19:56 PDT 2026
https://github.com/kamleshbhalui updated https://github.com/llvm/llvm-project/pull/220859
>From 2abfc5559d181c8b1fbf9865406a6935f8046bb1 Mon Sep 17 00:00:00 2001
From: Kamlesh Kumar <kamlesh.kumar at arm.com>
Date: Wed, 2 Sep 2026 20:12:36 +0100
Subject: [PATCH 1/3] [CodeGen] Add Action types for vector_(de)interleave ops
Targets could not set lowering actions with setOperationAction
for Interleave operations due to dependence on type and factor.
Instead of requiring two API to find legality of operation one
for type and one for factor.
It is cleaner to add a separate action type for these operation.
---
llvm/include/llvm/CodeGen/TargetLowering.h | 46 +++++++++++++++++++
llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp | 5 ++
.../Target/AArch64/AArch64ISelLowering.cpp | 36 +++++++++++----
llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 35 ++++++++------
4 files changed, 99 insertions(+), 23 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h
index a91b575dc82d2..601b99c24d357 100644
--- a/llvm/include/llvm/CodeGen/TargetLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetLowering.h
@@ -1775,6 +1775,25 @@ class LLVM_ABI TargetLoweringBase {
return Action == Legal || Action == Custom;
}
+ /// Return how a VECTOR_INTERLEAVE or VECTOR_DEINTERLEAVE node with the
+ /// given interleave factor and VT should be handled.
+ LegalizeAction getVectorInterleaveAction(unsigned Opc, unsigned Factor,
+ EVT VT) const {
+ assert((Opc == ISD::VECTOR_INTERLEAVE || Opc == ISD::VECTOR_DEINTERLEAVE));
+ assert(Factor >= 2 && Factor <= 8);
+ VectorInterleaveActionKey Key = {Opc, Factor, VT.getSimpleVT().SimpleTy};
+ auto It = VectorInterleaveActions.find(Key);
+ return It != VectorInterleaveActions.end() ? It->second : Expand;
+ }
+
+ /// Return true if a VECTOR_INTERLEAVE or VECTOR_DEINTERLEAVE node with the
+ /// given interleave factor and fragment type is legal or custom.
+ bool isVectorInterleaveLegalOrCustom(unsigned Opc, unsigned Factor,
+ EVT VT) const {
+ LegalizeAction Action = getVectorInterleaveAction(Opc, Factor, VT);
+ return Action == Legal || Action == Custom;
+ }
+
/// If the action for this operation is to promote, this method returns the
/// ValueType to promote to.
MVT getTypeToPromoteTo(unsigned Op, MVT VT) const {
@@ -2875,6 +2894,27 @@ class LLVM_ABI TargetLoweringBase {
setPartialReduceMLAAction(Opc, AccVT, InputVT, Action);
}
+ /// Indicate how a VECTOR_INTERLEAVE or VECTOR_DEINTERLEAVE node with the
+ /// given interleave factor Factor and type VT should be
+ /// treated.
+ void setVectorInterleaveAction(unsigned Opc, unsigned Factor, MVT VT,
+ LegalizeAction Action) {
+ assert((Opc == ISD::VECTOR_INTERLEAVE || Opc == ISD::VECTOR_DEINTERLEAVE));
+ assert(Factor >= 2 && Factor <= 8);
+ assert((Action == Legal || Action == Custom || Action == Expand) &&
+ "Unsupported vector interleave action");
+ VectorInterleaveActionKey Key = {Opc, Factor, VT.SimpleTy};
+ VectorInterleaveActions[Key] = Action;
+ }
+
+ void setVectorInterleaveAction(ArrayRef<unsigned> Opcodes,
+ ArrayRef<unsigned> Factors, MVT VT,
+ LegalizeAction Action) {
+ for (unsigned Opc : Opcodes)
+ for (unsigned Factor : Factors)
+ setVectorInterleaveAction(Opc, Factor, VT, Action);
+ }
+
/// If Opc/OrigVT is specified as being promoted, the promotion code defaults
/// to trying a larger integer/fp until it can find one that works. If that
/// default is insufficient, this method can be used by the target to override
@@ -3909,6 +3949,12 @@ class LLVM_ABI TargetLoweringBase {
/// deal with this operation.
DenseMap<PartialReduceActionTypes, LegalizeAction> PartialReduceMLAActions;
+ using VectorInterleaveActionKey =
+ std::tuple<unsigned, unsigned, MVT::SimpleValueType>;
+ /// For each vector (de)interleave opcode, interleave factor and fragment
+ /// type combination, keep the corresponding LegalizeAction.
+ DenseMap<VectorInterleaveActionKey, LegalizeAction> VectorInterleaveActions;
+
ValueTypeActionImpl ValueTypeActions;
private:
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
index 70e094bc0d255..e1d144285bfd8 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
@@ -1271,6 +1271,11 @@ void SelectionDAGLegalize::LegalizeOp(SDNode *Node) {
Action = TLI.getOperationAction(Node->getOpcode(),
Node->getOperand(0).getValueType());
break;
+ case ISD::VECTOR_INTERLEAVE:
+ case ISD::VECTOR_DEINTERLEAVE:
+ Action = TLI.getVectorInterleaveAction(
+ Node->getOpcode(), Node->getNumOperands(), Node->getValueType(0));
+ break;
case ISD::EXPERIMENTAL_VECTOR_HISTOGRAM:
Action = TLI.getOperationAction(
Node->getOpcode(),
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 36a86241fe3b2..a30c18e9f248e 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -1640,8 +1640,9 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
{MVT::nxv16i1, MVT::nxv8i1, MVT::nxv4i1, MVT::nxv2i1, MVT::nxv1i1}) {
setOperationAction(ISD::SPLAT_VECTOR, VT, Custom);
setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
- setOperationAction(ISD::VECTOR_DEINTERLEAVE, VT, Custom);
- setOperationAction(ISD::VECTOR_INTERLEAVE, VT, Custom);
+ setVectorInterleaveAction(
+ {ISD::VECTOR_INTERLEAVE, ISD::VECTOR_DEINTERLEAVE}, {2, 3}, VT,
+ Custom);
}
for (auto VT : {MVT::nxv16i1, MVT::nxv8i1, MVT::nxv4i1, MVT::nxv2i1}) {
setOperationAction({ISD::CTTZ_ELTS, ISD::CTTZ_ELTS_ZERO_POISON}, VT,
@@ -1716,8 +1717,13 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
setOperationAction(ISD::VECREDUCE_UMAX, VT, Custom);
setOperationAction(ISD::VECREDUCE_SMIN, VT, Custom);
setOperationAction(ISD::VECREDUCE_SMAX, VT, Custom);
- setOperationAction(ISD::VECTOR_DEINTERLEAVE, VT, Custom);
- setOperationAction(ISD::VECTOR_INTERLEAVE, VT, Custom);
+ setVectorInterleaveAction(
+ {ISD::VECTOR_INTERLEAVE, ISD::VECTOR_DEINTERLEAVE}, {2, 3}, VT,
+ Custom);
+ if (Subtarget->hasSME2() && Subtarget->isStreaming())
+ setVectorInterleaveAction(
+ {ISD::VECTOR_INTERLEAVE, ISD::VECTOR_DEINTERLEAVE}, {4}, VT,
+ Custom);
setOperationAction(ISD::UMUL_LOHI, VT, Expand);
setOperationAction(ISD::SMUL_LOHI, VT, Expand);
@@ -1889,8 +1895,13 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
setOperationAction(ISD::VECREDUCE_FMUL, VT, Custom);
setOperationAction(ISD::VECTOR_SPLICE_LEFT, VT, Custom);
setOperationAction(ISD::VECTOR_SPLICE_RIGHT, VT, Custom);
- setOperationAction(ISD::VECTOR_DEINTERLEAVE, VT, Custom);
- setOperationAction(ISD::VECTOR_INTERLEAVE, VT, Custom);
+ setVectorInterleaveAction(
+ {ISD::VECTOR_INTERLEAVE, ISD::VECTOR_DEINTERLEAVE}, {2, 3}, VT,
+ Custom);
+ if (Subtarget->hasSME2() && Subtarget->isStreaming())
+ setVectorInterleaveAction(
+ {ISD::VECTOR_INTERLEAVE, ISD::VECTOR_DEINTERLEAVE}, {4}, VT,
+ Custom);
setOperationAction(ISD::SELECT_CC, VT, Expand);
setOperationAction({ISD::FREM, ISD::STRICT_FREM}, VT, Expand);
@@ -1959,8 +1970,13 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
setOperationAction(ISD::SELECT, VT, Custom);
setOperationAction(ISD::SELECT_CC, VT, Expand);
setOperationAction(ISD::SPLAT_VECTOR, VT, Legal);
- setOperationAction(ISD::VECTOR_DEINTERLEAVE, VT, Custom);
- setOperationAction(ISD::VECTOR_INTERLEAVE, VT, Custom);
+ setVectorInterleaveAction(
+ {ISD::VECTOR_INTERLEAVE, ISD::VECTOR_DEINTERLEAVE}, {2, 3}, VT,
+ Custom);
+ if (Subtarget->hasSME2() && Subtarget->isStreaming())
+ setVectorInterleaveAction(
+ {ISD::VECTOR_INTERLEAVE, ISD::VECTOR_DEINTERLEAVE}, {4}, VT,
+ Custom);
setOperationAction(ISD::VECTOR_SPLICE_LEFT, VT, Custom);
setOperationAction(ISD::VECTOR_SPLICE_RIGHT, VT, Custom);
}
@@ -2386,8 +2402,8 @@ void AArch64TargetLowering::addTypeForNEON(MVT VT) {
setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
setOperationAction(ISD::ZERO_EXTEND_VECTOR_INREG, VT, Custom);
- setOperationAction(ISD::VECTOR_DEINTERLEAVE, VT, Custom);
- setOperationAction(ISD::VECTOR_INTERLEAVE, VT, Custom);
+ setVectorInterleaveAction({ISD::VECTOR_INTERLEAVE, ISD::VECTOR_DEINTERLEAVE},
+ {2, 3}, VT, Custom);
setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom);
setOperationAction(ISD::SRA, VT, Custom);
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index 758d0369739ec..94ad7077cf318 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -1092,8 +1092,9 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
OtherVT, Expand);
}
- setOperationAction(ISD::VECTOR_DEINTERLEAVE, VT, Custom);
- setOperationAction(ISD::VECTOR_INTERLEAVE, VT, Custom);
+ setVectorInterleaveAction(
+ {ISD::VECTOR_INTERLEAVE, ISD::VECTOR_DEINTERLEAVE},
+ {2, 3, 4, 5, 6, 7, 8}, VT, Custom);
setOperationAction(ISD::VECTOR_REVERSE, VT, Custom);
@@ -1195,8 +1196,9 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
OtherVT, Expand);
}
- setOperationAction(ISD::VECTOR_DEINTERLEAVE, VT, Custom);
- setOperationAction(ISD::VECTOR_INTERLEAVE, VT, Custom);
+ setVectorInterleaveAction(
+ {ISD::VECTOR_INTERLEAVE, ISD::VECTOR_DEINTERLEAVE},
+ {2, 3, 4, 5, 6, 7, 8}, VT, Custom);
setOperationAction({ISD::VECTOR_SPLICE_LEFT, ISD::VECTOR_SPLICE_RIGHT},
VT, Custom);
@@ -1380,8 +1382,9 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
ISD::EXTRACT_SUBVECTOR, ISD::SCALAR_TO_VECTOR},
VT, Custom);
- setOperationAction(ISD::VECTOR_DEINTERLEAVE, VT, Custom);
- setOperationAction(ISD::VECTOR_INTERLEAVE, VT, Custom);
+ setVectorInterleaveAction(
+ {ISD::VECTOR_INTERLEAVE, ISD::VECTOR_DEINTERLEAVE},
+ {2, 3, 4, 5, 6, 7, 8}, VT, Custom);
setOperationAction({ISD::VECTOR_REVERSE, ISD::VECTOR_SPLICE_LEFT,
ISD::VECTOR_SPLICE_RIGHT},
@@ -1428,10 +1431,12 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
setOperationAction(ISD::SELECT_CC, VT, Expand);
setOperationAction({ISD::INSERT_VECTOR_ELT, ISD::CONCAT_VECTORS,
ISD::INSERT_SUBVECTOR, ISD::EXTRACT_SUBVECTOR,
- ISD::VECTOR_DEINTERLEAVE, ISD::VECTOR_INTERLEAVE,
ISD::VECTOR_REVERSE, ISD::VECTOR_SPLICE_LEFT,
ISD::VECTOR_SPLICE_RIGHT, ISD::VECTOR_COMPRESS},
VT, Custom);
+ setVectorInterleaveAction(
+ {ISD::VECTOR_INTERLEAVE, ISD::VECTOR_DEINTERLEAVE},
+ {2, 3, 4, 5, 6, 7, 8}, VT, Custom);
setOperationAction(ISD::EXPERIMENTAL_VP_SPLICE, VT, Custom);
setOperationAction(ISD::EXPERIMENTAL_VP_REVERSE, VT, Custom);
MVT EltVT = VT.getVectorElementType();
@@ -1481,11 +1486,13 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
setOperationAction(ISD::SELECT_CC, VT, Expand);
setOperationAction({ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT,
ISD::CONCAT_VECTORS, ISD::INSERT_SUBVECTOR,
- ISD::EXTRACT_SUBVECTOR, ISD::VECTOR_DEINTERLEAVE,
- ISD::VECTOR_INTERLEAVE, ISD::VECTOR_REVERSE,
+ ISD::EXTRACT_SUBVECTOR, ISD::VECTOR_REVERSE,
ISD::VECTOR_SPLICE_LEFT, ISD::VECTOR_SPLICE_RIGHT,
ISD::VECTOR_COMPRESS},
VT, Custom);
+ setVectorInterleaveAction(
+ {ISD::VECTOR_INTERLEAVE, ISD::VECTOR_DEINTERLEAVE},
+ {2, 3, 4, 5, 6, 7, 8}, VT, Custom);
setOperationAction(
{ISD::FMINNUM, ISD::FMAXNUM, ISD::FMAXIMUMNUM, ISD::FMINIMUMNUM}, VT,
Legal);
@@ -1600,8 +1607,9 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
{ISD::BUILD_VECTOR, ISD::CONCAT_VECTORS, ISD::VECTOR_REVERSE}, VT,
Custom);
- setOperationAction({ISD::VECTOR_INTERLEAVE, ISD::VECTOR_DEINTERLEAVE},
- VT, Custom);
+ setVectorInterleaveAction(
+ {ISD::VECTOR_INTERLEAVE, ISD::VECTOR_DEINTERLEAVE},
+ {2, 3, 4, 5, 6, 7, 8}, VT, Custom);
setOperationAction({ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT},
VT, Custom);
@@ -1779,8 +1787,9 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
setOperationAction(ISD::EXPERIMENTAL_VP_SPLICE, VT, Custom);
setOperationAction(ISD::EXPERIMENTAL_VP_REVERSE, VT, Custom);
- setOperationAction({ISD::VECTOR_INTERLEAVE, ISD::VECTOR_DEINTERLEAVE},
- VT, Custom);
+ setVectorInterleaveAction(
+ {ISD::VECTOR_INTERLEAVE, ISD::VECTOR_DEINTERLEAVE},
+ {2, 3, 4, 5, 6, 7, 8}, VT, Custom);
setOperationAction({ISD::LOAD, ISD::STORE, ISD::MLOAD, ISD::MSTORE,
ISD::MGATHER, ISD::MSCATTER},
>From fdf2e50864f5526ed3818beedcdf67b8e88d537d Mon Sep 17 00:00:00 2001
From: Kamlesh Kumar <kamlesh.kumar at arm.com>
Date: Thu, 3 Sep 2026 12:18:13 +0100
Subject: [PATCH 2/3] fixup remove unnecessary asserts
---
llvm/include/llvm/CodeGen/TargetLowering.h | 4 ----
1 file changed, 4 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h
index 601b99c24d357..0ff8d4521607b 100644
--- a/llvm/include/llvm/CodeGen/TargetLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetLowering.h
@@ -1780,7 +1780,6 @@ class LLVM_ABI TargetLoweringBase {
LegalizeAction getVectorInterleaveAction(unsigned Opc, unsigned Factor,
EVT VT) const {
assert((Opc == ISD::VECTOR_INTERLEAVE || Opc == ISD::VECTOR_DEINTERLEAVE));
- assert(Factor >= 2 && Factor <= 8);
VectorInterleaveActionKey Key = {Opc, Factor, VT.getSimpleVT().SimpleTy};
auto It = VectorInterleaveActions.find(Key);
return It != VectorInterleaveActions.end() ? It->second : Expand;
@@ -2900,9 +2899,6 @@ class LLVM_ABI TargetLoweringBase {
void setVectorInterleaveAction(unsigned Opc, unsigned Factor, MVT VT,
LegalizeAction Action) {
assert((Opc == ISD::VECTOR_INTERLEAVE || Opc == ISD::VECTOR_DEINTERLEAVE));
- assert(Factor >= 2 && Factor <= 8);
- assert((Action == Legal || Action == Custom || Action == Expand) &&
- "Unsupported vector interleave action");
VectorInterleaveActionKey Key = {Opc, Factor, VT.SimpleTy};
VectorInterleaveActions[Key] = Action;
}
>From 676ca13036a5da74756512932ff8601154c39bb6 Mon Sep 17 00:00:00 2001
From: Kamlesh Kumar <kamlesh.kumar at arm.com>
Date: Thu, 3 Sep 2026 15:18:18 +0000
Subject: [PATCH 3/3] fixup apply suggestion
---
llvm/include/llvm/CodeGen/TargetLowering.h | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h
index 0ff8d4521607b..171f8253b3e70 100644
--- a/llvm/include/llvm/CodeGen/TargetLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetLowering.h
@@ -2894,8 +2894,7 @@ class LLVM_ABI TargetLoweringBase {
}
/// Indicate how a VECTOR_INTERLEAVE or VECTOR_DEINTERLEAVE node with the
- /// given interleave factor Factor and type VT should be
- /// treated.
+ /// given interleave factor Factor and type VT should be treated.
void setVectorInterleaveAction(unsigned Opc, unsigned Factor, MVT VT,
LegalizeAction Action) {
assert((Opc == ISD::VECTOR_INTERLEAVE || Opc == ISD::VECTOR_DEINTERLEAVE));
More information about the llvm-commits
mailing list