[llvm] [SelectionDAG] Add an ISD node for for get.active.lane.mask (PR #139084)
Kerry McLaughlin via llvm-commits
llvm-commits at lists.llvm.org
Wed May 14 04:04:54 PDT 2025
https://github.com/kmclaughlin-arm updated https://github.com/llvm/llvm-project/pull/139084
>From b4a9650ab3b09d9e267fe2e7a6ec8e5f36bd40a1 Mon Sep 17 00:00:00 2001
From: Kerry McLaughlin <kerry.mclaughlin at arm.com>
Date: Tue, 1 Apr 2025 16:46:56 +0000
Subject: [PATCH 1/3] [SelectionDAG] Add an ISD node for for
get.active.lane.mask
For now expansion still happens in SelectionDAGBuilder when
GET_ACTIVE_LANE_MASK is not legal on the target.
This patch also includes changes in AArch64ISelLowering to replace
handling of the get.active.lane.mask intrinsic to use the ISD node.
Tablegen patterns are added which match to whilelo for scalable types.
A follow up change will add support for more types to be lowered to
GET_ACTIVE_LANE_MASK by allowing splitting of the node.
---
llvm/include/llvm/CodeGen/ISDOpcodes.h | 6 +
.../include/llvm/Target/TargetSelectionDAG.td | 6 +
.../SelectionDAG/LegalizeIntegerTypes.cpp | 10 +
llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h | 1 +
.../SelectionDAG/SelectionDAGBuilder.cpp | 5 +-
.../SelectionDAG/SelectionDAGDumper.cpp | 3 +
.../Target/AArch64/AArch64ISelLowering.cpp | 220 ++++++++++--------
llvm/lib/Target/AArch64/AArch64ISelLowering.h | 1 +
.../lib/Target/AArch64/AArch64SVEInstrInfo.td | 18 ++
9 files changed, 165 insertions(+), 105 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/ISDOpcodes.h b/llvm/include/llvm/CodeGen/ISDOpcodes.h
index 80ef32aff62ae..ba29c118007dd 100644
--- a/llvm/include/llvm/CodeGen/ISDOpcodes.h
+++ b/llvm/include/llvm/CodeGen/ISDOpcodes.h
@@ -1524,6 +1524,12 @@ enum NodeType {
// Operands: Mask
VECTOR_FIND_LAST_ACTIVE,
+ // Creates a mask representing active and inactive
+ // vector lanes, active while base < trip count.
+ // Operands: Base, Trip Count
+ // Output: Mask
+ GET_ACTIVE_LANE_MASK,
+
// llvm.clear_cache intrinsic
// Operands: Input Chain, Start Addres, End Address
// Outputs: Output Chain
diff --git a/llvm/include/llvm/Target/TargetSelectionDAG.td b/llvm/include/llvm/Target/TargetSelectionDAG.td
index 41fed692c7025..5b8e01951edf2 100644
--- a/llvm/include/llvm/Target/TargetSelectionDAG.td
+++ b/llvm/include/llvm/Target/TargetSelectionDAG.td
@@ -860,6 +860,12 @@ def find_last_active
: SDNode<"ISD::VECTOR_FIND_LAST_ACTIVE",
SDTypeProfile<1, 1, [SDTCisInt<0>, SDTCisVec<1>]>, []>;
+def get_active_lane_mask
+ : SDNode<
+ "ISD::GET_ACTIVE_LANE_MASK",
+ SDTypeProfile<1, 2, [SDTCisVec<0>, SDTCisInt<1>, SDTCisSameAs<1, 2>]>,
+ []>;
+
// Nodes for intrinsics, you should use the intrinsic itself and let tblgen use
// these internally. Don't reference these directly.
def intrinsic_void : SDNode<"ISD::INTRINSIC_VOID",
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
index 25e74a2ae5b71..9ccd6a4d1684c 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
@@ -160,6 +160,10 @@ void DAGTypeLegalizer::PromoteIntegerResult(SDNode *N, unsigned ResNo) {
Res = PromoteIntRes_VECTOR_FIND_LAST_ACTIVE(N);
break;
+ case ISD::GET_ACTIVE_LANE_MASK:
+ Res = PromoteIntRes_GET_ACTIVE_LANE_MASK(N);
+ break;
+
case ISD::PARTIAL_REDUCE_UMLA:
case ISD::PARTIAL_REDUCE_SMLA:
Res = PromoteIntRes_PARTIAL_REDUCE_MLA(N);
@@ -6222,6 +6226,12 @@ SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_FIND_LAST_ACTIVE(SDNode *N) {
return DAG.getNode(ISD::VECTOR_FIND_LAST_ACTIVE, SDLoc(N), NVT, N->ops());
}
+SDValue DAGTypeLegalizer::PromoteIntRes_GET_ACTIVE_LANE_MASK(SDNode *N) {
+ EVT VT = N->getValueType(0);
+ EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
+ return DAG.getNode(ISD::GET_ACTIVE_LANE_MASK, SDLoc(N), NVT, N->ops());
+}
+
SDValue DAGTypeLegalizer::PromoteIntRes_PARTIAL_REDUCE_MLA(SDNode *N) {
SDLoc DL(N);
EVT VT = N->getValueType(0);
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
index 720393158aa5e..edd54fc550f53 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
@@ -379,6 +379,7 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
SDValue PromoteIntRes_IS_FPCLASS(SDNode *N);
SDValue PromoteIntRes_PATCHPOINT(SDNode *N);
SDValue PromoteIntRes_VECTOR_FIND_LAST_ACTIVE(SDNode *N);
+ SDValue PromoteIntRes_GET_ACTIVE_LANE_MASK(SDNode *N);
SDValue PromoteIntRes_PARTIAL_REDUCE_MLA(SDNode *N);
// Integer Operand Promotion.
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index 9d138d364bad7..3840f94a214e2 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -7987,14 +7987,15 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
case Intrinsic::get_active_lane_mask: {
EVT CCVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
SDValue Index = getValue(I.getOperand(0));
+ SDValue TripCount = getValue(I.getOperand(1));
EVT ElementVT = Index.getValueType();
if (!TLI.shouldExpandGetActiveLaneMask(CCVT, ElementVT)) {
- visitTargetIntrinsic(I, Intrinsic);
+ setValue(&I, DAG.getNode(ISD::GET_ACTIVE_LANE_MASK, sdl, CCVT, Index,
+ TripCount));
return;
}
- SDValue TripCount = getValue(I.getOperand(1));
EVT VecTy = EVT::getVectorVT(*DAG.getContext(), ElementVT,
CCVT.getVectorElementCount());
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
index 8faf97271d99e..cf35327b4a103 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
@@ -576,6 +576,9 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const {
case ISD::VECTOR_FIND_LAST_ACTIVE:
return "find_last_active";
+ case ISD::GET_ACTIVE_LANE_MASK:
+ return "get_active_lane_mask";
+
case ISD::PARTIAL_REDUCE_UMLA:
return "partial_reduce_umla";
case ISD::PARTIAL_REDUCE_SMLA:
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 6e7f13c20db68..0b10882e45924 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -1178,6 +1178,8 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
setTargetDAGCombine(ISD::CTLZ);
+ setTargetDAGCombine(ISD::GET_ACTIVE_LANE_MASK);
+
setTargetDAGCombine(ISD::VECREDUCE_AND);
setTargetDAGCombine(ISD::VECREDUCE_OR);
setTargetDAGCombine(ISD::VECREDUCE_XOR);
@@ -1493,8 +1495,13 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
setOperationAction(ISD::VECTOR_DEINTERLEAVE, VT, Custom);
setOperationAction(ISD::VECTOR_INTERLEAVE, VT, Custom);
}
- for (auto VT : {MVT::nxv16i1, MVT::nxv8i1, MVT::nxv4i1, MVT::nxv2i1})
+ for (auto VT : {MVT::nxv16i1, MVT::nxv8i1, MVT::nxv4i1, MVT::nxv2i1}) {
setOperationAction(ISD::VECTOR_FIND_LAST_ACTIVE, VT, Legal);
+ setOperationAction(ISD::GET_ACTIVE_LANE_MASK, VT, Legal);
+ }
+
+ for (auto VT : {MVT::v16i8, MVT::v8i8, MVT::v4i16, MVT::v2i32})
+ setOperationAction(ISD::GET_ACTIVE_LANE_MASK, VT, Custom);
}
if (Subtarget->isSVEorStreamingSVEAvailable()) {
@@ -5731,21 +5738,24 @@ static inline SDValue getPTrue(SelectionDAG &DAG, SDLoc DL, EVT VT,
DAG.getTargetConstant(Pattern, DL, MVT::i32));
}
-static SDValue optimizeIncrementingWhile(SDValue Op, SelectionDAG &DAG,
+static SDValue optimizeIncrementingWhile(SDNode *N, SelectionDAG &DAG,
bool IsSigned, bool IsEqual) {
- if (!isa<ConstantSDNode>(Op.getOperand(1)) ||
- !isa<ConstantSDNode>(Op.getOperand(2)))
+ unsigned Op0 = N->getOpcode() == ISD::INTRINSIC_WO_CHAIN ? 1 : 0;
+ unsigned Op1 = N->getOpcode() == ISD::INTRINSIC_WO_CHAIN ? 2 : 1;
+
+ if (!isa<ConstantSDNode>(N->getOperand(Op0)) ||
+ !isa<ConstantSDNode>(N->getOperand(Op1)))
return SDValue();
- SDLoc dl(Op);
- APInt X = Op.getConstantOperandAPInt(1);
- APInt Y = Op.getConstantOperandAPInt(2);
+ SDLoc dl(N);
+ APInt X = N->getConstantOperandAPInt(Op0);
+ APInt Y = N->getConstantOperandAPInt(Op1);
// When the second operand is the maximum value, comparisons that include
// equality can never fail and thus we can return an all active predicate.
if (IsEqual)
if (IsSigned ? Y.isMaxSignedValue() : Y.isMaxValue())
- return DAG.getConstant(1, dl, Op.getValueType());
+ return DAG.getConstant(1, dl, N->getValueType(0));
bool Overflow;
APInt NumActiveElems =
@@ -5766,10 +5776,10 @@ static SDValue optimizeIncrementingWhile(SDValue Op, SelectionDAG &DAG,
getSVEPredPatternFromNumElements(NumActiveElems.getZExtValue());
unsigned MinSVEVectorSize = std::max(
DAG.getSubtarget<AArch64Subtarget>().getMinSVEVectorSizeInBits(), 128u);
- unsigned ElementSize = 128 / Op.getValueType().getVectorMinNumElements();
+ unsigned ElementSize = 128 / N->getValueType(0).getVectorMinNumElements();
if (PredPattern != std::nullopt &&
NumActiveElems.getZExtValue() <= (MinSVEVectorSize / ElementSize))
- return getPTrue(DAG, dl, Op.getValueType(), *PredPattern);
+ return getPTrue(DAG, dl, N->getValueType(0), *PredPattern);
return SDValue();
}
@@ -6222,16 +6232,16 @@ SDValue AArch64TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
AArch64ISD::URSHR_I, dl, Op.getOperand(1).getValueType(), Op.getOperand(1), Op.getOperand(2)));
return SDValue();
case Intrinsic::aarch64_sve_whilelo:
- return optimizeIncrementingWhile(Op, DAG, /*IsSigned=*/false,
+ return optimizeIncrementingWhile(Op.getNode(), DAG, /*IsSigned=*/false,
/*IsEqual=*/false);
case Intrinsic::aarch64_sve_whilelt:
- return optimizeIncrementingWhile(Op, DAG, /*IsSigned=*/true,
+ return optimizeIncrementingWhile(Op.getNode(), DAG, /*IsSigned=*/true,
/*IsEqual=*/false);
case Intrinsic::aarch64_sve_whilels:
- return optimizeIncrementingWhile(Op, DAG, /*IsSigned=*/false,
+ return optimizeIncrementingWhile(Op.getNode(), DAG, /*IsSigned=*/false,
/*IsEqual=*/true);
case Intrinsic::aarch64_sve_whilele:
- return optimizeIncrementingWhile(Op, DAG, /*IsSigned=*/true,
+ return optimizeIncrementingWhile(Op.getNode(), DAG, /*IsSigned=*/true,
/*IsEqual=*/true);
case Intrinsic::aarch64_sve_sunpkhi:
return DAG.getNode(AArch64ISD::SUNPKHI, dl, Op.getValueType(),
@@ -6532,28 +6542,6 @@ SDValue AArch64TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
return DAG.getNode(AArch64ISD::USDOT, dl, Op.getValueType(),
Op.getOperand(1), Op.getOperand(2), Op.getOperand(3));
}
- case Intrinsic::get_active_lane_mask: {
- SDValue ID =
- DAG.getTargetConstant(Intrinsic::aarch64_sve_whilelo, dl, MVT::i64);
-
- EVT VT = Op.getValueType();
- if (VT.isScalableVector())
- return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT, ID, Op.getOperand(1),
- Op.getOperand(2));
-
- // We can use the SVE whilelo instruction to lower this intrinsic by
- // creating the appropriate sequence of scalable vector operations and
- // then extracting a fixed-width subvector from the scalable vector.
-
- EVT ContainerVT = getContainerForFixedLengthVector(DAG, VT);
- EVT WhileVT = ContainerVT.changeElementType(MVT::i1);
-
- SDValue Mask = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, WhileVT, ID,
- Op.getOperand(1), Op.getOperand(2));
- SDValue MaskAsInt = DAG.getNode(ISD::SIGN_EXTEND, dl, ContainerVT, Mask);
- return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, MaskAsInt,
- DAG.getVectorIdxConstant(0, dl));
- }
case Intrinsic::aarch64_neon_saddlv:
case Intrinsic::aarch64_neon_uaddlv: {
EVT OpVT = Op.getOperand(1).getValueType();
@@ -7692,6 +7680,8 @@ SDValue AArch64TargetLowering::LowerOperation(SDValue Op,
return LowerVECTOR_DEINTERLEAVE(Op, DAG);
case ISD::VECTOR_INTERLEAVE:
return LowerVECTOR_INTERLEAVE(Op, DAG);
+ case ISD::GET_ACTIVE_LANE_MASK:
+ return LowerGET_ACTIVE_LANE_MASK(Op, DAG);
case ISD::LRINT:
case ISD::LLRINT:
if (Op.getValueType().isVector())
@@ -18142,6 +18132,70 @@ static SDValue performVecReduceAddCombineWithUADDLP(SDNode *N,
return DAG.getNode(ISD::VECREDUCE_ADD, DL, MVT::i32, UADDLP);
}
+static SDValue
+performActiveLaneMaskCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,
+ const AArch64Subtarget *ST) {
+ if (SDValue While = optimizeIncrementingWhile(N, DCI.DAG, /*IsSigned=*/false,
+ /*IsEqual=*/false))
+ return While;
+
+ if (DCI.isBeforeLegalize())
+ return SDValue();
+
+ if (!ST->hasSVE2p1())
+ return SDValue();
+
+ if (!N->hasNUsesOfValue(2, 0))
+ return SDValue();
+
+ const uint64_t HalfSize = N->getValueType(0).getVectorMinNumElements() / 2;
+ if (HalfSize < 2)
+ return SDValue();
+
+ auto It = N->user_begin();
+ SDNode *Lo = *It++;
+ SDNode *Hi = *It;
+
+ if (Lo->getOpcode() != ISD::EXTRACT_SUBVECTOR ||
+ Hi->getOpcode() != ISD::EXTRACT_SUBVECTOR)
+ return SDValue();
+
+ uint64_t OffLo = Lo->getConstantOperandVal(1);
+ uint64_t OffHi = Hi->getConstantOperandVal(1);
+
+ if (OffLo > OffHi) {
+ std::swap(Lo, Hi);
+ std::swap(OffLo, OffHi);
+ }
+
+ if (OffLo != 0 || OffHi != HalfSize)
+ return SDValue();
+
+ EVT HalfVec = Lo->getValueType(0);
+ if (HalfVec != Hi->getValueType(0) ||
+ HalfVec.getVectorElementCount() != ElementCount::getScalable(HalfSize))
+ return SDValue();
+
+ SelectionDAG &DAG = DCI.DAG;
+ SDLoc DL(N);
+ SDValue ID =
+ DAG.getTargetConstant(Intrinsic::aarch64_sve_whilelo_x2, DL, MVT::i64);
+ SDValue Idx = N->getOperand(0);
+ SDValue TC = N->getOperand(1);
+ if (Idx.getValueType() != MVT::i64) {
+ Idx = DAG.getZExtOrTrunc(Idx, DL, MVT::i64);
+ TC = DAG.getZExtOrTrunc(TC, DL, MVT::i64);
+ }
+ auto R =
+ DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL,
+ {Lo->getValueType(0), Hi->getValueType(0)}, {ID, Idx, TC});
+
+ DCI.CombineTo(Lo, R.getValue(0));
+ DCI.CombineTo(Hi, R.getValue(1));
+
+ return SDValue(N, 0);
+}
+
// Turn a v8i8/v16i8 extended vecreduce into a udot/sdot and vecreduce
// vecreduce.add(ext(A)) to vecreduce.add(DOT(zero, A, one))
// vecreduce.add(mul(ext(A), ext(B))) to vecreduce.add(DOT(zero, A, B))
@@ -19672,6 +19726,8 @@ static SDValue getPTest(SelectionDAG &DAG, EVT VT, SDValue Pg, SDValue Op,
static bool isPredicateCCSettingOp(SDValue N) {
if ((N.getOpcode() == ISD::SETCC) ||
+ // get_active_lane_mask is lowered to a whilelo instruction.
+ (N.getOpcode() == ISD::GET_ACTIVE_LANE_MASK) ||
(N.getOpcode() == ISD::INTRINSIC_WO_CHAIN &&
(N.getConstantOperandVal(0) == Intrinsic::aarch64_sve_whilege ||
N.getConstantOperandVal(0) == Intrinsic::aarch64_sve_whilegt ||
@@ -19680,9 +19736,7 @@ static bool isPredicateCCSettingOp(SDValue N) {
N.getConstantOperandVal(0) == Intrinsic::aarch64_sve_whilele ||
N.getConstantOperandVal(0) == Intrinsic::aarch64_sve_whilelo ||
N.getConstantOperandVal(0) == Intrinsic::aarch64_sve_whilels ||
- N.getConstantOperandVal(0) == Intrinsic::aarch64_sve_whilelt ||
- // get_active_lane_mask is lowered to a whilelo instruction.
- N.getConstantOperandVal(0) == Intrinsic::get_active_lane_mask)))
+ N.getConstantOperandVal(0) == Intrinsic::aarch64_sve_whilelt)))
return true;
return false;
@@ -21796,66 +21850,6 @@ static SDValue convertMergedOpToPredOp(SDNode *N, unsigned Opc,
return SDValue();
}
-static SDValue tryCombineWhileLo(SDNode *N,
- TargetLowering::DAGCombinerInfo &DCI,
- const AArch64Subtarget *Subtarget) {
- if (DCI.isBeforeLegalize())
- return SDValue();
-
- if (!Subtarget->hasSVE2p1())
- return SDValue();
-
- if (!N->hasNUsesOfValue(2, 0))
- return SDValue();
-
- const uint64_t HalfSize = N->getValueType(0).getVectorMinNumElements() / 2;
- if (HalfSize < 2)
- return SDValue();
-
- auto It = N->user_begin();
- SDNode *Lo = *It++;
- SDNode *Hi = *It;
-
- if (Lo->getOpcode() != ISD::EXTRACT_SUBVECTOR ||
- Hi->getOpcode() != ISD::EXTRACT_SUBVECTOR)
- return SDValue();
-
- uint64_t OffLo = Lo->getConstantOperandVal(1);
- uint64_t OffHi = Hi->getConstantOperandVal(1);
-
- if (OffLo > OffHi) {
- std::swap(Lo, Hi);
- std::swap(OffLo, OffHi);
- }
-
- if (OffLo != 0 || OffHi != HalfSize)
- return SDValue();
-
- EVT HalfVec = Lo->getValueType(0);
- if (HalfVec != Hi->getValueType(0) ||
- HalfVec.getVectorElementCount() != ElementCount::getScalable(HalfSize))
- return SDValue();
-
- SelectionDAG &DAG = DCI.DAG;
- SDLoc DL(N);
- SDValue ID =
- DAG.getTargetConstant(Intrinsic::aarch64_sve_whilelo_x2, DL, MVT::i64);
- SDValue Idx = N->getOperand(1);
- SDValue TC = N->getOperand(2);
- if (Idx.getValueType() != MVT::i64) {
- Idx = DAG.getZExtOrTrunc(Idx, DL, MVT::i64);
- TC = DAG.getZExtOrTrunc(TC, DL, MVT::i64);
- }
- auto R =
- DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL,
- {Lo->getValueType(0), Hi->getValueType(0)}, {ID, Idx, TC});
-
- DCI.CombineTo(Lo, R.getValue(0));
- DCI.CombineTo(Hi, R.getValue(1));
-
- return SDValue(N, 0);
-}
-
SDValue tryLowerPartialReductionToDot(SDNode *N,
const AArch64Subtarget *Subtarget,
SelectionDAG &DAG) {
@@ -22310,8 +22304,6 @@ static SDValue performIntrinsicCombine(SDNode *N,
case Intrinsic::aarch64_sve_ptest_last:
return getPTest(DAG, N->getValueType(0), N->getOperand(1), N->getOperand(2),
AArch64CC::LAST_ACTIVE);
- case Intrinsic::aarch64_sve_whilelo:
- return tryCombineWhileLo(N, DCI, Subtarget);
}
return SDValue();
}
@@ -26738,6 +26730,8 @@ SDValue AArch64TargetLowering::PerformDAGCombine(SDNode *N,
return performExtractVectorEltCombine(N, DCI, Subtarget);
case ISD::VECREDUCE_ADD:
return performVecReduceAddCombine(N, DCI.DAG, Subtarget);
+ case ISD::GET_ACTIVE_LANE_MASK:
+ return performActiveLaneMaskCombine(N, DCI, Subtarget);
case AArch64ISD::UADDV:
return performUADDVCombine(N, DAG);
case AArch64ISD::SMULL:
@@ -27720,8 +27714,7 @@ void AArch64TargetLowering::ReplaceNodeResults(
DAG.getNode(ISD::TRUNCATE, DL, MVT::i1, RuntimePStateSM));
return;
}
- case Intrinsic::experimental_vector_match:
- case Intrinsic::get_active_lane_mask: {
+ case Intrinsic::experimental_vector_match: {
if (!VT.isFixedLengthVector() || VT.getVectorElementType() != MVT::i1)
return;
@@ -29515,6 +29508,27 @@ AArch64TargetLowering::LowerPARTIAL_REDUCE_MLA(SDValue Op,
return DAG.getNode(ISD::ADD, DL, ResultVT, Acc, Extended);
}
+SDValue
+AArch64TargetLowering::LowerGET_ACTIVE_LANE_MASK(SDValue Op,
+ SelectionDAG &DAG) const {
+ EVT VT = Op.getValueType();
+ assert(VT.isFixedLengthVector() && "Expected fixed length vector type!");
+
+ // We can use the SVE whilelo instruction to lower this intrinsic by
+ // creating the appropriate sequence of scalable vector operations and
+ // then extracting a fixed-width subvector from the scalable vector.
+
+ SDLoc DL(Op);
+ EVT ContainerVT = getContainerForFixedLengthVector(DAG, VT);
+ EVT WhileVT = ContainerVT.changeElementType(MVT::i1);
+
+ SDValue Mask = DAG.getNode(ISD::GET_ACTIVE_LANE_MASK, DL, WhileVT,
+ Op.getOperand(0), Op.getOperand(1));
+ SDValue MaskAsInt = DAG.getNode(ISD::SIGN_EXTEND, DL, ContainerVT, Mask);
+ return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, MaskAsInt,
+ DAG.getVectorIdxConstant(0, DL));
+}
+
SDValue
AArch64TargetLowering::LowerFixedLengthFPToIntToSVE(SDValue Op,
SelectionDAG &DAG) const {
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.h b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
index 9d8d1c22258be..ca93aea7c394f 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.h
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
@@ -1182,6 +1182,7 @@ class AArch64TargetLowering : public TargetLowering {
SDValue LowerVECTOR_INTERLEAVE(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerVECTOR_HISTOGRAM(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerPARTIAL_REDUCE_MLA(SDValue Op, SelectionDAG &DAG) const;
+ SDValue LowerGET_ACTIVE_LANE_MASK(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerDIV(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerMUL(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerVectorSRA_SRL_SHL(SDValue Op, SelectionDAG &DAG) const;
diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
index bd394671881e8..b9dd599bdb0ce 100644
--- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
@@ -3449,6 +3449,24 @@ let Predicates = [HasSVE_or_SME] in {
def : Pat<(i64(find_last_active nxv2i1:$P1)), (LASTB_RPZ_D $P1, (INDEX_II_D 0,
1))>;
+ def : Pat<(nxv16i1(get_active_lane_mask i32:$index, i32:$tc)),
+ (WHILELO_PWW_B $index, $tc)>;
+ def : Pat<(nxv8i1(get_active_lane_mask i32:$index, i32:$tc)),
+ (WHILELO_PWW_H $index, $tc)>;
+ def : Pat<(nxv4i1(get_active_lane_mask i32:$index, i32:$tc)),
+ (WHILELO_PWW_S $index, $tc)>;
+ def : Pat<(nxv2i1(get_active_lane_mask i32:$index, i32:$tc)),
+ (WHILELO_PWW_D $index, $tc)>;
+
+ def : Pat<(nxv16i1(get_active_lane_mask i64:$index, i64:$tc)),
+ (WHILELO_PXX_B $index, $tc)>;
+ def : Pat<(nxv8i1(get_active_lane_mask i64:$index, i64:$tc)),
+ (WHILELO_PXX_H $index, $tc)>;
+ def : Pat<(nxv4i1(get_active_lane_mask i64:$index, i64:$tc)),
+ (WHILELO_PXX_S $index, $tc)>;
+ def : Pat<(nxv2i1(get_active_lane_mask i64:$index, i64:$tc)),
+ (WHILELO_PXX_D $index, $tc)>;
+
// Move element from the bottom 128-bits of a scalable vector to a single-element vector.
// Alternative case where insertelement is just scalar_to_vector rather than vector_insert.
def : Pat<(v1f64 (scalar_to_vector
>From 1cf5105ddce67b21fe2f602b5f94d3147c9bbc93 Mon Sep 17 00:00:00 2001
From: Kerry McLaughlin <kerry.mclaughlin at arm.com>
Date: Thu, 8 May 2025 15:42:28 +0000
Subject: [PATCH 2/3] - Lower whilelo intrinsic to GET_ACTIVE_LANE_MASK in
performIntrinsicCombine - Remove the additional tablegen patterns matching
get_active_lane_mask and reuse those for the whilelo intrinsic
---
.../Target/AArch64/AArch64ISelLowering.cpp | 7 ++---
.../lib/Target/AArch64/AArch64SVEInstrInfo.td | 26 +++----------------
2 files changed, 8 insertions(+), 25 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 0b10882e45924..daf34e72c02e2 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -290,6 +290,7 @@ static bool isZeroingInactiveLanes(SDValue Op) {
return false;
// We guarantee i1 splat_vectors to zero the other lanes
case ISD::SPLAT_VECTOR:
+ case ISD::GET_ACTIVE_LANE_MASK:
case AArch64ISD::PTRUE:
case AArch64ISD::SETCC_MERGE_ZERO:
return true;
@@ -6231,9 +6232,6 @@ SDValue AArch64TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
DAG.getNode(
AArch64ISD::URSHR_I, dl, Op.getOperand(1).getValueType(), Op.getOperand(1), Op.getOperand(2)));
return SDValue();
- case Intrinsic::aarch64_sve_whilelo:
- return optimizeIncrementingWhile(Op.getNode(), DAG, /*IsSigned=*/false,
- /*IsEqual=*/false);
case Intrinsic::aarch64_sve_whilelt:
return optimizeIncrementingWhile(Op.getNode(), DAG, /*IsSigned=*/true,
/*IsEqual=*/false);
@@ -22304,6 +22302,9 @@ static SDValue performIntrinsicCombine(SDNode *N,
case Intrinsic::aarch64_sve_ptest_last:
return getPTest(DAG, N->getValueType(0), N->getOperand(1), N->getOperand(2),
AArch64CC::LAST_ACTIVE);
+ case Intrinsic::aarch64_sve_whilelo:
+ return DAG.getNode(ISD::GET_ACTIVE_LANE_MASK, SDLoc(N), N->getValueType(0),
+ N->getOperand(1), N->getOperand(2));
}
return SDValue();
}
diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
index b9dd599bdb0ce..9e77e8b032feb 100644
--- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
@@ -2135,12 +2135,12 @@ let Predicates = [HasSVE_or_SME] in {
defm WHILELT_PWW : sve_int_while4_rr<0b010, "whilelt", int_aarch64_sve_whilelt, int_aarch64_sve_whilegt>;
defm WHILELE_PWW : sve_int_while4_rr<0b011, "whilele", int_aarch64_sve_whilele, null_frag>;
- defm WHILELO_PWW : sve_int_while4_rr<0b110, "whilelo", int_aarch64_sve_whilelo, int_aarch64_sve_whilehi>;
+ defm WHILELO_PWW : sve_int_while4_rr<0b110, "whilelo", get_active_lane_mask, int_aarch64_sve_whilehi>;
defm WHILELS_PWW : sve_int_while4_rr<0b111, "whilels", int_aarch64_sve_whilels, null_frag>;
defm WHILELT_PXX : sve_int_while8_rr<0b010, "whilelt", int_aarch64_sve_whilelt, int_aarch64_sve_whilegt>;
defm WHILELE_PXX : sve_int_while8_rr<0b011, "whilele", int_aarch64_sve_whilele, null_frag>;
- defm WHILELO_PXX : sve_int_while8_rr<0b110, "whilelo", int_aarch64_sve_whilelo, int_aarch64_sve_whilehi>;
+ defm WHILELO_PXX : sve_int_while8_rr<0b110, "whilelo", get_active_lane_mask, int_aarch64_sve_whilehi>;
defm WHILELS_PXX : sve_int_while8_rr<0b111, "whilels", int_aarch64_sve_whilels, null_frag>;
def CTERMEQ_WW : sve_int_cterm<0b0, 0b0, "ctermeq", GPR32>;
@@ -3449,24 +3449,6 @@ let Predicates = [HasSVE_or_SME] in {
def : Pat<(i64(find_last_active nxv2i1:$P1)), (LASTB_RPZ_D $P1, (INDEX_II_D 0,
1))>;
- def : Pat<(nxv16i1(get_active_lane_mask i32:$index, i32:$tc)),
- (WHILELO_PWW_B $index, $tc)>;
- def : Pat<(nxv8i1(get_active_lane_mask i32:$index, i32:$tc)),
- (WHILELO_PWW_H $index, $tc)>;
- def : Pat<(nxv4i1(get_active_lane_mask i32:$index, i32:$tc)),
- (WHILELO_PWW_S $index, $tc)>;
- def : Pat<(nxv2i1(get_active_lane_mask i32:$index, i32:$tc)),
- (WHILELO_PWW_D $index, $tc)>;
-
- def : Pat<(nxv16i1(get_active_lane_mask i64:$index, i64:$tc)),
- (WHILELO_PXX_B $index, $tc)>;
- def : Pat<(nxv8i1(get_active_lane_mask i64:$index, i64:$tc)),
- (WHILELO_PXX_H $index, $tc)>;
- def : Pat<(nxv4i1(get_active_lane_mask i64:$index, i64:$tc)),
- (WHILELO_PXX_S $index, $tc)>;
- def : Pat<(nxv2i1(get_active_lane_mask i64:$index, i64:$tc)),
- (WHILELO_PXX_D $index, $tc)>;
-
// Move element from the bottom 128-bits of a scalable vector to a single-element vector.
// Alternative case where insertelement is just scalar_to_vector rather than vector_insert.
def : Pat<(v1f64 (scalar_to_vector
@@ -4010,12 +3992,12 @@ let Predicates = [HasSVE2_or_SME] in {
defm WHILEGE_PWW : sve_int_while4_rr<0b000, "whilege", int_aarch64_sve_whilege, null_frag>;
defm WHILEGT_PWW : sve_int_while4_rr<0b001, "whilegt", int_aarch64_sve_whilegt, int_aarch64_sve_whilelt>;
defm WHILEHS_PWW : sve_int_while4_rr<0b100, "whilehs", int_aarch64_sve_whilehs, null_frag>;
- defm WHILEHI_PWW : sve_int_while4_rr<0b101, "whilehi", int_aarch64_sve_whilehi, int_aarch64_sve_whilelo>;
+ defm WHILEHI_PWW : sve_int_while4_rr<0b101, "whilehi", int_aarch64_sve_whilehi, get_active_lane_mask>;
defm WHILEGE_PXX : sve_int_while8_rr<0b000, "whilege", int_aarch64_sve_whilege, null_frag>;
defm WHILEGT_PXX : sve_int_while8_rr<0b001, "whilegt", int_aarch64_sve_whilegt, int_aarch64_sve_whilelt>;
defm WHILEHS_PXX : sve_int_while8_rr<0b100, "whilehs", int_aarch64_sve_whilehs, null_frag>;
- defm WHILEHI_PXX : sve_int_while8_rr<0b101, "whilehi", int_aarch64_sve_whilehi, int_aarch64_sve_whilelo>;
+ defm WHILEHI_PXX : sve_int_while8_rr<0b101, "whilehi", int_aarch64_sve_whilehi, get_active_lane_mask>;
// SVE2 pointer conflict compare
defm WHILEWR_PXX : sve2_int_while_rr<0b0, "whilewr", "int_aarch64_sve_whilewr">;
>From d20d8914b71d7e2f1d367575584b859228599abd Mon Sep 17 00:00:00 2001
From: Kerry McLaughlin <kerry.mclaughlin at arm.com>
Date: Tue, 13 May 2025 16:20:54 +0000
Subject: [PATCH 3/3] - Move isBeforeLegalize before call to
optimizeIncrementingWhile - Add isSVEorStreamingSVEAvailable assert to
LowerGET_ACTIVE_LANE_MASK - Added more details to the comment in ISDOpcodes.h
---
llvm/include/llvm/CodeGen/ISDOpcodes.h | 11 +++++++----
llvm/lib/Target/AArch64/AArch64ISelLowering.cpp | 15 +++++++++------
2 files changed, 16 insertions(+), 10 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/ISDOpcodes.h b/llvm/include/llvm/CodeGen/ISDOpcodes.h
index ba29c118007dd..65da3a9d5746e 100644
--- a/llvm/include/llvm/CodeGen/ISDOpcodes.h
+++ b/llvm/include/llvm/CodeGen/ISDOpcodes.h
@@ -1524,10 +1524,13 @@ enum NodeType {
// Operands: Mask
VECTOR_FIND_LAST_ACTIVE,
- // Creates a mask representing active and inactive
- // vector lanes, active while base < trip count.
- // Operands: Base, Trip Count
- // Output: Mask
+ // GET_ACTIVE_LANE_MASK - this corrosponds to the llvm.get.active.lane.mask
+ // intrinsic. It creates a mask representing active and inactive vector
+ // lanes, active while Base + index < Trip Count. As with the intrinsic,
+ // the operands Base and Trip Count have the same scalar integer type and
+ // the internal addition of Base + index cannot overflow. However, the ISD
+ // node supports result types which are wider than i1, where the high
+ // bits conform to getBooleanContents similar to the SETCC operator.
GET_ACTIVE_LANE_MASK,
// llvm.clear_cache intrinsic
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index daf34e72c02e2..5b26ca52623a0 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -18133,13 +18133,13 @@ static SDValue performVecReduceAddCombineWithUADDLP(SDNode *N,
static SDValue
performActiveLaneMaskCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,
const AArch64Subtarget *ST) {
+ if (DCI.isBeforeLegalize())
+ return SDValue();
+
if (SDValue While = optimizeIncrementingWhile(N, DCI.DAG, /*IsSigned=*/false,
/*IsEqual=*/false))
return While;
- if (DCI.isBeforeLegalize())
- return SDValue();
-
if (!ST->hasSVE2p1())
return SDValue();
@@ -29515,9 +29515,12 @@ AArch64TargetLowering::LowerGET_ACTIVE_LANE_MASK(SDValue Op,
EVT VT = Op.getValueType();
assert(VT.isFixedLengthVector() && "Expected fixed length vector type!");
- // We can use the SVE whilelo instruction to lower this intrinsic by
- // creating the appropriate sequence of scalable vector operations and
- // then extracting a fixed-width subvector from the scalable vector.
+ auto &Subtarget = DAG.getSubtarget<AArch64Subtarget>();
+ assert(Subtarget.isSVEorStreamingSVEAvailable() &&
+ "Lowering fixed length get_active_lane_mask requires SVE!");
+
+ // There are no dedicated fixed-length instructions for GET_ACTIVE_LANE_MASK,
+ // but we can use SVE when available.
SDLoc DL(Op);
EVT ContainerVT = getContainerForFixedLengthVector(DAG, VT);
More information about the llvm-commits
mailing list