[llvm] [AArch64] Convert CLS intrinsics to use ISD::CTLS (PR #178885)
via llvm-commits
llvm-commits at lists.llvm.org
Fri Jan 30 05:22:43 PST 2026
llvmbot wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-llvm-selectiondag
Author: Hamza Hassanain (HamzaHassanain)
<details>
<summary>Changes</summary>
This patch converts AArch64 CLS intrinsics (aarch64_neon_cls) to use the generic ISD::CTLS node.
- aarch64_neon_cls: Lowered to ISD::CTLS, pattern-matched to CLS instruction
- Set ISD::CTLS as Legal for NEON vector types (v8i8, v16i8, v4i16, v8i16, v2i32, v4i32)
Also adds generic CTLS expansion support:
- ExpandIntRes_CTLS in LegalizeIntegerTypes for i64->i32 type expansion
- expandCTLS in TargetLowering for targets without native CLS instruction
Part of: https://github.com/llvm/llvm-project/issues/174337
---
Full diff: https://github.com/llvm/llvm-project/pull/178885.diff
8 Files Affected:
- (modified) llvm/include/llvm/CodeGen/TargetLowering.h (+6)
- (modified) llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp (+4)
- (modified) llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp (+29)
- (modified) llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h (+1)
- (modified) llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp (+18)
- (modified) llvm/lib/Target/AArch64/AArch64ISelLowering.cpp (+9)
- (modified) llvm/lib/Target/AArch64/AArch64InstrInfo.td (+1-1)
- (added) llvm/test/CodeGen/AArch64/neon-cls.ll (+88)
``````````diff
diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h
index df0fced112f73..f4010848aa8c4 100644
--- a/llvm/include/llvm/CodeGen/TargetLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetLowering.h
@@ -5583,6 +5583,12 @@ class LLVM_ABI TargetLowering : public TargetLoweringBase {
/// \returns The expansion result or SDValue() if it fails.
SDValue expandCTLZ(SDNode *N, SelectionDAG &DAG) const;
+ /// Expand CTLS (count leading sign bits) nodes.
+ /// CTLS(x) = CTLZ(OR(SHL(XOR(x, SRA(x, BW-1)), 1), 1))
+ /// \param N Node to expand
+ /// \returns The expansion result or SDValue() if it fails.
+ SDValue expandCTLS(SDNode *N, SelectionDAG &DAG) const;
+
/// Expand VP_CTLZ/VP_CTLZ_ZERO_UNDEF nodes.
/// \param N Node to expand
/// \returns The expansion result or SDValue() if it fails.
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
index d9a2409b35e4c..df3ee52b42638 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
@@ -3218,6 +3218,10 @@ bool SelectionDAGLegalize::ExpandNode(SDNode *Node) {
if ((Tmp1 = TLI.expandCTLZ(Node, DAG)))
Results.push_back(Tmp1);
break;
+ case ISD::CTLS:
+ if ((Tmp1 = TLI.expandCTLS(Node, DAG)))
+ Results.push_back(Tmp1);
+ break;
case ISD::CTTZ:
case ISD::CTTZ_ZERO_UNDEF:
if ((Tmp1 = TLI.expandCTTZ(Node, DAG)))
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
index 8ce41df6be69b..586cb6878869d 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
@@ -3066,6 +3066,9 @@ void DAGTypeLegalizer::ExpandIntegerResult(SDNode *N, unsigned ResNo) {
case ISD::ABDU: ExpandIntRes_ABD(N, Lo, Hi); break;
case ISD::CTLZ_ZERO_UNDEF:
case ISD::CTLZ: ExpandIntRes_CTLZ(N, Lo, Hi); break;
+ case ISD::CTLS:
+ ExpandIntRes_CTLS(N, Lo, Hi);
+ break;
case ISD::CTPOP: ExpandIntRes_CTPOP(N, Lo, Hi); break;
case ISD::CTTZ_ZERO_UNDEF:
case ISD::CTTZ: ExpandIntRes_CTTZ(N, Lo, Hi); break;
@@ -4150,6 +4153,32 @@ void DAGTypeLegalizer::ExpandIntRes_CTLZ(SDNode *N,
Hi = DAG.getConstant(0, dl, NVT);
}
+void DAGTypeLegalizer::ExpandIntRes_CTLS(SDNode *N, SDValue &Lo, SDValue &Hi) {
+ SDLoc dl(N);
+ // ctls(HiLo) -> if (IsAllSignBits = (ctls(Hi) == BW-1)) then
+ // BW-1 + clz(IsNegative = (Hi < 0) ? ~Lo : Lo)
+ // else ctls(Hi)
+ GetExpandedInteger(N->getOperand(0), Lo, Hi);
+ EVT NVT = Lo.getValueType();
+ unsigned NVTBits = NVT.getSizeInBits();
+
+ SDValue Constant0 = DAG.getConstant(0, dl, NVT);
+ SDValue ConstantBWM1 = DAG.getConstant(NVTBits - 1, dl, NVT);
+
+ SDValue HiCTLS = DAG.getNode(ISD::CTLS, dl, NVT, Hi);
+ SDValue IsAllSignBits = DAG.getSetCC(dl, getSetCCResultType(NVT), HiCTLS,
+ ConstantBWM1, ISD::SETEQ);
+ SDValue IsNegative =
+ DAG.getSetCC(dl, getSetCCResultType(NVT), Hi, Constant0, ISD::SETLT);
+ SDValue AdjustedLo =
+ DAG.getSelect(dl, NVT, IsNegative, DAG.getNOT(dl, Lo, NVT), Lo);
+ SDValue LoCLZ = DAG.getNode(ISD::CTLZ, dl, NVT, AdjustedLo);
+ Lo = DAG.getSelect(dl, NVT, IsAllSignBits,
+ DAG.getNode(ISD::ADD, dl, NVT, LoCLZ, ConstantBWM1),
+ HiCTLS);
+ Hi = DAG.getConstant(0, dl, NVT);
+}
+
void DAGTypeLegalizer::ExpandIntRes_ABD(SDNode *N, SDValue &Lo, SDValue &Hi) {
SDValue Result = TLI.expandABD(N, DAG);
SplitInteger(Result, Lo, Hi);
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
index a39e419e5ad1c..b32b53ef9efe0 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
@@ -466,6 +466,7 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
void ExpandIntRes_ABS (SDNode *N, SDValue &Lo, SDValue &Hi);
void ExpandIntRes_ABD (SDNode *N, SDValue &Lo, SDValue &Hi);
void ExpandIntRes_CTLZ (SDNode *N, SDValue &Lo, SDValue &Hi);
+ void ExpandIntRes_CTLS(SDNode *N, SDValue &Lo, SDValue &Hi);
void ExpandIntRes_CTPOP (SDNode *N, SDValue &Lo, SDValue &Hi);
void ExpandIntRes_CTTZ (SDNode *N, SDValue &Lo, SDValue &Hi);
void ExpandIntRes_LOAD (LoadSDNode *N, SDValue &Lo, SDValue &Hi);
diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
index edba0a7169c0a..c954ec2405ce6 100644
--- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
@@ -9563,6 +9563,24 @@ SDValue TargetLowering::expandCTLZ(SDNode *Node, SelectionDAG &DAG) const {
return DAG.getNode(ISD::CTPOP, dl, VT, Op);
}
+SDValue TargetLowering::expandCTLS(SDNode *Node, SelectionDAG &DAG) const {
+ SDLoc dl(Node);
+ EVT VT = Node->getValueType(0);
+ EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout());
+ SDValue Op = Node->getOperand(0);
+ unsigned NumBitsPerElt = VT.getScalarSizeInBits();
+
+ // CTLS(x) = CTLZ(OR(SHL(XOR(x, SRA(x, BW-1)), 1), 1))
+ // This transforms the sign bits into leading zeros that can be counted.
+ SDValue ShiftAmt = DAG.getConstant(NumBitsPerElt - 1, dl, ShVT);
+ SDValue One = DAG.getConstant(1, dl, VT);
+ SDValue SignBit = DAG.getNode(ISD::SRA, dl, VT, Op, ShiftAmt);
+ SDValue Xor = DAG.getNode(ISD::XOR, dl, VT, Op, SignBit);
+ SDValue Shl = DAG.getNode(ISD::SHL, dl, VT, Xor, One);
+ SDValue Or = DAG.getNode(ISD::OR, dl, VT, Shl, One);
+ return DAG.getNode(ISD::CTLZ, dl, VT, Or);
+}
+
SDValue TargetLowering::expandVPCTLZ(SDNode *Node, SelectionDAG &DAG) const {
SDLoc dl(Node);
EVT VT = Node->getValueType(0);
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 6c0544005e1dd..3c13474518fe6 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -1320,6 +1320,11 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
setOperationAction(ISD::CTLZ, MVT::v1i64, Expand);
setOperationAction(ISD::CTLZ, MVT::v2i64, Expand);
+ // CTLS (Count Leading Sign bits) - Legal for BHS types (8/16/32-bit
+ // elements) No hardware support for 64-bit element vectors
+ for (auto VT : {MVT::v8i8, MVT::v16i8, MVT::v4i16, MVT::v8i16, MVT::v2i32,
+ MVT::v4i32})
+ setOperationAction(ISD::CTLS, VT, Legal);
setOperationAction(ISD::BITREVERSE, MVT::v8i8, Legal);
setOperationAction(ISD::BITREVERSE, MVT::v16i8, Legal);
setOperationAction(ISD::BITREVERSE, MVT::v2i32, Custom);
@@ -6924,6 +6929,10 @@ SDValue AArch64TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
Op.getOperand(1));
return DAG.getNode(ISD::TRUNCATE, DL, Op.getValueType(), Res);
}
+ case Intrinsic::aarch64_neon_cls: {
+ // Lower NEON CLS intrinsic to ISD::CTLS
+ return DAG.getNode(ISD::CTLS, DL, Op.getValueType(), Op.getOperand(1));
+ }
case Intrinsic::aarch64_sve_pmul:
case Intrinsic::aarch64_neon_pmul:
return DAG.getNode(ISD::CLMUL, DL, Op.getValueType(), Op.getOperand(1),
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
index 96d45d4d48652..d6b18aa875dfe 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
@@ -5747,7 +5747,7 @@ let Predicates = [HasFullFP16] in {
//===----------------------------------------------------------------------===//
defm ABS : SIMDTwoVectorBHSD<0, 0b01011, "abs", abs>;
-defm CLS : SIMDTwoVectorBHS<0, 0b00100, "cls", int_aarch64_neon_cls>;
+defm CLS : SIMDTwoVectorBHS<0, 0b00100, "cls", ctls>;
defm CLZ : SIMDTwoVectorBHS<1, 0b00100, "clz", ctlz>;
defm CMEQ : SIMDCmpTwoVector<0, 0b01001, "cmeq", AArch64cmeqz>;
defm CMGE : SIMDCmpTwoVector<1, 0b01000, "cmge", AArch64cmgez>;
diff --git a/llvm/test/CodeGen/AArch64/neon-cls.ll b/llvm/test/CodeGen/AArch64/neon-cls.ll
new file mode 100644
index 0000000000000..c32349773fb27
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/neon-cls.ll
@@ -0,0 +1,88 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=aarch64-eabi | FileCheck %s
+
+; Test AArch64 NEON vector CLS (Count Leading Sign bits) operations
+; The intrinsics are lowered to ISD::CTLS and selected to CLS instructions
+
+define <8 x i8> @test_cls_v8i8(<8 x i8> %a) nounwind {
+; CHECK-LABEL: test_cls_v8i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: cls v0.8b, v0.8b
+; CHECK-NEXT: ret
+ %result = call <8 x i8> @llvm.aarch64.neon.cls.v8i8(<8 x i8> %a)
+ ret <8 x i8> %result
+}
+
+define <16 x i8> @test_cls_v16i8(<16 x i8> %a) nounwind {
+; CHECK-LABEL: test_cls_v16i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: cls v0.16b, v0.16b
+; CHECK-NEXT: ret
+ %result = call <16 x i8> @llvm.aarch64.neon.cls.v16i8(<16 x i8> %a)
+ ret <16 x i8> %result
+}
+
+define <4 x i16> @test_cls_v4i16(<4 x i16> %a) nounwind {
+; CHECK-LABEL: test_cls_v4i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: cls v0.4h, v0.4h
+; CHECK-NEXT: ret
+ %result = call <4 x i16> @llvm.aarch64.neon.cls.v4i16(<4 x i16> %a)
+ ret <4 x i16> %result
+}
+
+define <8 x i16> @test_cls_v8i16(<8 x i16> %a) nounwind {
+; CHECK-LABEL: test_cls_v8i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: cls v0.8h, v0.8h
+; CHECK-NEXT: ret
+ %result = call <8 x i16> @llvm.aarch64.neon.cls.v8i16(<8 x i16> %a)
+ ret <8 x i16> %result
+}
+
+define <2 x i32> @test_cls_v2i32(<2 x i32> %a) nounwind {
+; CHECK-LABEL: test_cls_v2i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: cls v0.2s, v0.2s
+; CHECK-NEXT: ret
+ %result = call <2 x i32> @llvm.aarch64.neon.cls.v2i32(<2 x i32> %a)
+ ret <2 x i32> %result
+}
+
+define <4 x i32> @test_cls_v4i32(<4 x i32> %a) nounwind {
+; CHECK-LABEL: test_cls_v4i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: cls v0.4s, v0.4s
+; CHECK-NEXT: ret
+ %result = call <4 x i32> @llvm.aarch64.neon.cls.v4i32(<4 x i32> %a)
+ ret <4 x i32> %result
+}
+
+; Test scalar CLS operations (i32 and i64)
+define i32 @test_cls_i32(i32 %a) nounwind {
+; CHECK-LABEL: test_cls_i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: cls w0, w0
+; CHECK-NEXT: ret
+ %result = call i32 @llvm.aarch64.cls(i32 %a)
+ ret i32 %result
+}
+
+define i32 @test_cls_i64(i64 %a) nounwind {
+; CHECK-LABEL: test_cls_i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: cls x0, x0
+; CHECK-NEXT: // kill: def $w0 killed $w0 killed $x0
+; CHECK-NEXT: ret
+ %result = call i32 @llvm.aarch64.cls64(i64 %a)
+ ret i32 %result
+}
+
+declare <8 x i8> @llvm.aarch64.neon.cls.v8i8(<8 x i8>) nounwind readnone
+declare <16 x i8> @llvm.aarch64.neon.cls.v16i8(<16 x i8>) nounwind readnone
+declare <4 x i16> @llvm.aarch64.neon.cls.v4i16(<4 x i16>) nounwind readnone
+declare <8 x i16> @llvm.aarch64.neon.cls.v8i16(<8 x i16>) nounwind readnone
+declare <2 x i32> @llvm.aarch64.neon.cls.v2i32(<2 x i32>) nounwind readnone
+declare <4 x i32> @llvm.aarch64.neon.cls.v4i32(<4 x i32>) nounwind readnone
+declare i32 @llvm.aarch64.cls(i32) nounwind readnone
+declare i32 @llvm.aarch64.cls64(i64) nounwind readnone
``````````
</details>
https://github.com/llvm/llvm-project/pull/178885
More information about the llvm-commits
mailing list