[llvm] [SelectionDAG] Add CTTZ_ELTS[_ZERO_POISON] nodes. NFCI (PR #185600)
Luke Lau via llvm-commits
llvm-commits at lists.llvm.org
Tue Mar 10 03:54:17 PDT 2026
https://github.com/lukel97 updated https://github.com/llvm/llvm-project/pull/185600
>From 0200c26ce3c285e008068c06db98436dbd778f6b Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Tue, 10 Mar 2026 17:00:12 +0800
Subject: [PATCH 1/3] [SDAG] Add CTTZ_ELTS[_ZERO_POISON] nodes. NFCI
Currently llvm.experimental.cttz.elts are directly lowered from the intrinsic.
If the type isn't legal then the target tells SelectionDAGBuilder to expand it into a reduction, but this means we can't split the operation.
E.g. it's possible to split a cttz.elts nxv32i1 into two nxv16i1, instead of expanding it into a nxv32i64 reduction.
vp.cttz.elts can be split because it has a dedicated SelectionDAG node.
This adds CTTZ_ELTS and CTTZ_ELTS[_ZERO_POISON] nodes and just enough legalization to get tests passing. A follow up patch will add splitting and move the expansion into LegalizeDAG.
test/CodeGen/RISCV/rvv/cttz-elts.ll has a test that requires a i64 result to be expanded to i32 on RV32. I don't think expanding is trivial, but because the test doesn't specify a vscale_range the result is undef per the lang ref since the result type can't fit the maximum number of elements in the vector.
---
llvm/include/llvm/CodeGen/ISDOpcodes.h | 5 +++
llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp | 2 +
.../SelectionDAG/LegalizeIntegerTypes.cpp | 29 ++++++++++++
.../SelectionDAG/SelectionDAGBuilder.cpp | 11 +++--
.../SelectionDAG/SelectionDAGDumper.cpp | 5 +++
llvm/lib/CodeGen/TargetLoweringBase.cpp | 4 ++
.../Target/AArch64/AArch64ISelLowering.cpp | 45 +++++++++++--------
llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 34 +++++++-------
llvm/lib/Target/RISCV/RISCVISelLowering.h | 1 +
llvm/test/CodeGen/RISCV/rvv/cttz-elts.ll | 33 ++++++++++----
10 files changed, 121 insertions(+), 48 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/ISDOpcodes.h b/llvm/include/llvm/CodeGen/ISDOpcodes.h
index a846aad90bc2b..67261a715968c 100644
--- a/llvm/include/llvm/CodeGen/ISDOpcodes.h
+++ b/llvm/include/llvm/CodeGen/ISDOpcodes.h
@@ -1577,6 +1577,11 @@ enum NodeType {
/// Output: Output Chain
EXPERIMENTAL_VECTOR_HISTOGRAM,
+ /// The `llvm.experimental.cttz.elts` intrinsic. Has a single i1 vector
+ /// operand.
+ CTTZ_ELTS,
+ CTTZ_ELTS_ZERO_POISON,
+
/// Finds the index of the last active mask element
/// Operands: Mask
VECTOR_FIND_LAST_ACTIVE,
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
index 08606c99097ae..5e54343f7f146 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
@@ -1267,6 +1267,8 @@ void SelectionDAGLegalize::LegalizeOp(SDNode *Node) {
Action = TLI.getOperationAction(
Node->getOpcode(), Node->getOperand(1).getValueType());
break;
+ case ISD::CTTZ_ELTS:
+ case ISD::CTTZ_ELTS_ZERO_POISON:
case ISD::VP_CTTZ_ELTS:
case ISD::VP_CTTZ_ELTS_ZERO_UNDEF:
Action = TLI.getOperationAction(Node->getOpcode(),
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
index 85eb59e5449e4..88c45729ecb55 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
@@ -19,6 +19,7 @@
#include "LegalizeTypes.h"
#include "llvm/Analysis/TargetLibraryInfo.h"
+#include "llvm/Analysis/ValueTracking.h"
#include "llvm/CodeGen/StackMaps.h"
#include "llvm/CodeGen/TargetLowering.h"
#include "llvm/IR/DerivedTypes.h"
@@ -78,6 +79,8 @@ void DAGTypeLegalizer::PromoteIntegerResult(SDNode *N, unsigned ResNo) {
case ISD::VP_CTTZ:
case ISD::CTTZ_ZERO_UNDEF:
case ISD::CTTZ: Res = PromoteIntRes_CTTZ(N); break;
+ case ISD::CTTZ_ELTS_ZERO_POISON:
+ case ISD::CTTZ_ELTS:
case ISD::VP_CTTZ_ELTS_ZERO_UNDEF:
case ISD::VP_CTTZ_ELTS:
Res = PromoteIntRes_VP_CttzElements(N);
@@ -3240,6 +3243,32 @@ void DAGTypeLegalizer::ExpandIntegerResult(SDNode *N, unsigned ResNo) {
case ISD::READ_REGISTER:
ExpandIntRes_READ_REGISTER(N, Lo, Hi);
break;
+
+ case ISD::CTTZ_ELTS:
+ case ISD::CTTZ_ELTS_ZERO_POISON: {
+ EVT VT = N->getSimpleValueType(0);
+ EVT HalfVT = VT.getHalfSizedIntegerVT(*DAG.getContext());
+
+ EVT OpVT = N->getOperand(0).getValueType();
+ ConstantRange CR(64, OpVT.getVectorMinNumElements());
+ const Function &Fn = DAG.getMachineFunction().getFunction();
+ if (OpVT.isScalableVector())
+ CR = CR.umul_sat(getVScaleRange(&Fn, 64));
+ if (N->getOpcode() == ISD::CTTZ_ELTS_ZERO_POISON)
+ CR = CR.subtract(APInt(64, 1));
+
+ // See if the half VT is large enough to fit the result into, or
+ // alternatively if there's no upper bound on vscale in which case the
+ // result is undefined.
+ if (!(CR.getUnsignedMax().getActiveBits() > HalfVT.getScalarSizeInBits() ||
+ CR.isUpperWrapped()))
+ break;
+
+ SDValue HalfOp =
+ DAG.getNode(N->getOpcode(), SDLoc(N), HalfVT, N->getOperand(0));
+ HalfOp = DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N), VT, HalfOp);
+ SplitInteger(HalfOp, Lo, Hi);
+ }
}
// If Lo/Hi is null, the sub-method took care of registering results etc.
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index 2f44ad5a2d108..b3122ae59ce48 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -8317,9 +8317,15 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
auto DL = getCurSDLoc();
SDValue Op = getValue(I.getOperand(0));
EVT OpVT = Op.getValueType();
+ EVT RetTy = TLI.getValueType(DAG.getDataLayout(), I.getType());
+ bool ZeroIsPoison =
+ !cast<ConstantSDNode>(getValue(I.getOperand(1)))->isZero();
if (!TLI.shouldExpandCttzElements(OpVT)) {
- visitTargetIntrinsic(I, Intrinsic);
+ SDValue Ret = DAG.getNode(ZeroIsPoison ? ISD::CTTZ_ELTS_ZERO_POISON
+ : ISD::CTTZ_ELTS,
+ sdl, RetTy, Op);
+ setValue(&I, Ret);
return;
}
@@ -8333,8 +8339,6 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
// If the zero-is-poison flag is set, we can assume the upper limit
// of the result is VF-1.
- bool ZeroIsPoison =
- !cast<ConstantSDNode>(getValue(I.getOperand(1)))->isZero();
ConstantRange VScaleRange(1, true); // Dummy value.
if (isa<ScalableVectorType>(I.getOperand(0)->getType()))
VScaleRange = getVScaleRange(I.getCaller(), 64);
@@ -8358,7 +8362,6 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
SDValue Max = DAG.getNode(ISD::VECREDUCE_UMAX, DL, NewEltTy, And);
SDValue Sub = DAG.getNode(ISD::SUB, DL, NewEltTy, VL, Max);
- EVT RetTy = TLI.getValueType(DAG.getDataLayout(), I.getType());
SDValue Ret = DAG.getZExtOrTrunc(Sub, DL, RetTy);
setValue(&I, Ret);
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
index 571830cc57b52..7161dd299f830 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
@@ -591,6 +591,11 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const {
case ISD::EXPERIMENTAL_VECTOR_HISTOGRAM:
return "histogram";
+ case ISD::CTTZ_ELTS:
+ return "cttz_elts";
+ case ISD::CTTZ_ELTS_ZERO_POISON:
+ return "cttz_elts_zero_poison";
+
case ISD::VECTOR_FIND_LAST_ACTIVE:
return "find_last_active";
diff --git a/llvm/lib/CodeGen/TargetLoweringBase.cpp b/llvm/lib/CodeGen/TargetLoweringBase.cpp
index 684ba3161f48c..4bc33dd73cacb 100644
--- a/llvm/lib/CodeGen/TargetLoweringBase.cpp
+++ b/llvm/lib/CodeGen/TargetLoweringBase.cpp
@@ -1231,6 +1231,10 @@ void TargetLoweringBase::initActions() {
// Only some target support this vector operation. Most need to expand it.
setOperationAction(ISD::VECTOR_COMPRESS, VT, Expand);
+ // cttz.elts defaults to expand.
+ setOperationAction({ISD::CTTZ_ELTS, ISD::CTTZ_ELTS_ZERO_POISON}, VT,
+ Expand);
+
// VP operations default to expand.
#define BEGIN_REGISTER_VP_SDNODE(SDOPC, ...) \
setOperationAction(ISD::SDOPC, VT, Expand);
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index dc5a3736ecaa1..273887058d377 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -1561,6 +1561,8 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
setOperationAction(ISD::VECTOR_INTERLEAVE, VT, Custom);
}
for (auto VT : {MVT::nxv16i1, MVT::nxv8i1, MVT::nxv4i1, MVT::nxv2i1}) {
+ setOperationAction({ISD::CTTZ_ELTS, ISD::CTTZ_ELTS_ZERO_POISON}, VT,
+ Custom);
setOperationAction(ISD::VECTOR_FIND_LAST_ACTIVE, VT, Legal);
setOperationAction(ISD::GET_ACTIVE_LANE_MASK, VT, Legal);
}
@@ -1994,7 +1996,9 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
setOperationAction(ISD::VSCALE, MVT::i32, Custom);
for (auto VT : {MVT::v16i1, MVT::v8i1, MVT::v4i1, MVT::v2i1})
- setOperationAction(ISD::INTRINSIC_WO_CHAIN, VT, Custom);
+ setOperationAction(
+ {ISD::INTRINSIC_WO_CHAIN, ISD::CTTZ_ELTS, ISD::CTTZ_ELTS_ZERO_POISON},
+ VT, Custom);
}
// Handle partial reduction operations
@@ -7002,24 +7006,6 @@ SDValue AArch64TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
ADDLV, DAG.getConstant(0, DL, MVT::i64));
return EXTRACT_VEC_ELT;
}
- case Intrinsic::experimental_cttz_elts: {
- SDValue CttzOp = Op.getOperand(1);
- EVT VT = CttzOp.getValueType();
- assert(VT.getVectorElementType() == MVT::i1 && "Expected MVT::i1");
-
- if (VT.isFixedLengthVector()) {
- // We can use SVE instructions to lower this intrinsic by first creating
- // an SVE predicate register mask from the fixed-width vector.
- VT = getTypeToTransformTo(*DAG.getContext(), VT);
- SDValue Mask = DAG.getNode(ISD::SIGN_EXTEND, DL, VT, CttzOp);
- CttzOp = convertFixedMaskToScalableVector(Mask, DAG);
- }
-
- SDValue Pg = getPredicateForVector(DAG, DL, VT);
- SDValue NewCttzElts =
- DAG.getNode(AArch64ISD::CTTZ_ELTS, DL, MVT::i64, Pg, CttzOp);
- return DAG.getZExtOrTrunc(NewCttzElts, DL, Op.getValueType());
- }
case Intrinsic::experimental_vector_match: {
return LowerVectorMatch(Op, DAG);
}
@@ -8478,6 +8464,27 @@ SDValue AArch64TargetLowering::LowerOperation(SDValue Op,
return LowerPARTIAL_REDUCE_MLA(Op, DAG);
case ISD::CLMUL:
return LowerCLMUL(Op, DAG);
+
+ case ISD::CTTZ_ELTS:
+ case ISD::CTTZ_ELTS_ZERO_POISON: {
+ SDLoc DL(Op);
+ SDValue CttzOp = Op.getOperand(0);
+ EVT VT = CttzOp.getValueType();
+ assert(VT.getVectorElementType() == MVT::i1 && "Expected MVT::i1");
+
+ if (VT.isFixedLengthVector()) {
+ // We can use SVE instructions to lower this intrinsic by first creating
+ // an SVE predicate register mask from the fixed-width vector.
+ VT = getTypeToTransformTo(*DAG.getContext(), VT);
+ SDValue Mask = DAG.getNode(ISD::SIGN_EXTEND, DL, VT, CttzOp);
+ CttzOp = convertFixedMaskToScalableVector(Mask, DAG);
+ }
+
+ SDValue Pg = getPredicateForVector(DAG, DL, VT);
+ SDValue NewCttzElts =
+ DAG.getNode(AArch64ISD::CTTZ_ELTS, DL, MVT::i64, Pg, CttzOp);
+ return DAG.getZExtOrTrunc(NewCttzElts, DL, Op.getValueType());
+ }
}
}
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index e766b3d10ccb3..d71aa90b41bfc 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -943,8 +943,9 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
Expand);
setOperationAction(ISD::VP_MERGE, VT, Custom);
- setOperationAction({ISD::VP_CTTZ_ELTS, ISD::VP_CTTZ_ELTS_ZERO_UNDEF}, VT,
- Custom);
+ setOperationAction({ISD::CTTZ_ELTS, ISD::CTTZ_ELTS_ZERO_POISON,
+ ISD::VP_CTTZ_ELTS, ISD::VP_CTTZ_ELTS_ZERO_UNDEF},
+ VT, Custom);
setOperationAction({ISD::VP_AND, ISD::VP_OR, ISD::VP_XOR}, VT, Custom);
@@ -1567,6 +1568,9 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
setOperationAction(ISD::EXPERIMENTAL_VP_SPLICE, VT, Custom);
setOperationAction(ISD::EXPERIMENTAL_VP_REVERSE, VT, Custom);
+
+ setOperationAction({ISD::CTTZ_ELTS, ISD::CTTZ_ELTS_ZERO_POISON}, VT,
+ Custom);
continue;
}
@@ -9189,6 +9193,9 @@ SDValue RISCVTargetLowering::LowerOperation(SDValue Op,
case ISD::PARTIAL_REDUCE_SMLA:
case ISD::PARTIAL_REDUCE_SUMLA:
return lowerPARTIAL_REDUCE_MLA(Op, DAG);
+ case ISD::CTTZ_ELTS:
+ case ISD::CTTZ_ELTS_ZERO_POISON:
+ return lowerCTTZ_ELTS(Op, DAG);
}
}
@@ -11518,20 +11525,20 @@ static SDValue lowerGetVectorLength(SDNode *N, SelectionDAG &DAG,
return DAG.getNode(ISD::TRUNCATE, DL, N->getValueType(0), Res);
}
-static SDValue lowerCttzElts(SDNode *N, SelectionDAG &DAG,
- const RISCVSubtarget &Subtarget) {
- SDValue Op0 = N->getOperand(1);
+SDValue RISCVTargetLowering::lowerCTTZ_ELTS(SDValue Op,
+ SelectionDAG &DAG) const {
+ SDValue Op0 = Op->getOperand(0);
MVT OpVT = Op0.getSimpleValueType();
MVT ContainerVT = OpVT;
if (OpVT.isFixedLengthVector()) {
- ContainerVT = getContainerForFixedLengthVector(DAG, OpVT, Subtarget);
- Op0 = convertToScalableVector(ContainerVT, Op0, DAG, Subtarget);
+ ContainerVT = getContainerForFixedLengthVector(OpVT);
+ Op0 = convertToScalableVector(ContainerVT, Op0, DAG, getSubtarget());
}
MVT XLenVT = Subtarget.getXLenVT();
- SDLoc DL(N);
- auto [Mask, VL] = getDefaultVLOps(OpVT, ContainerVT, DL, DAG, Subtarget);
+ SDLoc DL(Op);
+ auto [Mask, VL] = getDefaultVLOps(OpVT, ContainerVT, DL, DAG, getSubtarget());
SDValue Res = DAG.getNode(RISCVISD::VFIRST_VL, DL, XLenVT, Op0, Mask, VL);
- if (isOneConstant(N->getOperand(2)))
+ if (Op->getOpcode() == ISD::CTTZ_ELTS_ZERO_POISON)
return Res;
// Convert -1 to VL.
@@ -11686,8 +11693,6 @@ SDValue RISCVTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
}
case Intrinsic::experimental_get_vector_length:
return lowerGetVectorLength(Op.getNode(), DAG, Subtarget);
- case Intrinsic::experimental_cttz_elts:
- return lowerCttzElts(Op.getNode(), DAG, Subtarget);
case Intrinsic::riscv_vmv_x_s: {
SDValue Res = DAG.getNode(RISCVISD::VMV_X_S, DL, XLenVT, Op.getOperand(1));
return DAG.getNode(ISD::TRUNCATE, DL, Op.getValueType(), Res);
@@ -15888,11 +15893,6 @@ void RISCVTargetLowering::ReplaceNodeResults(SDNode *N,
Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
return;
}
- case Intrinsic::experimental_cttz_elts: {
- SDValue Res = lowerCttzElts(N, DAG, Subtarget);
- Results.push_back(DAG.getZExtOrTrunc(Res, DL, N->getValueType(0)));
- return;
- }
case Intrinsic::riscv_orc_b:
case Intrinsic::riscv_brev8:
case Intrinsic::riscv_sha256sig0:
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.h b/llvm/lib/Target/RISCV/RISCVISelLowering.h
index 8d88aeb7ae3fc..d5e1a2ceeda04 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.h
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.h
@@ -567,6 +567,7 @@ class RISCVTargetLowering : public TargetLowering {
SDValue lowerEH_DWARF_CFA(SDValue Op, SelectionDAG &DAG) const;
SDValue lowerCTLZ_CTTZ_ZERO_UNDEF(SDValue Op, SelectionDAG &DAG) const;
+ SDValue lowerCTTZ_ELTS(SDValue Op, SelectionDAG &DAG) const;
SDValue lowerStrictFPExtendOrRoundLike(SDValue Op, SelectionDAG &DAG) const;
diff --git a/llvm/test/CodeGen/RISCV/rvv/cttz-elts.ll b/llvm/test/CodeGen/RISCV/rvv/cttz-elts.ll
index 28ab573f59ac0..cdaed030e274c 100644
--- a/llvm/test/CodeGen/RISCV/rvv/cttz-elts.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/cttz-elts.ll
@@ -171,6 +171,23 @@ define i64 @i64_ctz_nxv16i1(<vscale x 16 x i1> %pg, <vscale x 16 x i1> %a) {
ret i64 %res
}
+define i64 @i64_ctz_nxv16i1_range(<vscale x 16 x i1> %pg, <vscale x 16 x i1> %a) vscale_range(2, 1024) {
+; RV32-LABEL: i64_ctz_nxv16i1_range:
+; RV32: # %bb.0:
+; RV32-NEXT: vsetvli a0, zero, e8, m2, ta, ma
+; RV32-NEXT: vfirst.m a0, v8
+; RV32-NEXT: li a1, 0
+; RV32-NEXT: ret
+;
+; RV64-LABEL: i64_ctz_nxv16i1_range:
+; RV64: # %bb.0:
+; RV64-NEXT: vsetvli a0, zero, e8, m2, ta, ma
+; RV64-NEXT: vfirst.m a0, v8
+; RV64-NEXT: ret
+ %res = call i64 @llvm.experimental.cttz.elts.i64.nxv16i1(<vscale x 16 x i1> %a, i1 1)
+ ret i64 %res
+}
+
define i32 @ctz_nxv16i1_poison(<vscale x 16 x i1> %pg, <vscale x 16 x i1> %a) {
; RV32-LABEL: ctz_nxv16i1_poison:
; RV32: # %bb.0:
@@ -192,20 +209,20 @@ define i32 @ctz_v16i1(<16 x i1> %pg, <16 x i1> %a) {
; RV32: # %bb.0:
; RV32-NEXT: vsetivli zero, 16, e8, m1, ta, ma
; RV32-NEXT: vfirst.m a0, v8
-; RV32-NEXT: bgez a0, .LBB5_2
+; RV32-NEXT: bgez a0, .LBB6_2
; RV32-NEXT: # %bb.1:
; RV32-NEXT: li a0, 16
-; RV32-NEXT: .LBB5_2:
+; RV32-NEXT: .LBB6_2:
; RV32-NEXT: ret
;
; RV64-LABEL: ctz_v16i1:
; RV64: # %bb.0:
; RV64-NEXT: vsetivli zero, 16, e8, m1, ta, ma
; RV64-NEXT: vfirst.m a0, v8
-; RV64-NEXT: bgez a0, .LBB5_2
+; RV64-NEXT: bgez a0, .LBB6_2
; RV64-NEXT: # %bb.1:
; RV64-NEXT: li a0, 16
-; RV64-NEXT: .LBB5_2:
+; RV64-NEXT: .LBB6_2:
; RV64-NEXT: ret
%res = call i32 @llvm.experimental.cttz.elts.i32.v16i1(<16 x i1> %a, i1 0)
ret i32 %res
@@ -232,20 +249,20 @@ define i16 @ctz_v8i1_i16_ret(<8 x i1> %a) {
; RV32: # %bb.0:
; RV32-NEXT: vsetivli zero, 8, e8, mf2, ta, ma
; RV32-NEXT: vfirst.m a0, v0
-; RV32-NEXT: bgez a0, .LBB7_2
+; RV32-NEXT: bgez a0, .LBB8_2
; RV32-NEXT: # %bb.1:
; RV32-NEXT: li a0, 8
-; RV32-NEXT: .LBB7_2:
+; RV32-NEXT: .LBB8_2:
; RV32-NEXT: ret
;
; RV64-LABEL: ctz_v8i1_i16_ret:
; RV64: # %bb.0:
; RV64-NEXT: vsetivli zero, 8, e8, mf2, ta, ma
; RV64-NEXT: vfirst.m a0, v0
-; RV64-NEXT: bgez a0, .LBB7_2
+; RV64-NEXT: bgez a0, .LBB8_2
; RV64-NEXT: # %bb.1:
; RV64-NEXT: li a0, 8
-; RV64-NEXT: .LBB7_2:
+; RV64-NEXT: .LBB8_2:
; RV64-NEXT: ret
%res = call i16 @llvm.experimental.cttz.elts.i16.v8i1(<8 x i1> %a, i1 0)
ret i16 %res
>From c516e26d4f88fb67b380f06e428dd39a9609eaed Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Tue, 10 Mar 2026 17:17:05 +0800
Subject: [PATCH 2/3] Move expansion into dedicated function, use
report_fatal_error
---
.../SelectionDAG/LegalizeIntegerTypes.cpp | 53 ++++++++++---------
llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h | 1 +
2 files changed, 30 insertions(+), 24 deletions(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
index 88c45729ecb55..7111ab7bab2e4 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
@@ -3245,30 +3245,9 @@ void DAGTypeLegalizer::ExpandIntegerResult(SDNode *N, unsigned ResNo) {
break;
case ISD::CTTZ_ELTS:
- case ISD::CTTZ_ELTS_ZERO_POISON: {
- EVT VT = N->getSimpleValueType(0);
- EVT HalfVT = VT.getHalfSizedIntegerVT(*DAG.getContext());
-
- EVT OpVT = N->getOperand(0).getValueType();
- ConstantRange CR(64, OpVT.getVectorMinNumElements());
- const Function &Fn = DAG.getMachineFunction().getFunction();
- if (OpVT.isScalableVector())
- CR = CR.umul_sat(getVScaleRange(&Fn, 64));
- if (N->getOpcode() == ISD::CTTZ_ELTS_ZERO_POISON)
- CR = CR.subtract(APInt(64, 1));
-
- // See if the half VT is large enough to fit the result into, or
- // alternatively if there's no upper bound on vscale in which case the
- // result is undefined.
- if (!(CR.getUnsignedMax().getActiveBits() > HalfVT.getScalarSizeInBits() ||
- CR.isUpperWrapped()))
- break;
-
- SDValue HalfOp =
- DAG.getNode(N->getOpcode(), SDLoc(N), HalfVT, N->getOperand(0));
- HalfOp = DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N), VT, HalfOp);
- SplitInteger(HalfOp, Lo, Hi);
- }
+ case ISD::CTTZ_ELTS_ZERO_POISON:
+ ExpandIntRes_CTTZ_ELTS(N, Lo, Hi);
+ break;
}
// If Lo/Hi is null, the sub-method took care of registering results etc.
@@ -5570,6 +5549,32 @@ void DAGTypeLegalizer::ExpandIntRes_READ_REGISTER(SDNode *N, SDValue &Lo,
Hi = DAG.getPOISON(HiVT);
}
+void DAGTypeLegalizer::ExpandIntRes_CTTZ_ELTS(SDNode *N, SDValue &Lo,
+ SDValue &Hi) {
+ EVT VT = N->getSimpleValueType(0);
+ EVT HalfVT = VT.getHalfSizedIntegerVT(*DAG.getContext());
+
+ EVT OpVT = N->getOperand(0).getValueType();
+ ConstantRange CR(64, OpVT.getVectorMinNumElements());
+ const Function &Fn = DAG.getMachineFunction().getFunction();
+ if (OpVT.isScalableVector())
+ CR = CR.umul_sat(getVScaleRange(&Fn, 64));
+ if (N->getOpcode() == ISD::CTTZ_ELTS_ZERO_POISON)
+ CR = CR.subtract(APInt(64, 1));
+
+ // See if the half VT is large enough to fit the result into, or
+ // alternatively if there's no upper bound on vscale in which case the
+ // result is undefined.
+ if (!(CR.getUnsignedMax().getActiveBits() > HalfVT.getScalarSizeInBits() ||
+ CR.isUpperWrapped()))
+ report_fatal_error("Unable to promote cttz_elts");
+
+ SDValue HalfOp =
+ DAG.getNode(N->getOpcode(), SDLoc(N), HalfVT, N->getOperand(0));
+ HalfOp = DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N), VT, HalfOp);
+ SplitInteger(HalfOp, Lo, Hi);
+}
+
//===----------------------------------------------------------------------===//
// Integer Operand Expansion
//===----------------------------------------------------------------------===//
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
index a8ffb66a9d911..14f361f8bcaed 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
@@ -502,6 +502,7 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
void ExpandIntRes_VSCALE (SDNode *N, SDValue &Lo, SDValue &Hi);
void ExpandIntRes_READ_REGISTER(SDNode *N, SDValue &Lo, SDValue &Hi);
+ void ExpandIntRes_CTTZ_ELTS(SDNode *N, SDValue &Lo, SDValue &Hi);
void ExpandShiftByConstant(SDNode *N, const APInt &Amt,
SDValue &Lo, SDValue &Hi);
>From a3a359d78acea12d80129cbff965344911571f8d Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Tue, 10 Mar 2026 18:53:58 +0800
Subject: [PATCH 3/3] Update comment
---
llvm/include/llvm/CodeGen/ISDOpcodes.h | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/ISDOpcodes.h b/llvm/include/llvm/CodeGen/ISDOpcodes.h
index 67261a715968c..fa578f733d4e8 100644
--- a/llvm/include/llvm/CodeGen/ISDOpcodes.h
+++ b/llvm/include/llvm/CodeGen/ISDOpcodes.h
@@ -1577,8 +1577,10 @@ enum NodeType {
/// Output: Output Chain
EXPERIMENTAL_VECTOR_HISTOGRAM,
- /// The `llvm.experimental.cttz.elts` intrinsic. Has a single i1 vector
- /// operand.
+ /// Returns the number of number of trailing (least significant) zero elements
+ /// in a vector. Has a single i1 vector operand. The result is poison if the
+ /// return type isn't wide enough to hold the maximum number of elements in
+ /// the input vector.
CTTZ_ELTS,
CTTZ_ELTS_ZERO_POISON,
More information about the llvm-commits
mailing list