[llvm] fa7a602 - [CodeGen] Add widening support for ISD::CTTZ_ELTS (#205841)
via llvm-commits
llvm-commits at lists.llvm.org
Tue Jun 30 02:39:50 PDT 2026
Author: Chandana Mudda
Date: 2026-06-30T09:39:44Z
New Revision: fa7a6025df042fab2bb69947b773c23eaa8dd62b
URL: https://github.com/llvm/llvm-project/commit/fa7a6025df042fab2bb69947b773c23eaa8dd62b
DIFF: https://github.com/llvm/llvm-project/commit/fa7a6025df042fab2bb69947b773c23eaa8dd62b.diff
LOG: [CodeGen] Add widening support for ISD::CTTZ_ELTS (#205841)
WidenVectorOperand had no handler forCTTZ_ELTS/
CTTZ_ELTS_ZERO_POISON, causing a fatal error when the input vector type
needed widening.
Add WidenVecOp_CttzElements which widens the input vector and pads the
extra lanes with all-ones, ensuring they do not contribute spurious
trailing zeros to the count. This follows the same pattern as the
existing
WidenVecOp_VP_CttzElements.
Assisted-by: Claude (Anthropic)
Added:
llvm/test/CodeGen/Hexagon/cttz-elts-widen.ll
Modified:
llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
llvm/test/CodeGen/AArch64/intrinsic-cttz-elts-sve.ll
llvm/test/CodeGen/RISCV/rvv/cttz-elts.ll
Removed:
################################################################################
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
index 71d3e1c66be86..d4d56a9563f71 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
@@ -1132,6 +1132,7 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
SDValue WidenVecOp_VECREDUCE_SEQ(SDNode *N);
SDValue WidenVecOp_VP_REDUCE(SDNode *N);
SDValue WidenVecOp_ExpOp(SDNode *N);
+ SDValue WidenVecOp_CttzElements(SDNode *N);
SDValue WidenVecOp_VP_CttzElements(SDNode *N);
SDValue WidenVecOp_VECTOR_FIND_LAST_ACTIVE(SDNode *N);
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
index 97aa765642ea7..bdb8cddda4909 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
@@ -7741,6 +7741,10 @@ bool DAGTypeLegalizer::WidenVectorOperand(SDNode *N, unsigned OpNo) {
case ISD::VP_REDUCE_FMINIMUM:
Res = WidenVecOp_VP_REDUCE(N);
break;
+ case ISD::CTTZ_ELTS:
+ case ISD::CTTZ_ELTS_ZERO_POISON:
+ Res = WidenVecOp_CttzElements(N);
+ break;
case ISD::VP_CTTZ_ELTS:
case ISD::VP_CTTZ_ELTS_ZERO_POISON:
Res = WidenVecOp_VP_CttzElements(N);
@@ -8723,6 +8727,20 @@ SDValue DAGTypeLegalizer::WidenVecOp_VSELECT(SDNode *N) {
return DAG.getExtractSubvector(DL, VT, Select, 0);
}
+SDValue DAGTypeLegalizer::WidenVecOp_CttzElements(SDNode *N) {
+ SDLoc DL(N);
+ SDValue Source = N->getOperand(0);
+ EVT WideVT = GetWidenedVector(Source).getValueType();
+
+ // Pad the widened portion with all-ones so the extra lanes appear as
+ // active (non-zero) elements and do not contribute trailing zeros.
+ SDValue AllOnes = DAG.getAllOnesConstant(DL, WideVT);
+ SDValue WideSource = DAG.getInsertSubvector(DL, AllOnes, Source, 0);
+
+ return DAG.getNode(N->getOpcode(), DL, N->getValueType(0), WideSource,
+ N->getFlags());
+}
+
SDValue DAGTypeLegalizer::WidenVecOp_VP_CttzElements(SDNode *N) {
SDLoc DL(N);
SDValue Source = GetWidenedVector(N->getOperand(0));
diff --git a/llvm/test/CodeGen/AArch64/intrinsic-cttz-elts-sve.ll b/llvm/test/CodeGen/AArch64/intrinsic-cttz-elts-sve.ll
index d7beaa5addddb..fa3673feaee46 100644
--- a/llvm/test/CodeGen/AArch64/intrinsic-cttz-elts-sve.ll
+++ b/llvm/test/CodeGen/AArch64/intrinsic-cttz-elts-sve.ll
@@ -121,6 +121,42 @@ define i32 @add_i32_ctz_nxv2i1_poison(<vscale x 2 x i1> %a, i32 %b) {
ret i32 %add
}
+define i32 @ctz_nxv3i1(<vscale x 3 x i1> %a) {
+; CHECK-LABEL: ctz_nxv3i1:
+; CHECK: // %bb.0:
+; CHECK-NEXT: rdvl x8, #1
+; CHECK-NEXT: mov w9, #3 // =0x3
+; CHECK-NEXT: ptrue p2.s
+; CHECK-NEXT: lsr x8, x8, #4
+; CHECK-NEXT: mul x8, x8, x9
+; CHECK-NEXT: whilelo p1.s, xzr, x8
+; CHECK-NEXT: not p1.b, p2/z, p1.b
+; CHECK-NEXT: mov p0.b, p1/m, p1.b
+; CHECK-NEXT: brkb p0.b, p2/z, p0.b
+; CHECK-NEXT: cntp x0, p0, p0.s
+; CHECK-NEXT: ret
+ %res = call i32 @llvm.experimental.cttz.elts.i32.nxv3i1(<vscale x 3 x i1> %a, i1 0)
+ ret i32 %res
+}
+
+define i32 @ctz_nxv3i1_poison(<vscale x 3 x i1> %a) {
+; CHECK-LABEL: ctz_nxv3i1_poison:
+; CHECK: // %bb.0:
+; CHECK-NEXT: rdvl x8, #1
+; CHECK-NEXT: mov w9, #3 // =0x3
+; CHECK-NEXT: ptrue p2.s
+; CHECK-NEXT: lsr x8, x8, #4
+; CHECK-NEXT: mul x8, x8, x9
+; CHECK-NEXT: whilelo p1.s, xzr, x8
+; CHECK-NEXT: not p1.b, p2/z, p1.b
+; CHECK-NEXT: mov p0.b, p1/m, p1.b
+; CHECK-NEXT: brkb p0.b, p2/z, p0.b
+; CHECK-NEXT: cntp x0, p0, p0.s
+; CHECK-NEXT: ret
+ %res = call i32 @llvm.experimental.cttz.elts.i32.nxv3i1(<vscale x 3 x i1> %a, i1 1)
+ ret i32 %res
+}
+
define i32 @ctz_nxv4i1(<vscale x 4 x i1> %a) {
; CHECK-LABEL: ctz_nxv4i1:
; CHECK: // %bb.0:
diff --git a/llvm/test/CodeGen/Hexagon/cttz-elts-widen.ll b/llvm/test/CodeGen/Hexagon/cttz-elts-widen.ll
new file mode 100644
index 0000000000000..53ce686d96d5a
--- /dev/null
+++ b/llvm/test/CodeGen/Hexagon/cttz-elts-widen.ll
@@ -0,0 +1,58 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --filter "r0 = #|vcmpw\.eq|sub\(#4|sub\(#8" --version 6
+; RUN: llc -march=hexagon < %s | FileCheck %s
+
+; Widening of ISD::CTTZ_ELTS / CTTZ_ELTS_ZERO_POISON when the operand
+; vector type is illegal (<3 x i32> widens to <4 x i32>). Padded lanes
+; must be all-ones so the count never falls into the synthetic lane.
+
+; All-zero input must return OrigElts (3), never WideElts (4).
+define i32 @cttz_elts_zero_v3i32() {
+; CHECK-LABEL: cttz_elts_zero_v3i32:
+; CHECK: r0 = #3
+ %res = call i32 @llvm.experimental.cttz.elts.i32.v3i32(<3 x i32> zeroinitializer, i1 false)
+ ret i32 %res
+}
+
+; First non-zero element at the highest *original* lane (lane 2 of
+; <3 x i32>). The padded lane 3 must not steal the result: answer is 2.
+define i32 @cttz_elts_high_lane_v3i32() {
+; CHECK-LABEL: cttz_elts_high_lane_v3i32:
+; CHECK: r0 = #2
+ %res = call i32 @llvm.experimental.cttz.elts.i32.v3i32(<3 x i32> <i32 0, i32 0, i32 9>, i1 false)
+ ret i32 %res
+}
+
+; Symbolic input: confirm the operand is actually widened to <4 x i32>.
+; Two vcmpw.eq compares against the zero pair cover both halves of the
+; widened vector, and the final sub(#4, ...) reflects the widened lane
+; count. A regression that fails to widen, or widens without all-ones
+; padding, would change these.
+define i32 @cttz_elts_v3i32(<3 x i32> %v) {
+; CHECK-LABEL: cttz_elts_v3i32:
+; CHECK: p2 = vcmpw.eq(r1:0,r5:4)
+; CHECK: p0 = vcmpw.eq(r7:6,r5:4)
+; CHECK: r0 = sub(#4,r0)
+ %res = call i32 @llvm.experimental.cttz.elts.i32.v3i32(<3 x i32> %v, i1 false)
+ ret i32 %res
+}
+
+; Same shape for the zero-poison variant; padding must still be emitted.
+define i32 @cttz_elts_zero_poison_v3i32(<3 x i32> %v) {
+; CHECK-LABEL: cttz_elts_zero_poison_v3i32:
+; CHECK: p2 = vcmpw.eq(r1:0,r5:4)
+; CHECK: p0 = vcmpw.eq(r7:6,r5:4)
+; CHECK: r0 = sub(#4,r0)
+ %res = call i32 @llvm.experimental.cttz.elts.i32.v3i32(<3 x i32> %v, i1 true)
+ ret i32 %res
+}
+
+; Predicate-style operand: <7 x i1> widens to <8 x i1>. This is the most
+; common use case. The padded lane must be all-ones so the widened lane
+; count (8) only appears when no original lane is active; the final
+; sub(#8, ...) reflects the widened vector.
+define i32 @cttz_elts_v7i1(<7 x i1> %v) {
+; CHECK-LABEL: cttz_elts_v7i1:
+; CHECK: r0 = sub(#8,r0)
+ %res = call i32 @llvm.experimental.cttz.elts.i32.v7i1(<7 x i1> %v, i1 true)
+ ret i32 %res
+}
diff --git a/llvm/test/CodeGen/RISCV/rvv/cttz-elts.ll b/llvm/test/CodeGen/RISCV/rvv/cttz-elts.ll
index fc892c1a5cae0..e761f14b0d4e7 100644
--- a/llvm/test/CodeGen/RISCV/rvv/cttz-elts.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/cttz-elts.ll
@@ -201,4 +201,64 @@ define i16 @ctz_v8i1_i16_ret(<8 x i1> %a) {
ret i16 %res
}
+define i32 @ctz_v3i32(<3 x i32> %a) {
+; RV32-LABEL: ctz_v3i32:
+; RV32: # %bb.0:
+; RV32-NEXT: vsetivli zero, 4, e8, mf4, ta, ma
+; RV32-NEXT: vmv.v.i v9, 0
+; RV32-NEXT: vsetvli zero, zero, e32, m1, ta, ma
+; RV32-NEXT: vmsne.vi v0, v8, 0
+; RV32-NEXT: vfirst.m a0, v0
+; RV32-NEXT: vsetvli zero, zero, e8, mf4, ta, ma
+; RV32-NEXT: vmerge.vim v8, v9, 1, v0
+; RV32-NEXT: vslidedown.vi v9, v8, 1
+; RV32-NEXT: vmv.x.s a1, v9
+; RV32-NEXT: vslidedown.vi v8, v8, 2
+; RV32-NEXT: vmv.x.s a2, v8
+; RV32-NEXT: seqz a0, a0
+; RV32-NEXT: vmv.v.x v8, a0
+; RV32-NEXT: vslide1down.vx v8, v8, a1
+; RV32-NEXT: vslide1down.vx v8, v8, a2
+; RV32-NEXT: li a0, 1
+; RV32-NEXT: vslide1down.vx v8, v8, a0
+; RV32-NEXT: vand.vi v8, v8, 1
+; RV32-NEXT: vmsne.vi v8, v8, 0
+; RV32-NEXT: vfirst.m a0, v8
+; RV32-NEXT: bgez a0, .LBB9_2
+; RV32-NEXT: # %bb.1:
+; RV32-NEXT: li a0, 4
+; RV32-NEXT: .LBB9_2:
+; RV32-NEXT: ret
+;
+; RV64-LABEL: ctz_v3i32:
+; RV64: # %bb.0:
+; RV64-NEXT: vsetivli zero, 4, e8, mf4, ta, ma
+; RV64-NEXT: vmv.v.i v9, 0
+; RV64-NEXT: vsetvli zero, zero, e32, m1, ta, ma
+; RV64-NEXT: vmsne.vi v0, v8, 0
+; RV64-NEXT: vfirst.m a0, v0
+; RV64-NEXT: vsetvli zero, zero, e8, mf4, ta, ma
+; RV64-NEXT: vmerge.vim v8, v9, 1, v0
+; RV64-NEXT: vslidedown.vi v9, v8, 1
+; RV64-NEXT: vmv.x.s a1, v9
+; RV64-NEXT: vslidedown.vi v8, v8, 2
+; RV64-NEXT: vmv.x.s a2, v8
+; RV64-NEXT: seqz a0, a0
+; RV64-NEXT: vmv.v.x v8, a0
+; RV64-NEXT: vslide1down.vx v8, v8, a1
+; RV64-NEXT: vslide1down.vx v8, v8, a2
+; RV64-NEXT: li a0, 1
+; RV64-NEXT: vslide1down.vx v8, v8, a0
+; RV64-NEXT: vand.vi v8, v8, 1
+; RV64-NEXT: vmsne.vi v8, v8, 0
+; RV64-NEXT: vfirst.m a0, v8
+; RV64-NEXT: bgez a0, .LBB9_2
+; RV64-NEXT: # %bb.1:
+; RV64-NEXT: li a0, 4
+; RV64-NEXT: .LBB9_2:
+; RV64-NEXT: ret
+ %res = call i32 @llvm.experimental.cttz.elts.i32.v3i32(<3 x i32> %a, i1 0)
+ ret i32 %res
+}
+
attributes #0 = { vscale_range(2,1024) }
More information about the llvm-commits
mailing list