[llvm] [SelectionDAG] Avoid irregular INSERT_SUBVECTOR when widening CTTZ_ELTS (PR #218019)
Oscar Priego via llvm-commits
llvm-commits at lists.llvm.org
Sat Aug 22 08:53:37 PDT 2026
https://github.com/Opriego updated https://github.com/llvm/llvm-project/pull/218019
>From 9f02b03ffbe8c5d959e8c3c7c1044d447af1dcc6 Mon Sep 17 00:00:00 2001
From: Oscar Priego Verdugo <oscar.priegov at gmail.com>
Date: Fri, 21 Aug 2026 14:06:39 -0600
Subject: [PATCH] [SelectionDAG] Avoid irregular INSERT_SUBVECTOR when widening
CTTZ_ELTS
Non-poison CTTZ_ELTS pads widened operands with active lanes using INSERT_SUBVECTOR. For irregular fixed-length i1 vectors whose widened type must subsequently be split, this can reach the SplitVecRes_INSERT_SUBVECTOR stack fallback, which cannot compute byte offsets for i1 elements and asserts.
Use VECTOR_SHUFFLE for this widening case, selecting original lanes from the normally widened source and padding lanes from an all-ones vector. This avoids the problematic INSERT_SUBVECTOR while preventing poison or undef widened lanes from being selected.
Keep the existing widening behavior for zero-poison, scalable, and non-splitting cases.
---
.../SelectionDAG/LegalizeVectorTypes.cpp | 18 +++-
llvm/test/CodeGen/X86/intrinsic-cttz-elts.ll | 89 +++++++++++++++++++
2 files changed, 104 insertions(+), 3 deletions(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
index 3417c9734af3b..3eaeb390c4a9c 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
@@ -8901,8 +8901,8 @@ SDValue DAGTypeLegalizer::WidenVecOp_VSELECT(SDNode *N) {
SDValue DAGTypeLegalizer::WidenVecOp_CttzElements(SDNode *N) {
SDLoc DL(N);
SDValue Source = N->getOperand(0);
- EVT WideVT =
- TLI.getTypeToTransformTo(*DAG.getContext(), Source.getValueType());
+ EVT SourceVT = Source.getValueType();
+ EVT WideVT = TLI.getTypeToTransformTo(*DAG.getContext(), SourceVT);
SDValue WideSource;
if (N->getOpcode() == ISD::CTTZ_ELTS_ZERO_POISON) {
@@ -8911,7 +8911,19 @@ SDValue DAGTypeLegalizer::WidenVecOp_CttzElements(SDNode *N) {
// Pad the widened portion with all-ones so the extra lanes appear as
// active (non-zero) elements and do not contribute trailing zeros.
SDValue AllOnes = DAG.getAllOnesConstant(DL, WideVT);
- WideSource = DAG.getInsertSubvector(DL, AllOnes, Source, 0);
+ if (WideVT.isFixedLengthVector() &&
+ SourceVT.getVectorElementType() == MVT::i1 &&
+ getTypeAction(WideVT) == TargetLowering::TypeSplitVector) {
+ WideSource = GetWidenedVector(Source);
+ unsigned WideElts = WideVT.getVectorNumElements();
+ SmallVector<int> Mask(WideElts);
+ std::iota(Mask.begin(), Mask.end(), 0);
+ for (unsigned I = SourceVT.getVectorNumElements(); I != WideElts; ++I)
+ Mask[I] += WideElts;
+ WideSource = DAG.getVectorShuffle(WideVT, DL, WideSource, AllOnes, Mask);
+ } else {
+ WideSource = DAG.getInsertSubvector(DL, AllOnes, Source, 0);
+ }
}
return DAG.getNode(N->getOpcode(), DL, N->getValueType(0), WideSource,
diff --git a/llvm/test/CodeGen/X86/intrinsic-cttz-elts.ll b/llvm/test/CodeGen/X86/intrinsic-cttz-elts.ll
index 65231c484db98..7e5dc2dd74c8c 100644
--- a/llvm/test/CodeGen/X86/intrinsic-cttz-elts.ll
+++ b/llvm/test/CodeGen/X86/intrinsic-cttz-elts.ll
@@ -101,5 +101,94 @@ define i8 @ctz_v8i16_poison(<8 x i16> %a) {
ret i8 %res
}
+; Irregular predicate vectors widen to a power-of-two number of lanes. The
+; widened lanes must be active so an all-zero input returns the original lane
+; count, without requiring an irregular INSERT_SUBVECTOR during legalization.
+define i32 @ctz_zero_v17i1() {
+; CHECK-LABEL: .LCPI3_0:
+; CHECK-NEXT: .long 84281096
+; CHECK-NEXT: .long 16909060
+; CHECK-NEXT: .long 84281096
+; CHECK-NEXT: .long 16909060
+; CHECK-LABEL: .LCPI3_1:
+; CHECK-NEXT: .byte 0
+; CHECK-NEXT: .byte 15
+; CHECK-NEXT: .byte 14
+; CHECK-NEXT: .byte 13
+; CHECK-NEXT: .byte 12
+; CHECK-NEXT: .byte 11
+; CHECK-NEXT: .byte 10
+; CHECK-NEXT: .byte 9
+; CHECK-NEXT: .byte 8
+; CHECK-NEXT: .byte 7
+; CHECK-NEXT: .byte 6
+; CHECK-NEXT: .byte 5
+; CHECK-NEXT: .byte 4
+; CHECK-NEXT: .byte 3
+; CHECK-NEXT: .byte 2
+; CHECK-NEXT: .byte 1
+; CHECK-LABEL: ctz_zero_v17i1:
+; CHECK: # %bb.0:
+; CHECK-NEXT: movdqa {{.*#+}} xmm0 = [84281096,16909060,84281096,16909060]
+; CHECK-NEXT: pmaxub {{\.LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; CHECK-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; CHECK-NEXT: pmaxub %xmm0, %xmm1
+; CHECK-NEXT: movdqa %xmm1, %xmm0
+; CHECK-NEXT: psrld $16, %xmm0
+; CHECK-NEXT: pmaxub %xmm1, %xmm0
+; CHECK-NEXT: movdqa %xmm0, %xmm1
+; CHECK-NEXT: psrlw $8, %xmm1
+; CHECK-NEXT: pmaxub %xmm0, %xmm1
+; CHECK-NEXT: movd %xmm1, %eax
+; CHECK-NEXT: movb $16, %cl
+; CHECK-NEXT: subb %al, %cl
+; CHECK-NEXT: movzbl %cl, %eax
+; CHECK-NEXT: addl $16, %eax
+; CHECK-NEXT: retq
+ %res = call i32 @llvm.experimental.cttz.elts.i32.v17i1(<17 x i1> zeroinitializer, i1 false)
+ ret i32 %res
+}
+
+define i32 @ctz_zero_v31i1() {
+; CHECK-LABEL: .LCPI4_0:
+; CHECK-NEXT: .long 16777216
+; CHECK-NEXT: .long 16777216
+; CHECK-NEXT: .long 16777216
+; CHECK-NEXT: .long 16777216
+; CHECK-LABEL: .LCPI4_1:
+; CHECK-NEXT: .byte 0
+; CHECK-NEXT: .byte 0
+; CHECK-NEXT: .byte 0
+; CHECK-NEXT: .byte 0
+; CHECK-NEXT: .byte 0
+; CHECK-NEXT: .byte 0
+; CHECK-NEXT: .byte 0
+; CHECK-NEXT: .byte 1
+; CHECK-NEXT: .zero 1
+; CHECK-NEXT: .zero 1
+; CHECK-NEXT: .zero 1
+; CHECK-NEXT: .zero 1
+; CHECK-NEXT: .zero 1
+; CHECK-NEXT: .zero 1
+; CHECK-NEXT: .zero 1
+; CHECK-NEXT: .zero 1
+; CHECK-LABEL: ctz_zero_v31i1:
+; CHECK: # %bb.0:
+; CHECK-NEXT: movdqa {{.*#+}} xmm0 = [16777216,16777216,16777216,16777216]
+; CHECK-NEXT: pmaxub {{\.LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; CHECK-NEXT: movdqa %xmm0, %xmm1
+; CHECK-NEXT: psrld $24, %xmm1
+; CHECK-NEXT: psrld $16, %xmm0
+; CHECK-NEXT: pmaxub %xmm1, %xmm0
+; CHECK-NEXT: movd %xmm0, %eax
+; CHECK-NEXT: movb $16, %cl
+; CHECK-NEXT: subb %al, %cl
+; CHECK-NEXT: movzbl %cl, %eax
+; CHECK-NEXT: addl $16, %eax
+; CHECK-NEXT: retq
+ %res = call i32 @llvm.experimental.cttz.elts.i32.v31i1(<31 x i1> zeroinitializer, i1 false)
+ ret i32 %res
+}
+
declare i8 @llvm.experimental.cttz.elts.i8.v8i16(<8 x i16>, i1)
declare i16 @llvm.experimental.cttz.elts.i16.v4i32(<4 x i32>, i1)
More information about the llvm-commits
mailing list