[llvm-branch-commits] [llvm] 5f27b70 - [SelectionDAG] Avoid irregular INSERT_SUBVECTOR when widening CTTZ_ELTS (#218019)
Tobias Hieta via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Sun Sep 6 23:07:43 PDT 2026
Author: Oscar Priego
Date: 2026-09-07T08:07:18+02:00
New Revision: 5f27b704b2fc7fe09aaabe4fcae964f432ba81db
URL: https://github.com/llvm/llvm-project/commit/5f27b704b2fc7fe09aaabe4fcae964f432ba81db
DIFF: https://github.com/llvm/llvm-project/commit/5f27b704b2fc7fe09aaabe4fcae964f432ba81db.diff
LOG: [SelectionDAG] Avoid irregular INSERT_SUBVECTOR when widening CTTZ_ELTS (#218019)
Fixes #217985
Non-poison `CTTZ_ELTS` pads widened operands with active lanes using
`INSERT_SUBVECTOR`. For irregular fixed-length vectors whose
widened type subsequently requires splitting, this can reach the
`SplitVecRes_INSERT_SUBVECTOR` stack-spill fallback.
That fallback calls `getVectorSubVecPointer`, which requires
byte-addressable elements.
Use `VECTOR_SHUFFLE` for this specific widening case. Original lanes are
selected from the normally widened source, while padding lanes are
selected from an all-ones vector. This avoids the irregular
`INSERT_SUBVECTOR` and prevents poison/undef widened padding from being
observed.
Other widening cases retain their existing behavior.
AI-assisted: OpenAI Codex was used during investigation and
implementation. I manually reviewed the final patch, validated the
SelectionDAG type-action behavior, and ran the X86, AArch64, and RISC-V
regression checks described above.
(cherry picked from commit 01ff7258c2374b3cca02bc8631a445ac32f9b930)
Added:
Modified:
llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
llvm/test/CodeGen/X86/intrinsic-cttz-elts.ll
Removed:
################################################################################
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
index 3817a77f8bd44..c0fab69cb518a 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
@@ -8738,8 +8738,8 @@ SDValue DAGTypeLegalizer::WidenVecOp_VSELECT(SDNode *N) {
SDValue DAGTypeLegalizer::WidenVecOp_CttzElements(SDNode *N) {
SDLoc DL(N);
SDValue Source = N->getOperand(0);
- EVT WideVT =
- TLI.getTypeToTransformTo(*DAG.getContext(), Source.getValueType());
+ EVT SourceVT = Source.getValueType();
+ EVT WideVT = TLI.getTypeToTransformTo(*DAG.getContext(), SourceVT);
SDValue WideSource;
if (N->getOpcode() == ISD::CTTZ_ELTS_ZERO_POISON) {
@@ -8748,7 +8748,18 @@ SDValue DAGTypeLegalizer::WidenVecOp_CttzElements(SDNode *N) {
// Pad the widened portion with all-ones so the extra lanes appear as
// active (non-zero) elements and do not contribute trailing zeros.
SDValue AllOnes = DAG.getAllOnesConstant(DL, WideVT);
- WideSource = DAG.getInsertSubvector(DL, AllOnes, Source, 0);
+ if (WideVT.isFixedLengthVector() &&
+ getTypeAction(WideVT) == TargetLowering::TypeSplitVector) {
+ WideSource = GetWidenedVector(Source);
+ unsigned WideElts = WideVT.getVectorNumElements();
+ SmallVector<int> Mask(WideElts);
+ std::iota(Mask.begin(), Mask.end(), 0);
+ for (unsigned I = SourceVT.getVectorNumElements(); I != WideElts; ++I)
+ Mask[I] += WideElts;
+ WideSource = DAG.getVectorShuffle(WideVT, DL, WideSource, AllOnes, Mask);
+ } else {
+ WideSource = DAG.getInsertSubvector(DL, AllOnes, Source, 0);
+ }
}
return DAG.getNode(N->getOpcode(), DL, N->getValueType(0), WideSource,
diff --git a/llvm/test/CodeGen/X86/intrinsic-cttz-elts.ll b/llvm/test/CodeGen/X86/intrinsic-cttz-elts.ll
index f8cc45c1b5bda..a2f2ffdfd3325 100644
--- a/llvm/test/CodeGen/X86/intrinsic-cttz-elts.ll
+++ b/llvm/test/CodeGen/X86/intrinsic-cttz-elts.ll
@@ -196,5 +196,72 @@ define i32 @ctz_zero_v8i1() {
ret i32 %res
}
+; Irregular predicate vectors widen to a power-of-two number of lanes. The
+; widened lanes must be active so an all-zero input returns the original lane
+; count, without requiring an irregular INSERT_SUBVECTOR during legalization.
+define i32 @ctz_zero_v17i1() {
+; CHECK-LABEL: ctz_zero_v17i1:
+; CHECK: # %bb.0:
+; CHECK-NEXT: movdqa {{.*#+}} xmm0 = [84281096,16909060,84281096,16909060]
+; CHECK-NEXT: pmaxub {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; CHECK-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; CHECK-NEXT: pmaxub %xmm0, %xmm1
+; CHECK-NEXT: movdqa %xmm1, %xmm0
+; CHECK-NEXT: psrld $16, %xmm0
+; CHECK-NEXT: pmaxub %xmm1, %xmm0
+; CHECK-NEXT: movdqa %xmm0, %xmm1
+; CHECK-NEXT: psrlw $8, %xmm1
+; CHECK-NEXT: pmaxub %xmm0, %xmm1
+; CHECK-NEXT: movd %xmm1, %eax
+; CHECK-NEXT: movb $16, %cl
+; CHECK-NEXT: subb %al, %cl
+; CHECK-NEXT: movzbl %cl, %eax
+; CHECK-NEXT: addl $16, %eax
+; CHECK-NEXT: retq
+;
+; AVX512-LABEL: ctz_zero_v17i1:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpmovzxbw {{.*#+}} xmm0 = [240,242,244,246,248,250,252,254]
+; AVX512-NEXT: vpminub {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; AVX512-NEXT: vphminposuw %xmm0, %xmm0
+; AVX512-NEXT: vmovd %xmm0, %eax
+; AVX512-NEXT: addb $17, %al
+; AVX512-NEXT: movzbl %al, %eax
+; AVX512-NEXT: addl $16, %eax
+; AVX512-NEXT: retq
+ %res = call i32 @llvm.experimental.cttz.elts.i32.v17i1(<17 x i1> zeroinitializer, i1 false)
+ ret i32 %res
+}
+
+define i32 @ctz_zero_v31i1() {
+; CHECK-LABEL: ctz_zero_v31i1:
+; CHECK: # %bb.0:
+; CHECK-NEXT: movdqa {{.*#+}} xmm0 = [16777216,16777216,16777216,16777216]
+; CHECK-NEXT: pmaxub {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; CHECK-NEXT: movdqa %xmm0, %xmm1
+; CHECK-NEXT: psrld $24, %xmm1
+; CHECK-NEXT: psrld $16, %xmm0
+; CHECK-NEXT: pmaxub %xmm1, %xmm0
+; CHECK-NEXT: movd %xmm0, %eax
+; CHECK-NEXT: movb $16, %cl
+; CHECK-NEXT: subb %al, %cl
+; CHECK-NEXT: movzbl %cl, %eax
+; CHECK-NEXT: addl $16, %eax
+; CHECK-NEXT: retq
+;
+; AVX512-LABEL: ctz_zero_v31i1:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpmovzxbw {{.*#+}} xmm0 = [255,255,255,255,255,255,255,254]
+; AVX512-NEXT: vpminub {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; AVX512-NEXT: vphminposuw %xmm0, %xmm0
+; AVX512-NEXT: vmovd %xmm0, %eax
+; AVX512-NEXT: addb $17, %al
+; AVX512-NEXT: movzbl %al, %eax
+; AVX512-NEXT: addl $16, %eax
+; AVX512-NEXT: retq
+ %res = call i32 @llvm.experimental.cttz.elts.i32.v31i1(<31 x i1> zeroinitializer, i1 false)
+ ret i32 %res
+}
+
declare i8 @llvm.experimental.cttz.elts.i8.v8i16(<8 x i16>, i1)
declare i16 @llvm.experimental.cttz.elts.i16.v4i32(<4 x i32>, i1)
More information about the llvm-branch-commits
mailing list