[llvm] [X86] Compute `CTLZ` using fp math for `v4i32` on SSE2 targets (PR #190930)
via llvm-commits
llvm-commits at lists.llvm.org
Thu Apr 30 06:57:52 PDT 2026
https://github.com/def3r updated https://github.com/llvm/llvm-project/pull/190930
>From ee01b3a693b68e26f8d9ada5b433fb3c17d339fb Mon Sep 17 00:00:00 2001
From: Ayaan Khan <ayaanandayaan2 at gmail.com>
Date: Wed, 8 Apr 2026 10:02:44 +0530
Subject: [PATCH 1/3] [X86] Compute `CTLZ` using fp math for `v4i32` on SSE2
targets
Co-authored-by: NishiB137 <cs23btech11041 at iith.ac.in>
Co-authored-by: VindhyaP312 <cs23btech11044 at iith.ac.in>
---
llvm/include/llvm/CodeGen/TargetLowering.h | 5 +
.../CodeGen/SelectionDAG/TargetLowering.cpp | 59 +++++
llvm/lib/Target/X86/X86ISelLowering.cpp | 13 +
llvm/test/CodeGen/X86/combine-srl.ll | 62 ++---
.../CodeGen/X86/expand-vp-int-intrinsics.ll | 62 ++---
llvm/test/CodeGen/X86/vec_ctbits.ll | 62 ++---
llvm/test/CodeGen/X86/vector-lzcnt-128.ll | 244 +++++++-----------
llvm/test/CodeGen/X86/vector-lzcnt-sub128.ll | 60 ++---
8 files changed, 273 insertions(+), 294 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h
index 59a0f2d2e0c2a..0d76b852e321e 100644
--- a/llvm/include/llvm/CodeGen/TargetLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetLowering.h
@@ -5653,6 +5653,11 @@ class LLVM_ABI TargetLowering : public TargetLoweringBase {
/// \returns The expansion result or SDValue() if it fails.
SDValue expandCTLZ(SDNode *N, SelectionDAG &DAG) const;
+ /// Expands a CTLZ node into a sequence of floating point operations.
+ /// \param N Node to expand
+ /// \returns The expansion result or SDValue() if it fails.
+ SDValue expandCTLZWithFP(SDNode *N, SelectionDAG &DAG) const;
+
/// Expand VP_CTLZ/VP_CTLZ_ZERO_UNDEF nodes.
/// \param N Node to expand
/// \returns The expansion result or SDValue() if it fails.
diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
index 092bc283c84dc..fe9046c8e0ac9 100644
--- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
@@ -9880,6 +9880,65 @@ SDValue TargetLowering::expandCTLZ(SDNode *Node, SelectionDAG &DAG) const {
return DAG.getNode(ISD::CTPOP, dl, VT, Op);
}
+SDValue TargetLowering::expandCTLZWithFP(SDNode *Node,
+ SelectionDAG &DAG) const {
+ // pseudocode :
+ // if (x == 0) return 32;
+ // f64 f = (f64)x;
+ // u64 i = bitcast<u64>(f);
+ // u32 ilog2 = (u32)(i >> 52) - 1023;
+ // return 31 - ilog2;
+
+ SDLoc dl(Node);
+ EVT VT = Node->getValueType(0);
+ SDValue Op = Node->getOperand(0);
+
+ assert(VT.isVector() && "This expansion is intended for vectors");
+ EVT EltVT = VT.getVectorElementType();
+ if (EltVT != MVT::i32) {
+ return SDValue();
+ }
+
+ EVT FloatVT = VT.changeVectorElementType(*DAG.getContext(), MVT::f64);
+ const fltSemantics &Sem = FloatVT.getVectorElementType().getFltSemantics();
+ unsigned BitWidth = EltVT.getSizeInBits();
+ unsigned MantissaBits = APFloat::semanticsPrecision(Sem);
+ unsigned ExponentBias = APFloat::semanticsMaxExponent(Sem);
+
+ unsigned NumElts = VT.getVectorNumElements();
+ SmallVector<SDValue, 4> FloatElts;
+ for (unsigned i = 0; i < NumElts; i++) {
+ SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op,
+ DAG.getIntPtrConstant(i, dl));
+ SDValue FElt = DAG.getNode(ISD::UINT_TO_FP, dl, MVT::f64, Elt);
+ FloatElts.push_back(FElt);
+ }
+ SDValue Float = DAG.getBuildVector(FloatVT, dl, FloatElts);
+
+ EVT FloatBitsVT = FloatVT.changeVectorElementTypeToInteger();
+ SDValue FloatBits = DAG.getNode(ISD::BITCAST, dl, FloatBitsVT, Float);
+ SDValue Exp = DAG.getNode(
+ ISD::SRL, dl, FloatBitsVT, FloatBits,
+ DAG.getShiftAmountConstant(MantissaBits - 1, FloatBitsVT, dl));
+ SDValue ExpTrunc = DAG.getNode(ISD::TRUNCATE, dl, VT, Exp);
+ SDValue NonZeroRes = DAG.getNode(
+ ISD::SUB, dl, VT, DAG.getConstant(BitWidth - 1 + ExponentBias, dl, VT),
+ ExpTrunc);
+
+ // Skip Op == 0 case for CTLZ_ZERO_UNDEF
+ if (Node->getOpcode() == ISD::CTLZ_ZERO_UNDEF) {
+ return NonZeroRes;
+ }
+
+ // This Handles the Op == 0 case
+ EVT CmpVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
+ SDValue Zero = DAG.getConstant(0, dl, VT);
+ SDValue IsZero = DAG.getSetCC(dl, CmpVT, Op, Zero, ISD::SETEQ);
+
+ return DAG.getNode(ISD::VSELECT, dl, VT, IsZero,
+ DAG.getConstant(BitWidth, dl, VT), NonZeroRes);
+}
+
SDValue TargetLowering::expandVPCTLZ(SDNode *Node, SelectionDAG &DAG) const {
SDLoc dl(Node);
EVT VT = Node->getValueType(0);
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 67d25e4eabbac..4fc4cc762c1a2 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -1365,6 +1365,12 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationAction(ISD::SUB, MVT::i32, Custom);
}
+ if (!Subtarget.useSoftFloat() && Subtarget.hasSSE2() &&
+ !Subtarget.hasSSSE3()) {
+ setOperationAction(ISD::CTLZ, MVT::v4i32, Custom);
+ setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::v4i32, Custom);
+ }
+
if (!Subtarget.useSoftFloat() && Subtarget.hasSSE41()) {
for (MVT RoundedTy : {MVT::f32, MVT::f64, MVT::v4f32, MVT::v2f64}) {
setOperationAction(ISD::FFLOOR, RoundedTy, Legal);
@@ -29548,6 +29554,13 @@ static SDValue LowerVectorCTLZ(SDValue Op, const SDLoc &DL,
if (VT.is512BitVector() && !Subtarget.hasBWI())
return splitVectorIntUnary(Op, DAG, DL);
+ if (VT == MVT::v4i32 && Subtarget.hasSSE2() && !Subtarget.hasSSSE3()) {
+ const TargetLowering &TLI = DAG.getTargetLoweringInfo();
+ SDValue New = TLI.expandCTLZWithFP(Op.getNode(), DAG);
+ if (New)
+ return New;
+ }
+
assert(Subtarget.hasSSSE3() && "Expected SSSE3 support for PSHUFB");
return LowerVectorCTLZInRegLUT(Op, DL, Subtarget, DAG);
}
diff --git a/llvm/test/CodeGen/X86/combine-srl.ll b/llvm/test/CodeGen/X86/combine-srl.ll
index 7bc90534dcc6e..40dee81d87a3c 100644
--- a/llvm/test/CodeGen/X86/combine-srl.ll
+++ b/llvm/test/CodeGen/X86/combine-srl.ll
@@ -426,44 +426,34 @@ define <4 x i32> @combine_vec_lshr_lzcnt_bit1(<4 x i32> %x) {
; SSE2-LABEL: combine_vec_lshr_lzcnt_bit1:
; SSE2: # %bb.0:
; SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[3,3,3,3]
+; SSE2-NEXT: movd %xmm1, %eax
+; SSE2-NEXT: xorps %xmm1, %xmm1
+; SSE2-NEXT: cvtsi2sd %eax, %xmm1
+; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
+; SSE2-NEXT: movd %xmm2, %eax
+; SSE2-NEXT: xorps %xmm2, %xmm2
+; SSE2-NEXT: cvtsi2sd %eax, %xmm2
+; SSE2-NEXT: unpcklpd {{.*#+}} xmm2 = xmm2[0],xmm1[0]
+; SSE2-NEXT: psrlq $52, %xmm2
+; SSE2-NEXT: movd %xmm0, %eax
+; SSE2-NEXT: xorps %xmm1, %xmm1
+; SSE2-NEXT: cvtsi2sd %eax, %xmm1
+; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,1,1]
+; SSE2-NEXT: movd %xmm3, %eax
+; SSE2-NEXT: xorps %xmm3, %xmm3
+; SSE2-NEXT: cvtsi2sd %eax, %xmm3
+; SSE2-NEXT: unpcklpd {{.*#+}} xmm1 = xmm1[0],xmm3[0]
+; SSE2-NEXT: psrlq $52, %xmm1
+; SSE2-NEXT: packssdw %xmm2, %xmm1
+; SSE2-NEXT: movdqa {{.*#+}} xmm2 = [1054,1054,1054,1054]
+; SSE2-NEXT: psubd %xmm1, %xmm2
+; SSE2-NEXT: pxor %xmm1, %xmm1
+; SSE2-NEXT: pcmpeqd %xmm1, %xmm0
; SSE2-NEXT: movdqa %xmm0, %xmm1
-; SSE2-NEXT: psrld $1, %xmm1
-; SSE2-NEXT: por %xmm1, %xmm0
-; SSE2-NEXT: movdqa %xmm0, %xmm1
-; SSE2-NEXT: psrld $2, %xmm1
-; SSE2-NEXT: por %xmm1, %xmm0
-; SSE2-NEXT: movdqa %xmm0, %xmm1
-; SSE2-NEXT: psrld $4, %xmm1
-; SSE2-NEXT: por %xmm1, %xmm0
-; SSE2-NEXT: movdqa %xmm0, %xmm1
-; SSE2-NEXT: psrld $8, %xmm1
-; SSE2-NEXT: por %xmm1, %xmm0
-; SSE2-NEXT: movdqa %xmm0, %xmm1
-; SSE2-NEXT: psrld $16, %xmm1
-; SSE2-NEXT: por %xmm1, %xmm0
-; SSE2-NEXT: pcmpeqd %xmm1, %xmm1
-; SSE2-NEXT: pxor %xmm1, %xmm0
-; SSE2-NEXT: movdqa %xmm0, %xmm1
-; SSE2-NEXT: psrlw $1, %xmm1
-; SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT: psubb %xmm1, %xmm0
-; SSE2-NEXT: movdqa {{.*#+}} xmm1 = [51,51,51,51,51,51,51,51,51,51,51,51,51,51,51,51]
-; SSE2-NEXT: movdqa %xmm0, %xmm2
-; SSE2-NEXT: pand %xmm1, %xmm2
-; SSE2-NEXT: psrlw $2, %xmm0
-; SSE2-NEXT: pand %xmm1, %xmm0
-; SSE2-NEXT: paddb %xmm2, %xmm0
-; SSE2-NEXT: movdqa %xmm0, %xmm1
-; SSE2-NEXT: psrlw $4, %xmm1
-; SSE2-NEXT: paddb %xmm1, %xmm0
+; SSE2-NEXT: pandn %xmm2, %xmm1
; SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; SSE2-NEXT: pxor %xmm1, %xmm1
-; SSE2-NEXT: movdqa %xmm0, %xmm2
-; SSE2-NEXT: punpckhdq {{.*#+}} xmm2 = xmm2[2],xmm1[2],xmm2[3],xmm1[3]
-; SSE2-NEXT: psadbw %xmm1, %xmm2
-; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; SSE2-NEXT: psadbw %xmm1, %xmm0
-; SSE2-NEXT: packuswb %xmm2, %xmm0
+; SSE2-NEXT: por %xmm1, %xmm0
; SSE2-NEXT: psrld $5, %xmm0
; SSE2-NEXT: retq
;
diff --git a/llvm/test/CodeGen/X86/expand-vp-int-intrinsics.ll b/llvm/test/CodeGen/X86/expand-vp-int-intrinsics.ll
index dbfa69d497698..328c0bac7495c 100644
--- a/llvm/test/CodeGen/X86/expand-vp-int-intrinsics.ll
+++ b/llvm/test/CodeGen/X86/expand-vp-int-intrinsics.ll
@@ -1476,44 +1476,34 @@ define <4 x i32> @vp_ctlz_v4i32(<4 x i32> %va, <4 x i1> %m, i32 zeroext %evl) {
;
; SSE-LABEL: vp_ctlz_v4i32:
; SSE: # %bb.0:
+; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[3,3,3,3]
+; SSE-NEXT: movd %xmm1, %eax
+; SSE-NEXT: xorps %xmm1, %xmm1
+; SSE-NEXT: cvtsi2sd %rax, %xmm1
+; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
+; SSE-NEXT: movd %xmm2, %eax
+; SSE-NEXT: xorps %xmm2, %xmm2
+; SSE-NEXT: cvtsi2sd %rax, %xmm2
+; SSE-NEXT: unpcklpd {{.*#+}} xmm2 = xmm2[0],xmm1[0]
+; SSE-NEXT: psrlq $52, %xmm2
+; SSE-NEXT: movd %xmm0, %eax
+; SSE-NEXT: xorps %xmm1, %xmm1
+; SSE-NEXT: cvtsi2sd %rax, %xmm1
+; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,1,1]
+; SSE-NEXT: movd %xmm3, %eax
+; SSE-NEXT: xorps %xmm3, %xmm3
+; SSE-NEXT: cvtsi2sd %rax, %xmm3
+; SSE-NEXT: unpcklpd {{.*#+}} xmm1 = xmm1[0],xmm3[0]
+; SSE-NEXT: psrlq $52, %xmm1
+; SSE-NEXT: packssdw %xmm2, %xmm1
+; SSE-NEXT: movdqa {{.*#+}} xmm2 = [1054,1054,1054,1054]
+; SSE-NEXT: psubd %xmm1, %xmm2
+; SSE-NEXT: pxor %xmm1, %xmm1
+; SSE-NEXT: pcmpeqd %xmm1, %xmm0
; SSE-NEXT: movdqa %xmm0, %xmm1
-; SSE-NEXT: psrld $1, %xmm1
-; SSE-NEXT: por %xmm1, %xmm0
-; SSE-NEXT: movdqa %xmm0, %xmm1
-; SSE-NEXT: psrld $2, %xmm1
-; SSE-NEXT: por %xmm1, %xmm0
-; SSE-NEXT: movdqa %xmm0, %xmm1
-; SSE-NEXT: psrld $4, %xmm1
-; SSE-NEXT: por %xmm1, %xmm0
-; SSE-NEXT: movdqa %xmm0, %xmm1
-; SSE-NEXT: psrld $8, %xmm1
-; SSE-NEXT: por %xmm1, %xmm0
-; SSE-NEXT: movdqa %xmm0, %xmm1
-; SSE-NEXT: psrld $16, %xmm1
-; SSE-NEXT: por %xmm1, %xmm0
-; SSE-NEXT: pcmpeqd %xmm1, %xmm1
-; SSE-NEXT: pxor %xmm1, %xmm0
-; SSE-NEXT: movdqa %xmm0, %xmm1
-; SSE-NEXT: psrlw $1, %xmm1
-; SSE-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE-NEXT: psubb %xmm1, %xmm0
-; SSE-NEXT: movdqa {{.*#+}} xmm1 = [51,51,51,51,51,51,51,51,51,51,51,51,51,51,51,51]
-; SSE-NEXT: movdqa %xmm0, %xmm2
-; SSE-NEXT: pand %xmm1, %xmm2
-; SSE-NEXT: psrlw $2, %xmm0
-; SSE-NEXT: pand %xmm1, %xmm0
-; SSE-NEXT: paddb %xmm2, %xmm0
-; SSE-NEXT: movdqa %xmm0, %xmm1
-; SSE-NEXT: psrlw $4, %xmm1
-; SSE-NEXT: paddb %xmm1, %xmm0
+; SSE-NEXT: pandn %xmm2, %xmm1
; SSE-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; SSE-NEXT: pxor %xmm1, %xmm1
-; SSE-NEXT: movdqa %xmm0, %xmm2
-; SSE-NEXT: punpckhdq {{.*#+}} xmm2 = xmm2[2],xmm1[2],xmm2[3],xmm1[3]
-; SSE-NEXT: psadbw %xmm1, %xmm2
-; SSE-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; SSE-NEXT: psadbw %xmm1, %xmm0
-; SSE-NEXT: packuswb %xmm2, %xmm0
+; SSE-NEXT: por %xmm1, %xmm0
; SSE-NEXT: retq
;
; AVX1-LABEL: vp_ctlz_v4i32:
diff --git a/llvm/test/CodeGen/X86/vec_ctbits.ll b/llvm/test/CodeGen/X86/vec_ctbits.ll
index 370f88d644b57..0666e6316885e 100644
--- a/llvm/test/CodeGen/X86/vec_ctbits.ll
+++ b/llvm/test/CodeGen/X86/vec_ctbits.ll
@@ -140,44 +140,34 @@ define <2 x i32> @promtz(<2 x i32> %a) nounwind {
define <2 x i32> @promlz(<2 x i32> %a) nounwind {
; CHECK-LABEL: promlz:
; CHECK: # %bb.0:
+; CHECK-NEXT: pshufd {{.*#+}} xmm1 = xmm0[3,3,3,3]
+; CHECK-NEXT: movd %xmm1, %eax
+; CHECK-NEXT: xorps %xmm1, %xmm1
+; CHECK-NEXT: cvtsi2sd %rax, %xmm1
+; CHECK-NEXT: pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
+; CHECK-NEXT: movd %xmm2, %eax
+; CHECK-NEXT: xorps %xmm2, %xmm2
+; CHECK-NEXT: cvtsi2sd %rax, %xmm2
+; CHECK-NEXT: unpcklpd {{.*#+}} xmm2 = xmm2[0],xmm1[0]
+; CHECK-NEXT: psrlq $52, %xmm2
+; CHECK-NEXT: movd %xmm0, %eax
+; CHECK-NEXT: xorps %xmm1, %xmm1
+; CHECK-NEXT: cvtsi2sd %rax, %xmm1
+; CHECK-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,1,1]
+; CHECK-NEXT: movd %xmm3, %eax
+; CHECK-NEXT: xorps %xmm3, %xmm3
+; CHECK-NEXT: cvtsi2sd %rax, %xmm3
+; CHECK-NEXT: unpcklpd {{.*#+}} xmm1 = xmm1[0],xmm3[0]
+; CHECK-NEXT: psrlq $52, %xmm1
+; CHECK-NEXT: packssdw %xmm2, %xmm1
+; CHECK-NEXT: movdqa {{.*#+}} xmm2 = [1054,1054,1054,1054]
+; CHECK-NEXT: psubd %xmm1, %xmm2
+; CHECK-NEXT: pxor %xmm1, %xmm1
+; CHECK-NEXT: pcmpeqd %xmm1, %xmm0
; CHECK-NEXT: movdqa %xmm0, %xmm1
-; CHECK-NEXT: psrld $1, %xmm1
-; CHECK-NEXT: por %xmm1, %xmm0
-; CHECK-NEXT: movdqa %xmm0, %xmm1
-; CHECK-NEXT: psrld $2, %xmm1
-; CHECK-NEXT: por %xmm1, %xmm0
-; CHECK-NEXT: movdqa %xmm0, %xmm1
-; CHECK-NEXT: psrld $4, %xmm1
-; CHECK-NEXT: por %xmm1, %xmm0
-; CHECK-NEXT: movdqa %xmm0, %xmm1
-; CHECK-NEXT: psrld $8, %xmm1
-; CHECK-NEXT: por %xmm1, %xmm0
-; CHECK-NEXT: movdqa %xmm0, %xmm1
-; CHECK-NEXT: psrld $16, %xmm1
-; CHECK-NEXT: por %xmm1, %xmm0
-; CHECK-NEXT: pcmpeqd %xmm1, %xmm1
-; CHECK-NEXT: pxor %xmm1, %xmm0
-; CHECK-NEXT: movdqa %xmm0, %xmm1
-; CHECK-NEXT: psrlw $1, %xmm1
-; CHECK-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; CHECK-NEXT: psubb %xmm1, %xmm0
-; CHECK-NEXT: movdqa {{.*#+}} xmm1 = [51,51,51,51,51,51,51,51,51,51,51,51,51,51,51,51]
-; CHECK-NEXT: movdqa %xmm0, %xmm2
-; CHECK-NEXT: pand %xmm1, %xmm2
-; CHECK-NEXT: psrlw $2, %xmm0
-; CHECK-NEXT: pand %xmm1, %xmm0
-; CHECK-NEXT: paddb %xmm2, %xmm0
-; CHECK-NEXT: movdqa %xmm0, %xmm1
-; CHECK-NEXT: psrlw $4, %xmm1
-; CHECK-NEXT: paddb %xmm1, %xmm0
+; CHECK-NEXT: pandn %xmm2, %xmm1
; CHECK-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; CHECK-NEXT: pxor %xmm1, %xmm1
-; CHECK-NEXT: movdqa %xmm0, %xmm2
-; CHECK-NEXT: punpckhdq {{.*#+}} xmm2 = xmm2[2],xmm1[2],xmm2[3],xmm1[3]
-; CHECK-NEXT: psadbw %xmm1, %xmm2
-; CHECK-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; CHECK-NEXT: psadbw %xmm1, %xmm0
-; CHECK-NEXT: packuswb %xmm2, %xmm0
+; CHECK-NEXT: por %xmm1, %xmm0
; CHECK-NEXT: retq
%c = call <2 x i32> @llvm.ctlz.v2i32(<2 x i32> %a, i1 false)
ret <2 x i32> %c
diff --git a/llvm/test/CodeGen/X86/vector-lzcnt-128.ll b/llvm/test/CodeGen/X86/vector-lzcnt-128.ll
index cfb5fac2fd7aa..380e8ca9e26c1 100644
--- a/llvm/test/CodeGen/X86/vector-lzcnt-128.ll
+++ b/llvm/test/CodeGen/X86/vector-lzcnt-128.ll
@@ -572,86 +572,66 @@ define <2 x i64> @testv2i64u(<2 x i64> %in) nounwind {
define <4 x i32> @testv4i32(<4 x i32> %in) nounwind {
; SSE2-LABEL: testv4i32:
; SSE2: # %bb.0:
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[3,3,3,3]
+; SSE2-NEXT: movd %xmm1, %eax
+; SSE2-NEXT: xorps %xmm1, %xmm1
+; SSE2-NEXT: cvtsi2sd %rax, %xmm1
+; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
+; SSE2-NEXT: movd %xmm2, %eax
+; SSE2-NEXT: xorps %xmm2, %xmm2
+; SSE2-NEXT: cvtsi2sd %rax, %xmm2
+; SSE2-NEXT: unpcklpd {{.*#+}} xmm2 = xmm2[0],xmm1[0]
+; SSE2-NEXT: psrlq $52, %xmm2
+; SSE2-NEXT: movd %xmm0, %eax
+; SSE2-NEXT: xorps %xmm1, %xmm1
+; SSE2-NEXT: cvtsi2sd %rax, %xmm1
+; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,1,1]
+; SSE2-NEXT: movd %xmm3, %eax
+; SSE2-NEXT: xorps %xmm3, %xmm3
+; SSE2-NEXT: cvtsi2sd %rax, %xmm3
+; SSE2-NEXT: unpcklpd {{.*#+}} xmm1 = xmm1[0],xmm3[0]
+; SSE2-NEXT: psrlq $52, %xmm1
+; SSE2-NEXT: packssdw %xmm2, %xmm1
+; SSE2-NEXT: movdqa {{.*#+}} xmm2 = [1054,1054,1054,1054]
+; SSE2-NEXT: psubd %xmm1, %xmm2
+; SSE2-NEXT: pxor %xmm1, %xmm1
+; SSE2-NEXT: pcmpeqd %xmm1, %xmm0
; SSE2-NEXT: movdqa %xmm0, %xmm1
-; SSE2-NEXT: psrld $1, %xmm1
-; SSE2-NEXT: por %xmm1, %xmm0
-; SSE2-NEXT: movdqa %xmm0, %xmm1
-; SSE2-NEXT: psrld $2, %xmm1
-; SSE2-NEXT: por %xmm1, %xmm0
-; SSE2-NEXT: movdqa %xmm0, %xmm1
-; SSE2-NEXT: psrld $4, %xmm1
-; SSE2-NEXT: por %xmm1, %xmm0
-; SSE2-NEXT: movdqa %xmm0, %xmm1
-; SSE2-NEXT: psrld $8, %xmm1
-; SSE2-NEXT: por %xmm1, %xmm0
-; SSE2-NEXT: movdqa %xmm0, %xmm1
-; SSE2-NEXT: psrld $16, %xmm1
-; SSE2-NEXT: por %xmm1, %xmm0
-; SSE2-NEXT: pcmpeqd %xmm1, %xmm1
-; SSE2-NEXT: pxor %xmm1, %xmm0
-; SSE2-NEXT: movdqa %xmm0, %xmm1
-; SSE2-NEXT: psrlw $1, %xmm1
-; SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT: psubb %xmm1, %xmm0
-; SSE2-NEXT: movdqa {{.*#+}} xmm1 = [51,51,51,51,51,51,51,51,51,51,51,51,51,51,51,51]
-; SSE2-NEXT: movdqa %xmm0, %xmm2
-; SSE2-NEXT: pand %xmm1, %xmm2
-; SSE2-NEXT: psrlw $2, %xmm0
-; SSE2-NEXT: pand %xmm1, %xmm0
-; SSE2-NEXT: paddb %xmm2, %xmm0
-; SSE2-NEXT: movdqa %xmm0, %xmm1
-; SSE2-NEXT: psrlw $4, %xmm1
-; SSE2-NEXT: paddb %xmm1, %xmm0
+; SSE2-NEXT: pandn %xmm2, %xmm1
; SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; SSE2-NEXT: pxor %xmm1, %xmm1
-; SSE2-NEXT: movdqa %xmm0, %xmm2
-; SSE2-NEXT: punpckhdq {{.*#+}} xmm2 = xmm2[2],xmm1[2],xmm2[3],xmm1[3]
-; SSE2-NEXT: psadbw %xmm1, %xmm2
-; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; SSE2-NEXT: psadbw %xmm1, %xmm0
-; SSE2-NEXT: packuswb %xmm2, %xmm0
+; SSE2-NEXT: por %xmm1, %xmm0
; SSE2-NEXT: retq
;
; SSE3-LABEL: testv4i32:
; SSE3: # %bb.0:
+; SSE3-NEXT: pshufd {{.*#+}} xmm1 = xmm0[3,3,3,3]
+; SSE3-NEXT: movd %xmm1, %eax
+; SSE3-NEXT: xorps %xmm1, %xmm1
+; SSE3-NEXT: cvtsi2sd %rax, %xmm1
+; SSE3-NEXT: pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
+; SSE3-NEXT: movd %xmm2, %eax
+; SSE3-NEXT: xorps %xmm2, %xmm2
+; SSE3-NEXT: cvtsi2sd %rax, %xmm2
+; SSE3-NEXT: unpcklpd {{.*#+}} xmm2 = xmm2[0],xmm1[0]
+; SSE3-NEXT: psrlq $52, %xmm2
+; SSE3-NEXT: movd %xmm0, %eax
+; SSE3-NEXT: xorps %xmm1, %xmm1
+; SSE3-NEXT: cvtsi2sd %rax, %xmm1
+; SSE3-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,1,1]
+; SSE3-NEXT: movd %xmm3, %eax
+; SSE3-NEXT: xorps %xmm3, %xmm3
+; SSE3-NEXT: cvtsi2sd %rax, %xmm3
+; SSE3-NEXT: unpcklpd {{.*#+}} xmm1 = xmm1[0],xmm3[0]
+; SSE3-NEXT: psrlq $52, %xmm1
+; SSE3-NEXT: packssdw %xmm2, %xmm1
+; SSE3-NEXT: movdqa {{.*#+}} xmm2 = [1054,1054,1054,1054]
+; SSE3-NEXT: psubd %xmm1, %xmm2
+; SSE3-NEXT: pxor %xmm1, %xmm1
+; SSE3-NEXT: pcmpeqd %xmm1, %xmm0
; SSE3-NEXT: movdqa %xmm0, %xmm1
-; SSE3-NEXT: psrld $1, %xmm1
-; SSE3-NEXT: por %xmm1, %xmm0
-; SSE3-NEXT: movdqa %xmm0, %xmm1
-; SSE3-NEXT: psrld $2, %xmm1
-; SSE3-NEXT: por %xmm1, %xmm0
-; SSE3-NEXT: movdqa %xmm0, %xmm1
-; SSE3-NEXT: psrld $4, %xmm1
-; SSE3-NEXT: por %xmm1, %xmm0
-; SSE3-NEXT: movdqa %xmm0, %xmm1
-; SSE3-NEXT: psrld $8, %xmm1
-; SSE3-NEXT: por %xmm1, %xmm0
-; SSE3-NEXT: movdqa %xmm0, %xmm1
-; SSE3-NEXT: psrld $16, %xmm1
-; SSE3-NEXT: por %xmm1, %xmm0
-; SSE3-NEXT: pcmpeqd %xmm1, %xmm1
-; SSE3-NEXT: pxor %xmm1, %xmm0
-; SSE3-NEXT: movdqa %xmm0, %xmm1
-; SSE3-NEXT: psrlw $1, %xmm1
-; SSE3-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT: psubb %xmm1, %xmm0
-; SSE3-NEXT: movdqa {{.*#+}} xmm1 = [51,51,51,51,51,51,51,51,51,51,51,51,51,51,51,51]
-; SSE3-NEXT: movdqa %xmm0, %xmm2
-; SSE3-NEXT: pand %xmm1, %xmm2
-; SSE3-NEXT: psrlw $2, %xmm0
-; SSE3-NEXT: pand %xmm1, %xmm0
-; SSE3-NEXT: paddb %xmm2, %xmm0
-; SSE3-NEXT: movdqa %xmm0, %xmm1
-; SSE3-NEXT: psrlw $4, %xmm1
-; SSE3-NEXT: paddb %xmm1, %xmm0
+; SSE3-NEXT: pandn %xmm2, %xmm1
; SSE3-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; SSE3-NEXT: pxor %xmm1, %xmm1
-; SSE3-NEXT: movdqa %xmm0, %xmm2
-; SSE3-NEXT: punpckhdq {{.*#+}} xmm2 = xmm2[2],xmm1[2],xmm2[3],xmm1[3]
-; SSE3-NEXT: psadbw %xmm1, %xmm2
-; SSE3-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; SSE3-NEXT: psadbw %xmm1, %xmm0
-; SSE3-NEXT: packuswb %xmm2, %xmm0
+; SSE3-NEXT: por %xmm1, %xmm0
; SSE3-NEXT: retq
;
; SSSE3-LABEL: testv4i32:
@@ -821,86 +801,54 @@ define <4 x i32> @testv4i32(<4 x i32> %in) nounwind {
define <4 x i32> @testv4i32u(<4 x i32> %in) nounwind {
; SSE2-LABEL: testv4i32u:
; SSE2: # %bb.0:
-; SSE2-NEXT: movdqa %xmm0, %xmm1
-; SSE2-NEXT: psrld $1, %xmm1
-; SSE2-NEXT: por %xmm1, %xmm0
-; SSE2-NEXT: movdqa %xmm0, %xmm1
-; SSE2-NEXT: psrld $2, %xmm1
-; SSE2-NEXT: por %xmm1, %xmm0
-; SSE2-NEXT: movdqa %xmm0, %xmm1
-; SSE2-NEXT: psrld $4, %xmm1
-; SSE2-NEXT: por %xmm1, %xmm0
-; SSE2-NEXT: movdqa %xmm0, %xmm1
-; SSE2-NEXT: psrld $8, %xmm1
-; SSE2-NEXT: por %xmm1, %xmm0
-; SSE2-NEXT: movdqa %xmm0, %xmm1
-; SSE2-NEXT: psrld $16, %xmm1
-; SSE2-NEXT: por %xmm1, %xmm0
-; SSE2-NEXT: pcmpeqd %xmm1, %xmm1
-; SSE2-NEXT: pxor %xmm1, %xmm0
-; SSE2-NEXT: movdqa %xmm0, %xmm1
-; SSE2-NEXT: psrlw $1, %xmm1
-; SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT: psubb %xmm1, %xmm0
-; SSE2-NEXT: movdqa {{.*#+}} xmm1 = [51,51,51,51,51,51,51,51,51,51,51,51,51,51,51,51]
-; SSE2-NEXT: movdqa %xmm0, %xmm2
-; SSE2-NEXT: pand %xmm1, %xmm2
-; SSE2-NEXT: psrlw $2, %xmm0
-; SSE2-NEXT: pand %xmm1, %xmm0
-; SSE2-NEXT: paddb %xmm2, %xmm0
-; SSE2-NEXT: movdqa %xmm0, %xmm1
-; SSE2-NEXT: psrlw $4, %xmm1
-; SSE2-NEXT: paddb %xmm1, %xmm0
-; SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; SSE2-NEXT: pxor %xmm1, %xmm1
-; SSE2-NEXT: movdqa %xmm0, %xmm2
-; SSE2-NEXT: punpckhdq {{.*#+}} xmm2 = xmm2[2],xmm1[2],xmm2[3],xmm1[3]
-; SSE2-NEXT: psadbw %xmm1, %xmm2
-; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; SSE2-NEXT: psadbw %xmm1, %xmm0
-; SSE2-NEXT: packuswb %xmm2, %xmm0
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[3,3,3,3]
+; SSE2-NEXT: movd %xmm1, %eax
+; SSE2-NEXT: xorps %xmm1, %xmm1
+; SSE2-NEXT: cvtsi2sd %rax, %xmm1
+; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
+; SSE2-NEXT: movd %xmm2, %eax
+; SSE2-NEXT: xorps %xmm2, %xmm2
+; SSE2-NEXT: cvtsi2sd %rax, %xmm2
+; SSE2-NEXT: unpcklpd {{.*#+}} xmm2 = xmm2[0],xmm1[0]
+; SSE2-NEXT: psrlq $52, %xmm2
+; SSE2-NEXT: movd %xmm0, %eax
+; SSE2-NEXT: xorps %xmm1, %xmm1
+; SSE2-NEXT: cvtsi2sd %rax, %xmm1
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
+; SSE2-NEXT: movd %xmm0, %eax
+; SSE2-NEXT: xorps %xmm0, %xmm0
+; SSE2-NEXT: cvtsi2sd %rax, %xmm0
+; SSE2-NEXT: unpcklpd {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; SSE2-NEXT: psrlq $52, %xmm1
+; SSE2-NEXT: packssdw %xmm2, %xmm1
+; SSE2-NEXT: movdqa {{.*#+}} xmm0 = [1054,1054,1054,1054]
+; SSE2-NEXT: psubd %xmm1, %xmm0
; SSE2-NEXT: retq
;
; SSE3-LABEL: testv4i32u:
; SSE3: # %bb.0:
-; SSE3-NEXT: movdqa %xmm0, %xmm1
-; SSE3-NEXT: psrld $1, %xmm1
-; SSE3-NEXT: por %xmm1, %xmm0
-; SSE3-NEXT: movdqa %xmm0, %xmm1
-; SSE3-NEXT: psrld $2, %xmm1
-; SSE3-NEXT: por %xmm1, %xmm0
-; SSE3-NEXT: movdqa %xmm0, %xmm1
-; SSE3-NEXT: psrld $4, %xmm1
-; SSE3-NEXT: por %xmm1, %xmm0
-; SSE3-NEXT: movdqa %xmm0, %xmm1
-; SSE3-NEXT: psrld $8, %xmm1
-; SSE3-NEXT: por %xmm1, %xmm0
-; SSE3-NEXT: movdqa %xmm0, %xmm1
-; SSE3-NEXT: psrld $16, %xmm1
-; SSE3-NEXT: por %xmm1, %xmm0
-; SSE3-NEXT: pcmpeqd %xmm1, %xmm1
-; SSE3-NEXT: pxor %xmm1, %xmm0
-; SSE3-NEXT: movdqa %xmm0, %xmm1
-; SSE3-NEXT: psrlw $1, %xmm1
-; SSE3-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT: psubb %xmm1, %xmm0
-; SSE3-NEXT: movdqa {{.*#+}} xmm1 = [51,51,51,51,51,51,51,51,51,51,51,51,51,51,51,51]
-; SSE3-NEXT: movdqa %xmm0, %xmm2
-; SSE3-NEXT: pand %xmm1, %xmm2
-; SSE3-NEXT: psrlw $2, %xmm0
-; SSE3-NEXT: pand %xmm1, %xmm0
-; SSE3-NEXT: paddb %xmm2, %xmm0
-; SSE3-NEXT: movdqa %xmm0, %xmm1
-; SSE3-NEXT: psrlw $4, %xmm1
-; SSE3-NEXT: paddb %xmm1, %xmm0
-; SSE3-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; SSE3-NEXT: pxor %xmm1, %xmm1
-; SSE3-NEXT: movdqa %xmm0, %xmm2
-; SSE3-NEXT: punpckhdq {{.*#+}} xmm2 = xmm2[2],xmm1[2],xmm2[3],xmm1[3]
-; SSE3-NEXT: psadbw %xmm1, %xmm2
-; SSE3-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; SSE3-NEXT: psadbw %xmm1, %xmm0
-; SSE3-NEXT: packuswb %xmm2, %xmm0
+; SSE3-NEXT: pshufd {{.*#+}} xmm1 = xmm0[3,3,3,3]
+; SSE3-NEXT: movd %xmm1, %eax
+; SSE3-NEXT: xorps %xmm1, %xmm1
+; SSE3-NEXT: cvtsi2sd %rax, %xmm1
+; SSE3-NEXT: pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
+; SSE3-NEXT: movd %xmm2, %eax
+; SSE3-NEXT: xorps %xmm2, %xmm2
+; SSE3-NEXT: cvtsi2sd %rax, %xmm2
+; SSE3-NEXT: unpcklpd {{.*#+}} xmm2 = xmm2[0],xmm1[0]
+; SSE3-NEXT: psrlq $52, %xmm2
+; SSE3-NEXT: movd %xmm0, %eax
+; SSE3-NEXT: xorps %xmm1, %xmm1
+; SSE3-NEXT: cvtsi2sd %rax, %xmm1
+; SSE3-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
+; SSE3-NEXT: movd %xmm0, %eax
+; SSE3-NEXT: xorps %xmm0, %xmm0
+; SSE3-NEXT: cvtsi2sd %rax, %xmm0
+; SSE3-NEXT: unpcklpd {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; SSE3-NEXT: psrlq $52, %xmm1
+; SSE3-NEXT: packssdw %xmm2, %xmm1
+; SSE3-NEXT: movdqa {{.*#+}} xmm0 = [1054,1054,1054,1054]
+; SSE3-NEXT: psubd %xmm1, %xmm0
; SSE3-NEXT: retq
;
; SSSE3-LABEL: testv4i32u:
diff --git a/llvm/test/CodeGen/X86/vector-lzcnt-sub128.ll b/llvm/test/CodeGen/X86/vector-lzcnt-sub128.ll
index a1b277efde6ff..f70f7e204c86a 100644
--- a/llvm/test/CodeGen/X86/vector-lzcnt-sub128.ll
+++ b/llvm/test/CodeGen/X86/vector-lzcnt-sub128.ll
@@ -6,44 +6,28 @@ declare <2 x i32> @llvm.ctlz.v2i32(<2 x i32>, i1 immarg)
define <2 x i32> @illegal_ctlz(<2 x i32> %v1) {
; CHECK-LABEL: illegal_ctlz:
; CHECK: # %bb.0:
-; CHECK-NEXT: movdqa %xmm0, %xmm1
-; CHECK-NEXT: psrld $1, %xmm1
-; CHECK-NEXT: por %xmm1, %xmm0
-; CHECK-NEXT: movdqa %xmm0, %xmm1
-; CHECK-NEXT: psrld $2, %xmm1
-; CHECK-NEXT: por %xmm1, %xmm0
-; CHECK-NEXT: movdqa %xmm0, %xmm1
-; CHECK-NEXT: psrld $4, %xmm1
-; CHECK-NEXT: por %xmm1, %xmm0
-; CHECK-NEXT: movdqa %xmm0, %xmm1
-; CHECK-NEXT: psrld $8, %xmm1
-; CHECK-NEXT: por %xmm1, %xmm0
-; CHECK-NEXT: movdqa %xmm0, %xmm1
-; CHECK-NEXT: psrld $16, %xmm1
-; CHECK-NEXT: por %xmm1, %xmm0
-; CHECK-NEXT: pcmpeqd %xmm1, %xmm1
-; CHECK-NEXT: pxor %xmm1, %xmm0
-; CHECK-NEXT: movdqa %xmm0, %xmm1
-; CHECK-NEXT: psrlw $1, %xmm1
-; CHECK-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; CHECK-NEXT: psubb %xmm1, %xmm0
-; CHECK-NEXT: movdqa {{.*#+}} xmm1 = [51,51,51,51,51,51,51,51,51,51,51,51,51,51,51,51]
-; CHECK-NEXT: movdqa %xmm0, %xmm2
-; CHECK-NEXT: pand %xmm1, %xmm2
-; CHECK-NEXT: psrlw $2, %xmm0
-; CHECK-NEXT: pand %xmm1, %xmm0
-; CHECK-NEXT: paddb %xmm2, %xmm0
-; CHECK-NEXT: movdqa %xmm0, %xmm1
-; CHECK-NEXT: psrlw $4, %xmm1
-; CHECK-NEXT: paddb %xmm1, %xmm0
-; CHECK-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; CHECK-NEXT: pxor %xmm1, %xmm1
-; CHECK-NEXT: movdqa %xmm0, %xmm2
-; CHECK-NEXT: punpckhdq {{.*#+}} xmm2 = xmm2[2],xmm1[2],xmm2[3],xmm1[3]
-; CHECK-NEXT: psadbw %xmm1, %xmm2
-; CHECK-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; CHECK-NEXT: psadbw %xmm1, %xmm0
-; CHECK-NEXT: packuswb %xmm2, %xmm0
+; CHECK-NEXT: pshufd {{.*#+}} xmm1 = xmm0[3,3,3,3]
+; CHECK-NEXT: movd %xmm1, %eax
+; CHECK-NEXT: xorps %xmm1, %xmm1
+; CHECK-NEXT: cvtsi2sd %rax, %xmm1
+; CHECK-NEXT: pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
+; CHECK-NEXT: movd %xmm2, %eax
+; CHECK-NEXT: xorps %xmm2, %xmm2
+; CHECK-NEXT: cvtsi2sd %rax, %xmm2
+; CHECK-NEXT: unpcklpd {{.*#+}} xmm2 = xmm2[0],xmm1[0]
+; CHECK-NEXT: psrlq $52, %xmm2
+; CHECK-NEXT: movd %xmm0, %eax
+; CHECK-NEXT: xorps %xmm1, %xmm1
+; CHECK-NEXT: cvtsi2sd %rax, %xmm1
+; CHECK-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
+; CHECK-NEXT: movd %xmm0, %eax
+; CHECK-NEXT: xorps %xmm0, %xmm0
+; CHECK-NEXT: cvtsi2sd %rax, %xmm0
+; CHECK-NEXT: unpcklpd {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; CHECK-NEXT: psrlq $52, %xmm1
+; CHECK-NEXT: packssdw %xmm2, %xmm1
+; CHECK-NEXT: movdqa {{.*#+}} xmm0 = [1054,1054,1054,1054]
+; CHECK-NEXT: psubd %xmm1, %xmm0
; CHECK-NEXT: retq
%v2 = call <2 x i32> @llvm.ctlz.v2i32(<2 x i32> %v1, i1 true)
ret <2 x i32> %v2
>From 5fc07c6d700f65940366d185f4651577ea02fda4 Mon Sep 17 00:00:00 2001
From: Ayaan Khan <ayaanandayaan2 at gmail.com>
Date: Thu, 9 Apr 2026 01:19:48 +0530
Subject: [PATCH 2/3] Apply suggestions
---
.../CodeGen/SelectionDAG/TargetLowering.cpp | 47 +++---
llvm/lib/Target/X86/X86ISelLowering.cpp | 14 +-
llvm/test/CodeGen/X86/combine-srl.ll | 36 ++---
.../CodeGen/X86/expand-vp-int-intrinsics.ll | 34 ++---
llvm/test/CodeGen/X86/vec_ctbits.ll | 34 ++---
llvm/test/CodeGen/X86/vector-lzcnt-128.ll | 138 +++++++-----------
llvm/test/CodeGen/X86/vector-lzcnt-sub128.ll | 35 ++---
7 files changed, 135 insertions(+), 203 deletions(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
index fe9046c8e0ac9..b54bcc9a6fc0e 100644
--- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
@@ -9893,42 +9893,37 @@ SDValue TargetLowering::expandCTLZWithFP(SDNode *Node,
EVT VT = Node->getValueType(0);
SDValue Op = Node->getOperand(0);
- assert(VT.isVector() && "This expansion is intended for vectors");
EVT EltVT = VT.getVectorElementType();
- if (EltVT != MVT::i32) {
+ if (EltVT != MVT::i32)
return SDValue();
- }
- EVT FloatVT = VT.changeVectorElementType(*DAG.getContext(), MVT::f64);
- const fltSemantics &Sem = FloatVT.getVectorElementType().getFltSemantics();
+ EVT HalfVT = VT.getHalfNumVectorElementsVT(*DAG.getContext());
+ EVT HalfFloatVT = HalfVT.changeVectorElementType(*DAG.getContext(), MVT::f64);
+ EVT HalfFloatBitsVT = HalfFloatVT.changeVectorElementTypeToInteger();
+
+ const fltSemantics &Sem = HalfFloatVT.getVectorElementType().getFltSemantics();
unsigned BitWidth = EltVT.getSizeInBits();
unsigned MantissaBits = APFloat::semanticsPrecision(Sem);
unsigned ExponentBias = APFloat::semanticsMaxExponent(Sem);
- unsigned NumElts = VT.getVectorNumElements();
- SmallVector<SDValue, 4> FloatElts;
- for (unsigned i = 0; i < NumElts; i++) {
- SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op,
- DAG.getIntPtrConstant(i, dl));
- SDValue FElt = DAG.getNode(ISD::UINT_TO_FP, dl, MVT::f64, Elt);
- FloatElts.push_back(FElt);
- }
- SDValue Float = DAG.getBuildVector(FloatVT, dl, FloatElts);
-
- EVT FloatBitsVT = FloatVT.changeVectorElementTypeToInteger();
- SDValue FloatBits = DAG.getNode(ISD::BITCAST, dl, FloatBitsVT, Float);
- SDValue Exp = DAG.getNode(
- ISD::SRL, dl, FloatBitsVT, FloatBits,
- DAG.getShiftAmountConstant(MantissaBits - 1, FloatBitsVT, dl));
- SDValue ExpTrunc = DAG.getNode(ISD::TRUNCATE, dl, VT, Exp);
- SDValue NonZeroRes = DAG.getNode(
- ISD::SUB, dl, VT, DAG.getConstant(BitWidth - 1 + ExponentBias, dl, VT),
- ExpTrunc);
+ auto ComputeExp = [&](SDValue Half) {
+ SDValue Float = DAG.getNode(ISD::UINT_TO_FP, dl, HalfFloatVT, Half);
+ SDValue Bits = DAG.getBitcast(HalfFloatBitsVT, Float);
+ SDValue Exp = DAG.getNode(ISD::SRL, dl, HalfFloatBitsVT, Bits,
+ DAG.getShiftAmountConstant(MantissaBits - 1, HalfFloatBitsVT, dl));
+ return DAG.getNode(ISD::TRUNCATE, dl, HalfVT, Exp);
+ };
+
+ SDValue ExpTruncLo = ComputeExp(DAG.getExtractSubvector(dl, HalfVT, Op, 0));
+ SDValue ExpTruncHi = ComputeExp(
+ DAG.getExtractSubvector(dl, HalfVT, Op, VT.getVectorNumElements() / 2));
+ SDValue Exp = DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, ExpTruncLo, ExpTruncHi);
+ SDValue NonZeroRes = DAG.getNode(ISD::SUB, dl, VT,
+ DAG.getConstant(BitWidth - 1 + ExponentBias, dl, VT), Exp);
// Skip Op == 0 case for CTLZ_ZERO_UNDEF
- if (Node->getOpcode() == ISD::CTLZ_ZERO_UNDEF) {
+ if (Node->getOpcode() == ISD::CTLZ_ZERO_UNDEF)
return NonZeroRes;
- }
// This Handles the Op == 0 case
EVT CmpVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 4fc4cc762c1a2..4c06ad5d2512c 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -1331,6 +1331,11 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationAction(ISD::STRICT_FSUB, MVT::v2f64, Legal);
setOperationAction(ISD::STRICT_FMUL, MVT::v2f64, Legal);
setOperationAction(ISD::STRICT_FDIV, MVT::v2f64, Legal);
+
+ if (!Subtarget.hasSSSE3()) {
+ setOperationAction(ISD::CTLZ, MVT::v4i32, Custom);
+ setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::v4i32, Custom);
+ }
}
if (!Subtarget.useSoftFloat() && Subtarget.hasGFNI()) {
@@ -1365,12 +1370,6 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationAction(ISD::SUB, MVT::i32, Custom);
}
- if (!Subtarget.useSoftFloat() && Subtarget.hasSSE2() &&
- !Subtarget.hasSSSE3()) {
- setOperationAction(ISD::CTLZ, MVT::v4i32, Custom);
- setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::v4i32, Custom);
- }
-
if (!Subtarget.useSoftFloat() && Subtarget.hasSSE41()) {
for (MVT RoundedTy : {MVT::f32, MVT::f64, MVT::v4f32, MVT::v2f64}) {
setOperationAction(ISD::FFLOOR, RoundedTy, Legal);
@@ -29556,8 +29555,7 @@ static SDValue LowerVectorCTLZ(SDValue Op, const SDLoc &DL,
if (VT == MVT::v4i32 && Subtarget.hasSSE2() && !Subtarget.hasSSSE3()) {
const TargetLowering &TLI = DAG.getTargetLoweringInfo();
- SDValue New = TLI.expandCTLZWithFP(Op.getNode(), DAG);
- if (New)
+ if (SDValue New = TLI.expandCTLZWithFP(Op.getNode(), DAG))
return New;
}
diff --git a/llvm/test/CodeGen/X86/combine-srl.ll b/llvm/test/CodeGen/X86/combine-srl.ll
index 40dee81d87a3c..ec56005082861 100644
--- a/llvm/test/CodeGen/X86/combine-srl.ll
+++ b/llvm/test/CodeGen/X86/combine-srl.ll
@@ -425,30 +425,22 @@ define <4 x i32> @combine_vec_lshr_lzcnt_bit0(<4 x i32> %x) {
define <4 x i32> @combine_vec_lshr_lzcnt_bit1(<4 x i32> %x) {
; SSE2-LABEL: combine_vec_lshr_lzcnt_bit1:
; SSE2: # %bb.0:
-; SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[3,3,3,3]
-; SSE2-NEXT: movd %xmm1, %eax
-; SSE2-NEXT: xorps %xmm1, %xmm1
-; SSE2-NEXT: cvtsi2sd %eax, %xmm1
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
-; SSE2-NEXT: movd %xmm2, %eax
-; SSE2-NEXT: xorps %xmm2, %xmm2
-; SSE2-NEXT: cvtsi2sd %eax, %xmm2
-; SSE2-NEXT: unpcklpd {{.*#+}} xmm2 = xmm2[0],xmm1[0]
+; SSE2-NEXT: andpd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE2-NEXT: xorpd %xmm1, %xmm1
+; SSE2-NEXT: movapd %xmm0, %xmm2
+; SSE2-NEXT: unpckhps {{.*#+}} xmm2 = xmm2[2],xmm1[2],xmm2[3],xmm1[3]
+; SSE2-NEXT: movapd {{.*#+}} xmm3 = [4.503599627370496E+15,4.503599627370496E+15]
+; SSE2-NEXT: orpd %xmm3, %xmm2
+; SSE2-NEXT: subpd %xmm3, %xmm2
; SSE2-NEXT: psrlq $52, %xmm2
-; SSE2-NEXT: movd %xmm0, %eax
-; SSE2-NEXT: xorps %xmm1, %xmm1
-; SSE2-NEXT: cvtsi2sd %eax, %xmm1
-; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,1,1]
-; SSE2-NEXT: movd %xmm3, %eax
-; SSE2-NEXT: xorps %xmm3, %xmm3
-; SSE2-NEXT: cvtsi2sd %eax, %xmm3
-; SSE2-NEXT: unpcklpd {{.*#+}} xmm1 = xmm1[0],xmm3[0]
-; SSE2-NEXT: psrlq $52, %xmm1
-; SSE2-NEXT: packssdw %xmm2, %xmm1
+; SSE2-NEXT: movapd %xmm0, %xmm4
+; SSE2-NEXT: unpcklps {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1]
+; SSE2-NEXT: orpd %xmm3, %xmm4
+; SSE2-NEXT: subpd %xmm3, %xmm4
+; SSE2-NEXT: psrlq $52, %xmm4
+; SSE2-NEXT: packssdw %xmm2, %xmm4
; SSE2-NEXT: movdqa {{.*#+}} xmm2 = [1054,1054,1054,1054]
-; SSE2-NEXT: psubd %xmm1, %xmm2
-; SSE2-NEXT: pxor %xmm1, %xmm1
+; SSE2-NEXT: psubd %xmm4, %xmm2
; SSE2-NEXT: pcmpeqd %xmm1, %xmm0
; SSE2-NEXT: movdqa %xmm0, %xmm1
; SSE2-NEXT: pandn %xmm2, %xmm1
diff --git a/llvm/test/CodeGen/X86/expand-vp-int-intrinsics.ll b/llvm/test/CodeGen/X86/expand-vp-int-intrinsics.ll
index 328c0bac7495c..bbdeb1b7f1290 100644
--- a/llvm/test/CodeGen/X86/expand-vp-int-intrinsics.ll
+++ b/llvm/test/CodeGen/X86/expand-vp-int-intrinsics.ll
@@ -1476,29 +1476,21 @@ define <4 x i32> @vp_ctlz_v4i32(<4 x i32> %va, <4 x i1> %m, i32 zeroext %evl) {
;
; SSE-LABEL: vp_ctlz_v4i32:
; SSE: # %bb.0:
-; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[3,3,3,3]
-; SSE-NEXT: movd %xmm1, %eax
-; SSE-NEXT: xorps %xmm1, %xmm1
-; SSE-NEXT: cvtsi2sd %rax, %xmm1
-; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
-; SSE-NEXT: movd %xmm2, %eax
-; SSE-NEXT: xorps %xmm2, %xmm2
-; SSE-NEXT: cvtsi2sd %rax, %xmm2
-; SSE-NEXT: unpcklpd {{.*#+}} xmm2 = xmm2[0],xmm1[0]
+; SSE-NEXT: xorpd %xmm1, %xmm1
+; SSE-NEXT: movapd %xmm0, %xmm2
+; SSE-NEXT: unpckhps {{.*#+}} xmm2 = xmm2[2],xmm1[2],xmm2[3],xmm1[3]
+; SSE-NEXT: movapd {{.*#+}} xmm3 = [4.503599627370496E+15,4.503599627370496E+15]
+; SSE-NEXT: orpd %xmm3, %xmm2
+; SSE-NEXT: subpd %xmm3, %xmm2
; SSE-NEXT: psrlq $52, %xmm2
-; SSE-NEXT: movd %xmm0, %eax
-; SSE-NEXT: xorps %xmm1, %xmm1
-; SSE-NEXT: cvtsi2sd %rax, %xmm1
-; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,1,1]
-; SSE-NEXT: movd %xmm3, %eax
-; SSE-NEXT: xorps %xmm3, %xmm3
-; SSE-NEXT: cvtsi2sd %rax, %xmm3
-; SSE-NEXT: unpcklpd {{.*#+}} xmm1 = xmm1[0],xmm3[0]
-; SSE-NEXT: psrlq $52, %xmm1
-; SSE-NEXT: packssdw %xmm2, %xmm1
+; SSE-NEXT: movapd %xmm0, %xmm4
+; SSE-NEXT: unpcklps {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1]
+; SSE-NEXT: orpd %xmm3, %xmm4
+; SSE-NEXT: subpd %xmm3, %xmm4
+; SSE-NEXT: psrlq $52, %xmm4
+; SSE-NEXT: packssdw %xmm2, %xmm4
; SSE-NEXT: movdqa {{.*#+}} xmm2 = [1054,1054,1054,1054]
-; SSE-NEXT: psubd %xmm1, %xmm2
-; SSE-NEXT: pxor %xmm1, %xmm1
+; SSE-NEXT: psubd %xmm4, %xmm2
; SSE-NEXT: pcmpeqd %xmm1, %xmm0
; SSE-NEXT: movdqa %xmm0, %xmm1
; SSE-NEXT: pandn %xmm2, %xmm1
diff --git a/llvm/test/CodeGen/X86/vec_ctbits.ll b/llvm/test/CodeGen/X86/vec_ctbits.ll
index 0666e6316885e..22c2577ff1b6b 100644
--- a/llvm/test/CodeGen/X86/vec_ctbits.ll
+++ b/llvm/test/CodeGen/X86/vec_ctbits.ll
@@ -140,29 +140,21 @@ define <2 x i32> @promtz(<2 x i32> %a) nounwind {
define <2 x i32> @promlz(<2 x i32> %a) nounwind {
; CHECK-LABEL: promlz:
; CHECK: # %bb.0:
-; CHECK-NEXT: pshufd {{.*#+}} xmm1 = xmm0[3,3,3,3]
-; CHECK-NEXT: movd %xmm1, %eax
-; CHECK-NEXT: xorps %xmm1, %xmm1
-; CHECK-NEXT: cvtsi2sd %rax, %xmm1
-; CHECK-NEXT: pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
-; CHECK-NEXT: movd %xmm2, %eax
-; CHECK-NEXT: xorps %xmm2, %xmm2
-; CHECK-NEXT: cvtsi2sd %rax, %xmm2
-; CHECK-NEXT: unpcklpd {{.*#+}} xmm2 = xmm2[0],xmm1[0]
+; CHECK-NEXT: xorpd %xmm1, %xmm1
+; CHECK-NEXT: movapd %xmm0, %xmm2
+; CHECK-NEXT: unpckhps {{.*#+}} xmm2 = xmm2[2],xmm1[2],xmm2[3],xmm1[3]
+; CHECK-NEXT: movapd {{.*#+}} xmm3 = [4.503599627370496E+15,4.503599627370496E+15]
+; CHECK-NEXT: orpd %xmm3, %xmm2
+; CHECK-NEXT: subpd %xmm3, %xmm2
; CHECK-NEXT: psrlq $52, %xmm2
-; CHECK-NEXT: movd %xmm0, %eax
-; CHECK-NEXT: xorps %xmm1, %xmm1
-; CHECK-NEXT: cvtsi2sd %rax, %xmm1
-; CHECK-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,1,1]
-; CHECK-NEXT: movd %xmm3, %eax
-; CHECK-NEXT: xorps %xmm3, %xmm3
-; CHECK-NEXT: cvtsi2sd %rax, %xmm3
-; CHECK-NEXT: unpcklpd {{.*#+}} xmm1 = xmm1[0],xmm3[0]
-; CHECK-NEXT: psrlq $52, %xmm1
-; CHECK-NEXT: packssdw %xmm2, %xmm1
+; CHECK-NEXT: movapd %xmm0, %xmm4
+; CHECK-NEXT: unpcklps {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1]
+; CHECK-NEXT: orpd %xmm3, %xmm4
+; CHECK-NEXT: subpd %xmm3, %xmm4
+; CHECK-NEXT: psrlq $52, %xmm4
+; CHECK-NEXT: packssdw %xmm2, %xmm4
; CHECK-NEXT: movdqa {{.*#+}} xmm2 = [1054,1054,1054,1054]
-; CHECK-NEXT: psubd %xmm1, %xmm2
-; CHECK-NEXT: pxor %xmm1, %xmm1
+; CHECK-NEXT: psubd %xmm4, %xmm2
; CHECK-NEXT: pcmpeqd %xmm1, %xmm0
; CHECK-NEXT: movdqa %xmm0, %xmm1
; CHECK-NEXT: pandn %xmm2, %xmm1
diff --git a/llvm/test/CodeGen/X86/vector-lzcnt-128.ll b/llvm/test/CodeGen/X86/vector-lzcnt-128.ll
index 380e8ca9e26c1..13b7bee8474e1 100644
--- a/llvm/test/CodeGen/X86/vector-lzcnt-128.ll
+++ b/llvm/test/CodeGen/X86/vector-lzcnt-128.ll
@@ -572,29 +572,21 @@ define <2 x i64> @testv2i64u(<2 x i64> %in) nounwind {
define <4 x i32> @testv4i32(<4 x i32> %in) nounwind {
; SSE2-LABEL: testv4i32:
; SSE2: # %bb.0:
-; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[3,3,3,3]
-; SSE2-NEXT: movd %xmm1, %eax
-; SSE2-NEXT: xorps %xmm1, %xmm1
-; SSE2-NEXT: cvtsi2sd %rax, %xmm1
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
-; SSE2-NEXT: movd %xmm2, %eax
-; SSE2-NEXT: xorps %xmm2, %xmm2
-; SSE2-NEXT: cvtsi2sd %rax, %xmm2
-; SSE2-NEXT: unpcklpd {{.*#+}} xmm2 = xmm2[0],xmm1[0]
+; SSE2-NEXT: xorpd %xmm1, %xmm1
+; SSE2-NEXT: movapd %xmm0, %xmm2
+; SSE2-NEXT: unpckhps {{.*#+}} xmm2 = xmm2[2],xmm1[2],xmm2[3],xmm1[3]
+; SSE2-NEXT: movapd {{.*#+}} xmm3 = [4.503599627370496E+15,4.503599627370496E+15]
+; SSE2-NEXT: orpd %xmm3, %xmm2
+; SSE2-NEXT: subpd %xmm3, %xmm2
; SSE2-NEXT: psrlq $52, %xmm2
-; SSE2-NEXT: movd %xmm0, %eax
-; SSE2-NEXT: xorps %xmm1, %xmm1
-; SSE2-NEXT: cvtsi2sd %rax, %xmm1
-; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,1,1]
-; SSE2-NEXT: movd %xmm3, %eax
-; SSE2-NEXT: xorps %xmm3, %xmm3
-; SSE2-NEXT: cvtsi2sd %rax, %xmm3
-; SSE2-NEXT: unpcklpd {{.*#+}} xmm1 = xmm1[0],xmm3[0]
-; SSE2-NEXT: psrlq $52, %xmm1
-; SSE2-NEXT: packssdw %xmm2, %xmm1
+; SSE2-NEXT: movapd %xmm0, %xmm4
+; SSE2-NEXT: unpcklps {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1]
+; SSE2-NEXT: orpd %xmm3, %xmm4
+; SSE2-NEXT: subpd %xmm3, %xmm4
+; SSE2-NEXT: psrlq $52, %xmm4
+; SSE2-NEXT: packssdw %xmm2, %xmm4
; SSE2-NEXT: movdqa {{.*#+}} xmm2 = [1054,1054,1054,1054]
-; SSE2-NEXT: psubd %xmm1, %xmm2
-; SSE2-NEXT: pxor %xmm1, %xmm1
+; SSE2-NEXT: psubd %xmm4, %xmm2
; SSE2-NEXT: pcmpeqd %xmm1, %xmm0
; SSE2-NEXT: movdqa %xmm0, %xmm1
; SSE2-NEXT: pandn %xmm2, %xmm1
@@ -604,29 +596,21 @@ define <4 x i32> @testv4i32(<4 x i32> %in) nounwind {
;
; SSE3-LABEL: testv4i32:
; SSE3: # %bb.0:
-; SSE3-NEXT: pshufd {{.*#+}} xmm1 = xmm0[3,3,3,3]
-; SSE3-NEXT: movd %xmm1, %eax
-; SSE3-NEXT: xorps %xmm1, %xmm1
-; SSE3-NEXT: cvtsi2sd %rax, %xmm1
-; SSE3-NEXT: pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
-; SSE3-NEXT: movd %xmm2, %eax
-; SSE3-NEXT: xorps %xmm2, %xmm2
-; SSE3-NEXT: cvtsi2sd %rax, %xmm2
-; SSE3-NEXT: unpcklpd {{.*#+}} xmm2 = xmm2[0],xmm1[0]
+; SSE3-NEXT: xorpd %xmm1, %xmm1
+; SSE3-NEXT: movapd %xmm0, %xmm2
+; SSE3-NEXT: unpckhps {{.*#+}} xmm2 = xmm2[2],xmm1[2],xmm2[3],xmm1[3]
+; SSE3-NEXT: movapd {{.*#+}} xmm3 = [4.503599627370496E+15,4.503599627370496E+15]
+; SSE3-NEXT: orpd %xmm3, %xmm2
+; SSE3-NEXT: subpd %xmm3, %xmm2
; SSE3-NEXT: psrlq $52, %xmm2
-; SSE3-NEXT: movd %xmm0, %eax
-; SSE3-NEXT: xorps %xmm1, %xmm1
-; SSE3-NEXT: cvtsi2sd %rax, %xmm1
-; SSE3-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,1,1]
-; SSE3-NEXT: movd %xmm3, %eax
-; SSE3-NEXT: xorps %xmm3, %xmm3
-; SSE3-NEXT: cvtsi2sd %rax, %xmm3
-; SSE3-NEXT: unpcklpd {{.*#+}} xmm1 = xmm1[0],xmm3[0]
-; SSE3-NEXT: psrlq $52, %xmm1
-; SSE3-NEXT: packssdw %xmm2, %xmm1
+; SSE3-NEXT: movapd %xmm0, %xmm4
+; SSE3-NEXT: unpcklps {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1]
+; SSE3-NEXT: orpd %xmm3, %xmm4
+; SSE3-NEXT: subpd %xmm3, %xmm4
+; SSE3-NEXT: psrlq $52, %xmm4
+; SSE3-NEXT: packssdw %xmm2, %xmm4
; SSE3-NEXT: movdqa {{.*#+}} xmm2 = [1054,1054,1054,1054]
-; SSE3-NEXT: psubd %xmm1, %xmm2
-; SSE3-NEXT: pxor %xmm1, %xmm1
+; SSE3-NEXT: psubd %xmm4, %xmm2
; SSE3-NEXT: pcmpeqd %xmm1, %xmm0
; SSE3-NEXT: movdqa %xmm0, %xmm1
; SSE3-NEXT: pandn %xmm2, %xmm1
@@ -801,54 +785,40 @@ define <4 x i32> @testv4i32(<4 x i32> %in) nounwind {
define <4 x i32> @testv4i32u(<4 x i32> %in) nounwind {
; SSE2-LABEL: testv4i32u:
; SSE2: # %bb.0:
-; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[3,3,3,3]
-; SSE2-NEXT: movd %xmm1, %eax
-; SSE2-NEXT: xorps %xmm1, %xmm1
-; SSE2-NEXT: cvtsi2sd %rax, %xmm1
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
-; SSE2-NEXT: movd %xmm2, %eax
-; SSE2-NEXT: xorps %xmm2, %xmm2
-; SSE2-NEXT: cvtsi2sd %rax, %xmm2
-; SSE2-NEXT: unpcklpd {{.*#+}} xmm2 = xmm2[0],xmm1[0]
+; SSE2-NEXT: xorpd %xmm1, %xmm1
+; SSE2-NEXT: movapd %xmm0, %xmm2
+; SSE2-NEXT: unpckhps {{.*#+}} xmm2 = xmm2[2],xmm1[2],xmm2[3],xmm1[3]
+; SSE2-NEXT: movapd {{.*#+}} xmm3 = [4.503599627370496E+15,4.503599627370496E+15]
+; SSE2-NEXT: orpd %xmm3, %xmm2
+; SSE2-NEXT: subpd %xmm3, %xmm2
; SSE2-NEXT: psrlq $52, %xmm2
-; SSE2-NEXT: movd %xmm0, %eax
-; SSE2-NEXT: xorps %xmm1, %xmm1
-; SSE2-NEXT: cvtsi2sd %rax, %xmm1
-; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
-; SSE2-NEXT: movd %xmm0, %eax
-; SSE2-NEXT: xorps %xmm0, %xmm0
-; SSE2-NEXT: cvtsi2sd %rax, %xmm0
-; SSE2-NEXT: unpcklpd {{.*#+}} xmm1 = xmm1[0],xmm0[0]
-; SSE2-NEXT: psrlq $52, %xmm1
-; SSE2-NEXT: packssdw %xmm2, %xmm1
-; SSE2-NEXT: movdqa {{.*#+}} xmm0 = [1054,1054,1054,1054]
-; SSE2-NEXT: psubd %xmm1, %xmm0
+; SSE2-NEXT: unpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE2-NEXT: orpd %xmm3, %xmm0
+; SSE2-NEXT: subpd %xmm3, %xmm0
+; SSE2-NEXT: psrlq $52, %xmm0
+; SSE2-NEXT: packssdw %xmm2, %xmm0
+; SSE2-NEXT: movdqa {{.*#+}} xmm1 = [1054,1054,1054,1054]
+; SSE2-NEXT: psubd %xmm0, %xmm1
+; SSE2-NEXT: movdqa %xmm1, %xmm0
; SSE2-NEXT: retq
;
; SSE3-LABEL: testv4i32u:
; SSE3: # %bb.0:
-; SSE3-NEXT: pshufd {{.*#+}} xmm1 = xmm0[3,3,3,3]
-; SSE3-NEXT: movd %xmm1, %eax
-; SSE3-NEXT: xorps %xmm1, %xmm1
-; SSE3-NEXT: cvtsi2sd %rax, %xmm1
-; SSE3-NEXT: pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
-; SSE3-NEXT: movd %xmm2, %eax
-; SSE3-NEXT: xorps %xmm2, %xmm2
-; SSE3-NEXT: cvtsi2sd %rax, %xmm2
-; SSE3-NEXT: unpcklpd {{.*#+}} xmm2 = xmm2[0],xmm1[0]
+; SSE3-NEXT: xorpd %xmm1, %xmm1
+; SSE3-NEXT: movapd %xmm0, %xmm2
+; SSE3-NEXT: unpckhps {{.*#+}} xmm2 = xmm2[2],xmm1[2],xmm2[3],xmm1[3]
+; SSE3-NEXT: movapd {{.*#+}} xmm3 = [4.503599627370496E+15,4.503599627370496E+15]
+; SSE3-NEXT: orpd %xmm3, %xmm2
+; SSE3-NEXT: subpd %xmm3, %xmm2
; SSE3-NEXT: psrlq $52, %xmm2
-; SSE3-NEXT: movd %xmm0, %eax
-; SSE3-NEXT: xorps %xmm1, %xmm1
-; SSE3-NEXT: cvtsi2sd %rax, %xmm1
-; SSE3-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
-; SSE3-NEXT: movd %xmm0, %eax
-; SSE3-NEXT: xorps %xmm0, %xmm0
-; SSE3-NEXT: cvtsi2sd %rax, %xmm0
-; SSE3-NEXT: unpcklpd {{.*#+}} xmm1 = xmm1[0],xmm0[0]
-; SSE3-NEXT: psrlq $52, %xmm1
-; SSE3-NEXT: packssdw %xmm2, %xmm1
-; SSE3-NEXT: movdqa {{.*#+}} xmm0 = [1054,1054,1054,1054]
-; SSE3-NEXT: psubd %xmm1, %xmm0
+; SSE3-NEXT: unpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE3-NEXT: orpd %xmm3, %xmm0
+; SSE3-NEXT: subpd %xmm3, %xmm0
+; SSE3-NEXT: psrlq $52, %xmm0
+; SSE3-NEXT: packssdw %xmm2, %xmm0
+; SSE3-NEXT: movdqa {{.*#+}} xmm1 = [1054,1054,1054,1054]
+; SSE3-NEXT: psubd %xmm0, %xmm1
+; SSE3-NEXT: movdqa %xmm1, %xmm0
; SSE3-NEXT: retq
;
; SSSE3-LABEL: testv4i32u:
diff --git a/llvm/test/CodeGen/X86/vector-lzcnt-sub128.ll b/llvm/test/CodeGen/X86/vector-lzcnt-sub128.ll
index f70f7e204c86a..ad8a60cb95f07 100644
--- a/llvm/test/CodeGen/X86/vector-lzcnt-sub128.ll
+++ b/llvm/test/CodeGen/X86/vector-lzcnt-sub128.ll
@@ -6,28 +6,21 @@ declare <2 x i32> @llvm.ctlz.v2i32(<2 x i32>, i1 immarg)
define <2 x i32> @illegal_ctlz(<2 x i32> %v1) {
; CHECK-LABEL: illegal_ctlz:
; CHECK: # %bb.0:
-; CHECK-NEXT: pshufd {{.*#+}} xmm1 = xmm0[3,3,3,3]
-; CHECK-NEXT: movd %xmm1, %eax
-; CHECK-NEXT: xorps %xmm1, %xmm1
-; CHECK-NEXT: cvtsi2sd %rax, %xmm1
-; CHECK-NEXT: pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
-; CHECK-NEXT: movd %xmm2, %eax
-; CHECK-NEXT: xorps %xmm2, %xmm2
-; CHECK-NEXT: cvtsi2sd %rax, %xmm2
-; CHECK-NEXT: unpcklpd {{.*#+}} xmm2 = xmm2[0],xmm1[0]
+; CHECK-NEXT: xorpd %xmm1, %xmm1
+; CHECK-NEXT: movapd %xmm0, %xmm2
+; CHECK-NEXT: unpckhps {{.*#+}} xmm2 = xmm2[2],xmm1[2],xmm2[3],xmm1[3]
+; CHECK-NEXT: movapd {{.*#+}} xmm3 = [4.503599627370496E+15,4.503599627370496E+15]
+; CHECK-NEXT: orpd %xmm3, %xmm2
+; CHECK-NEXT: subpd %xmm3, %xmm2
; CHECK-NEXT: psrlq $52, %xmm2
-; CHECK-NEXT: movd %xmm0, %eax
-; CHECK-NEXT: xorps %xmm1, %xmm1
-; CHECK-NEXT: cvtsi2sd %rax, %xmm1
-; CHECK-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
-; CHECK-NEXT: movd %xmm0, %eax
-; CHECK-NEXT: xorps %xmm0, %xmm0
-; CHECK-NEXT: cvtsi2sd %rax, %xmm0
-; CHECK-NEXT: unpcklpd {{.*#+}} xmm1 = xmm1[0],xmm0[0]
-; CHECK-NEXT: psrlq $52, %xmm1
-; CHECK-NEXT: packssdw %xmm2, %xmm1
-; CHECK-NEXT: movdqa {{.*#+}} xmm0 = [1054,1054,1054,1054]
-; CHECK-NEXT: psubd %xmm1, %xmm0
+; CHECK-NEXT: unpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; CHECK-NEXT: orpd %xmm3, %xmm0
+; CHECK-NEXT: subpd %xmm3, %xmm0
+; CHECK-NEXT: psrlq $52, %xmm0
+; CHECK-NEXT: packssdw %xmm2, %xmm0
+; CHECK-NEXT: movdqa {{.*#+}} xmm1 = [1054,1054,1054,1054]
+; CHECK-NEXT: psubd %xmm0, %xmm1
+; CHECK-NEXT: movdqa %xmm1, %xmm0
; CHECK-NEXT: retq
%v2 = call <2 x i32> @llvm.ctlz.v2i32(<2 x i32> %v1, i1 true)
ret <2 x i32> %v2
>From 5619237126d09528c92eb28d2a36059aebff233e Mon Sep 17 00:00:00 2001
From: Ayaan Khan <ayaanandayaan2 at gmail.com>
Date: Thu, 30 Apr 2026 19:27:35 +0530
Subject: [PATCH 3/3] fix format
---
llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp | 14 +++++++++-----
1 file changed, 9 insertions(+), 5 deletions(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
index b54bcc9a6fc0e..d629efe6b3100 100644
--- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
@@ -9901,7 +9901,8 @@ SDValue TargetLowering::expandCTLZWithFP(SDNode *Node,
EVT HalfFloatVT = HalfVT.changeVectorElementType(*DAG.getContext(), MVT::f64);
EVT HalfFloatBitsVT = HalfFloatVT.changeVectorElementTypeToInteger();
- const fltSemantics &Sem = HalfFloatVT.getVectorElementType().getFltSemantics();
+ const fltSemantics &Sem =
+ HalfFloatVT.getVectorElementType().getFltSemantics();
unsigned BitWidth = EltVT.getSizeInBits();
unsigned MantissaBits = APFloat::semanticsPrecision(Sem);
unsigned ExponentBias = APFloat::semanticsMaxExponent(Sem);
@@ -9909,7 +9910,8 @@ SDValue TargetLowering::expandCTLZWithFP(SDNode *Node,
auto ComputeExp = [&](SDValue Half) {
SDValue Float = DAG.getNode(ISD::UINT_TO_FP, dl, HalfFloatVT, Half);
SDValue Bits = DAG.getBitcast(HalfFloatBitsVT, Float);
- SDValue Exp = DAG.getNode(ISD::SRL, dl, HalfFloatBitsVT, Bits,
+ SDValue Exp = DAG.getNode(
+ ISD::SRL, dl, HalfFloatBitsVT, Bits,
DAG.getShiftAmountConstant(MantissaBits - 1, HalfFloatBitsVT, dl));
return DAG.getNode(ISD::TRUNCATE, dl, HalfVT, Exp);
};
@@ -9917,9 +9919,11 @@ SDValue TargetLowering::expandCTLZWithFP(SDNode *Node,
SDValue ExpTruncLo = ComputeExp(DAG.getExtractSubvector(dl, HalfVT, Op, 0));
SDValue ExpTruncHi = ComputeExp(
DAG.getExtractSubvector(dl, HalfVT, Op, VT.getVectorNumElements() / 2));
- SDValue Exp = DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, ExpTruncLo, ExpTruncHi);
- SDValue NonZeroRes = DAG.getNode(ISD::SUB, dl, VT,
- DAG.getConstant(BitWidth - 1 + ExponentBias, dl, VT), Exp);
+ SDValue Exp =
+ DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, ExpTruncLo, ExpTruncHi);
+ SDValue NonZeroRes =
+ DAG.getNode(ISD::SUB, dl, VT,
+ DAG.getConstant(BitWidth - 1 + ExponentBias, dl, VT), Exp);
// Skip Op == 0 case for CTLZ_ZERO_UNDEF
if (Node->getOpcode() == ISD::CTLZ_ZERO_UNDEF)
More information about the llvm-commits
mailing list