[llvm] [X86] Prefer AVX512 VPCMP against zero over splat(1) for sle/slt (PR #216716)
Shaurya Srivastava via llvm-commits
llvm-commits at lists.llvm.org
Mon Aug 17 10:35:59 PDT 2026
https://github.com/Shaurya2k06 updated https://github.com/llvm/llvm-project/pull/216716
>From bbfbea78b2bafc3cb653719556537b20e518167b Mon Sep 17 00:00:00 2001
From: shaurya2k06 <shaurya2k06 at gmail.com>
Date: Mon, 17 Aug 2026 17:49:06 +0530
Subject: [PATCH 1/2] [X86] Prefer AVX512 VPCMP against zero over splat(1) for
sle/slt
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
InstCombine canonicalizes icmp sle x, 0 to icmp slt x, 1. AVX512 VPCMP
can encode LE against a zeroed register, but we were loading splat(1)
from the constant pool.
Rewrite slt/sgt vs splat(±1) back to sle/sge vs zero for AVX512 mask
compares, and skip the existing LE/GE → LT/GT constant adjust when it
would destroy a zero compare.
Fixes #216660
Signed-off-by: shaurya2k06 <shaurya2k06 at gmail.com>
Assisted-by: Cursor
---
llvm/lib/Target/X86/X86ISelLowering.cpp | 56 +++++-
llvm/test/CodeGen/X86/avx512-icmp-sle-zero.ll | 179 ++++++++++++++++++
llvm/test/CodeGen/X86/combine-icmp.ll | 3 +-
3 files changed, 231 insertions(+), 7 deletions(-)
create mode 100644 llvm/test/CodeGen/X86/avx512-icmp-sle-zero.ll
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 0dc3340a13c73..9c9993e131022 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -58307,6 +58307,38 @@ static SDValue combineAVX512SetCCToKMOV(EVT VT, SDValue Op0, ISD::CondCode CC,
return Bitcast;
}
+/// AVX512 VPCMP encodes all integer condcodes. Prefer a compare against zero
+/// over the canonical slt/sgt vs splat(±1), which otherwise becomes a
+/// constant-pool load. Analogous to TranslateX86CC for scalars.
+static SDValue combineAVX512SetCCPreferZero(EVT VT, SDValue LHS, SDValue RHS,
+ ISD::CondCode CC, const SDLoc &DL,
+ SelectionDAG &DAG,
+ const X86Subtarget &Subtarget) {
+ if (!Subtarget.hasAVX512() || !VT.isVectorOf(MVT::i1) ||
+ !LHS.getValueType().isInteger())
+ return SDValue();
+
+ EVT OpVT = LHS.getValueType();
+ APInt C;
+ // slt x, 1 / sgt 1, x -> sle x, 0
+ if ((CC == ISD::SETLT && ISD::isConstantSplatVector(RHS.getNode(), C) &&
+ C.isOne()) ||
+ (CC == ISD::SETGT && ISD::isConstantSplatVector(LHS.getNode(), C) &&
+ C.isOne())) {
+ SDValue X = CC == ISD::SETLT ? LHS : RHS;
+ return DAG.getSetCC(DL, VT, X, DAG.getConstant(0, DL, OpVT), ISD::SETLE);
+ }
+ // sgt x, -1 / slt -1, x -> sge x, 0
+ if ((CC == ISD::SETGT && ISD::isConstantSplatVector(RHS.getNode(), C) &&
+ C.isAllOnes()) ||
+ (CC == ISD::SETLT && ISD::isConstantSplatVector(LHS.getNode(), C) &&
+ C.isAllOnes())) {
+ SDValue X = CC == ISD::SETGT ? LHS : RHS;
+ return DAG.getSetCC(DL, VT, X, DAG.getConstant(0, DL, OpVT), ISD::SETGE);
+ }
+ return SDValue();
+}
+
static SDValue combineSetCC(SDNode *N, SelectionDAG &DAG,
TargetLowering::DAGCombinerInfo &DCI,
const X86Subtarget &Subtarget) {
@@ -58439,6 +58471,10 @@ static SDValue combineSetCC(SDNode *N, SelectionDAG &DAG,
// use `PCMPGT` if the result is mean to stay in a vector (and if its going to
// a mask, there are signed AVX512 comparisons).
if (VT.isVector() && OpVT.isVector() && OpVT.isInteger()) {
+ if (SDValue V = combineAVX512SetCCPreferZero(VT, LHS, RHS, CC, DL, DAG,
+ Subtarget))
+ return V;
+
bool CanMakeSigned = false;
if (ISD::isUnsignedIntSetCC(CC)) {
KnownBits CmpKnown =
@@ -58455,6 +58491,10 @@ static SDValue combineSetCC(SDNode *N, SelectionDAG &DAG,
CmpKnown.Zero.isSignBitSet() || CmpKnown.One.isSignBitSet();
}
if (CanMakeSigned || ISD::isSignedIntSetCC(CC)) {
+ // AVX512 can encode LE/GE against zero; do not turn that into LT/GT
+ // vs ±1.
+ const bool KeepZeroCmp =
+ Subtarget.hasAVX512() && VT.isVectorOf(MVT::i1);
SDValue LHSOut = LHS;
SDValue RHSOut = RHS;
ISD::CondCode NewCC = CC;
@@ -58462,10 +58502,12 @@ static SDValue combineSetCC(SDNode *N, SelectionDAG &DAG,
case ISD::SETGE:
case ISD::SETUGE:
if (SDValue NewLHS = incDecVectorConstant(LHS, DAG, /*IsInc*/ true,
- /*NSW*/ true))
+ /*NSW*/ true)) {
+ if (KeepZeroCmp && ISD::isConstantSplatVectorAllZeros(LHS.getNode()))
+ break;
LHSOut = NewLHS;
- else if (SDValue NewRHS = incDecVectorConstant(
- RHS, DAG, /*IsInc*/ false, /*NSW*/ true))
+ } else if (SDValue NewRHS = incDecVectorConstant(
+ RHS, DAG, /*IsInc*/ false, /*NSW*/ true))
RHSOut = NewRHS;
else
break;
@@ -58480,10 +58522,12 @@ static SDValue combineSetCC(SDNode *N, SelectionDAG &DAG,
if (SDValue NewLHS = incDecVectorConstant(LHS, DAG, /*IsInc*/ false,
/*NSW*/ true))
LHSOut = NewLHS;
- else if (SDValue NewRHS = incDecVectorConstant(RHS, DAG, /*IsInc*/ true,
- /*NSW*/ true))
+ else if (SDValue NewRHS = incDecVectorConstant(
+ RHS, DAG, /*IsInc*/ true, /*NSW*/ true)) {
+ if (KeepZeroCmp && ISD::isConstantSplatVectorAllZeros(RHS.getNode()))
+ break;
RHSOut = NewRHS;
- else
+ } else
break;
[[fallthrough]];
diff --git a/llvm/test/CodeGen/X86/avx512-icmp-sle-zero.ll b/llvm/test/CodeGen/X86/avx512-icmp-sle-zero.ll
new file mode 100644
index 0000000000000..ef8da7d206ac0
--- /dev/null
+++ b/llvm/test/CodeGen/X86/avx512-icmp-sle-zero.ll
@@ -0,0 +1,179 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64-v3 | FileCheck %s --check-prefix=AVX2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64-v4 | FileCheck %s --check-prefix=AVX512
+
+; Prefer VPCMP against zero over a constant-pool splat(1)/splat(-1).
+; See https://github.com/llvm/llvm-project/issues/216660
+
+define i16 @cmp_sle_v16i8(<16 x i8> %x) {
+; AVX2-LABEL: cmp_sle_v16i8:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX2-NEXT: vpcmpgtb %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpmovmskb %xmm0, %eax
+; AVX2-NEXT: notl %eax
+; AVX2-NEXT: # kill: def $ax killed $ax killed $eax
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: cmp_sle_v16i8:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512-NEXT: vpcmpleb %xmm1, %xmm0, %k0
+; AVX512-NEXT: kmovd %k0, %eax
+; AVX512-NEXT: # kill: def $ax killed $ax killed $eax
+; AVX512-NEXT: retq
+ %c = icmp sle <16 x i8> %x, splat (i8 0)
+ %r = bitcast <16 x i1> %c to i16
+ ret i16 %r
+}
+
+define i16 @cmp_slt1_v16i8(<16 x i8> %x) {
+; AVX2-LABEL: cmp_slt1_v16i8:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX2-NEXT: vpcmpgtb %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpmovmskb %xmm0, %eax
+; AVX2-NEXT: notl %eax
+; AVX2-NEXT: # kill: def $ax killed $ax killed $eax
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: cmp_slt1_v16i8:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512-NEXT: vpcmpleb %xmm1, %xmm0, %k0
+; AVX512-NEXT: kmovd %k0, %eax
+; AVX512-NEXT: # kill: def $ax killed $ax killed $eax
+; AVX512-NEXT: retq
+ %c = icmp slt <16 x i8> %x, splat (i8 1)
+ %r = bitcast <16 x i1> %c to i16
+ ret i16 %r
+}
+
+define i16 @cmp_sgt1_lhs_v16i8(<16 x i8> %x) {
+; AVX2-LABEL: cmp_sgt1_lhs_v16i8:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX2-NEXT: vpcmpgtb %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpmovmskb %xmm0, %eax
+; AVX2-NEXT: notl %eax
+; AVX2-NEXT: # kill: def $ax killed $ax killed $eax
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: cmp_sgt1_lhs_v16i8:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512-NEXT: vpcmpleb %xmm1, %xmm0, %k0
+; AVX512-NEXT: kmovd %k0, %eax
+; AVX512-NEXT: # kill: def $ax killed $ax killed $eax
+; AVX512-NEXT: retq
+ %c = icmp sgt <16 x i8> splat (i8 1), %x
+ %r = bitcast <16 x i1> %c to i16
+ ret i16 %r
+}
+
+define i16 @cmp_sge_v16i8(<16 x i8> %x) {
+; AVX2-LABEL: cmp_sge_v16i8:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vpmovmskb %xmm0, %eax
+; AVX2-NEXT: notl %eax
+; AVX2-NEXT: # kill: def $ax killed $ax killed $eax
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: cmp_sge_v16i8:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512-NEXT: vpcmpnltb %xmm1, %xmm0, %k0
+; AVX512-NEXT: kmovd %k0, %eax
+; AVX512-NEXT: # kill: def $ax killed $ax killed $eax
+; AVX512-NEXT: retq
+ %c = icmp sge <16 x i8> %x, splat (i8 0)
+ %r = bitcast <16 x i1> %c to i16
+ ret i16 %r
+}
+
+define i16 @cmp_sgt_neg1_v16i8(<16 x i8> %x) {
+; AVX2-LABEL: cmp_sgt_neg1_v16i8:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vpmovmskb %xmm0, %eax
+; AVX2-NEXT: notl %eax
+; AVX2-NEXT: # kill: def $ax killed $ax killed $eax
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: cmp_sgt_neg1_v16i8:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512-NEXT: vpcmpnltb %xmm1, %xmm0, %k0
+; AVX512-NEXT: kmovd %k0, %eax
+; AVX512-NEXT: # kill: def $ax killed $ax killed $eax
+; AVX512-NEXT: retq
+ %c = icmp sgt <16 x i8> %x, splat (i8 -1)
+ %r = bitcast <16 x i1> %c to i16
+ ret i16 %r
+}
+
+define i8 @cmp_slt1_v8i16(<8 x i16> %x) {
+; AVX2-LABEL: cmp_slt1_v8i16:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX2-NEXT: vpcmpgtw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpacksswb %xmm0, %xmm0, %xmm0
+; AVX2-NEXT: vpmovmskb %xmm0, %eax
+; AVX2-NEXT: notl %eax
+; AVX2-NEXT: # kill: def $al killed $al killed $eax
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: cmp_slt1_v8i16:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512-NEXT: vpcmplew %xmm1, %xmm0, %k0
+; AVX512-NEXT: kmovd %k0, %eax
+; AVX512-NEXT: # kill: def $al killed $al killed $eax
+; AVX512-NEXT: retq
+ %c = icmp slt <8 x i16> %x, splat (i16 1)
+ %r = bitcast <8 x i1> %c to i8
+ ret i8 %r
+}
+
+define i4 @cmp_slt1_v4i32(<4 x i32> %x) {
+; AVX2-LABEL: cmp_slt1_v4i32:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX2-NEXT: vpcmpgtd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vmovmskps %xmm0, %eax
+; AVX2-NEXT: xorl $15, %eax
+; AVX2-NEXT: # kill: def $al killed $al killed $eax
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: cmp_slt1_v4i32:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512-NEXT: vpcmpled %xmm1, %xmm0, %k0
+; AVX512-NEXT: kmovd %k0, %eax
+; AVX512-NEXT: # kill: def $al killed $al killed $eax
+; AVX512-NEXT: retq
+ %c = icmp slt <4 x i32> %x, splat (i32 1)
+ %r = bitcast <4 x i1> %c to i4
+ ret i4 %r
+}
+
+define i32 @cmp_slt1_v32i8(<32 x i8> %x) {
+; AVX2-LABEL: cmp_slt1_v32i8:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX2-NEXT: vpcmpgtb %ymm1, %ymm0, %ymm0
+; AVX2-NEXT: vpmovmskb %ymm0, %eax
+; AVX2-NEXT: notl %eax
+; AVX2-NEXT: vzeroupper
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: cmp_slt1_v32i8:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512-NEXT: vpcmpleb %ymm1, %ymm0, %k0
+; AVX512-NEXT: kmovd %k0, %eax
+; AVX512-NEXT: vzeroupper
+; AVX512-NEXT: retq
+ %c = icmp slt <32 x i8> %x, splat (i8 1)
+ %r = bitcast <32 x i1> %c to i32
+ ret i32 %r
+}
diff --git a/llvm/test/CodeGen/X86/combine-icmp.ll b/llvm/test/CodeGen/X86/combine-icmp.ll
index b9cdaa04c1055..b056991284e7e 100644
--- a/llvm/test/CodeGen/X86/combine-icmp.ll
+++ b/llvm/test/CodeGen/X86/combine-icmp.ll
@@ -890,7 +890,8 @@ define i64 @concat_icmp_v64i8_v32i8(<32 x i8> %a0, <32 x i8> %a1) {
; AVX512: # %bb.0:
; AVX512-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0
; AVX512-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0
-; AVX512-NEXT: vpcmpltb {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %k0
+; AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512-NEXT: vpcmpleb %zmm1, %zmm0, %k0
; AVX512-NEXT: kmovq %k0, %rax
; AVX512-NEXT: vzeroupper
; AVX512-NEXT: retq
>From c7cb3f8ed3b0c3ccb691fc7c8d8eef053c668839 Mon Sep 17 00:00:00 2001
From: shaurya2k06 <shaurya2k06 at gmail.com>
Date: Mon, 17 Aug 2026 23:05:43 +0530
Subject: [PATCH 2/2] [X86] Move AVX512 sle-vs-zero rewrite into
LowerIntVSETCC_AVX512
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Do the splat(±1) → vs-zero adjust next to the existing SETLT→SETGT
swap, as suggested in review. Keep the combineSetCC guard so we do
not turn SETLE/SETGE vs zero back into LT/GT vs ±1.
Update tests that compared against splat(-1)/splat(1).
Signed-off-by: shaurya2k06 <shaurya2k06 at gmail.com>
Assisted-by: Cursor
---
llvm/lib/Target/X86/X86ISelLowering.cpp | 67 ++++++++-----------
.../CodeGen/X86/avx512-broadcast-unfold.ll | 24 +++----
llvm/test/CodeGen/X86/cmp-select-sign.ll | 4 +-
llvm/test/CodeGen/X86/vector-pcmp.ll | 54 +++++++--------
4 files changed, 68 insertions(+), 81 deletions(-)
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 9c9993e131022..95105db3a3242 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -24418,6 +24418,30 @@ static SDValue LowerIntVSETCC_AVX512(SDValue Op, const SDLoc &dl,
"Cannot set masked compare for this operation");
ISD::CondCode SetCCOpcode = cast<CondCodeSDNode>(CC)->get();
+ EVT OpVT = Op0.getValueType();
+ APInt C;
+
+ // Prefer a compare against zero over splat(±1), which becomes a
+ // constant-pool load. Analogous to TranslateX86CC for scalars.
+ if (SetCCOpcode == ISD::SETLT &&
+ ISD::isConstantSplatVector(Op1.getNode(), C) && C.isOne()) {
+ SetCCOpcode = ISD::SETLE;
+ Op1 = DAG.getConstant(0, dl, OpVT);
+ } else if (SetCCOpcode == ISD::SETGT &&
+ ISD::isConstantSplatVector(Op0.getNode(), C) && C.isOne()) {
+ SetCCOpcode = ISD::SETLE;
+ Op0 = Op1;
+ Op1 = DAG.getConstant(0, dl, OpVT);
+ } else if (SetCCOpcode == ISD::SETGT &&
+ ISD::isConstantSplatVector(Op1.getNode(), C) && C.isAllOnes()) {
+ SetCCOpcode = ISD::SETGE;
+ Op1 = DAG.getConstant(0, dl, OpVT);
+ } else if (SetCCOpcode == ISD::SETLT &&
+ ISD::isConstantSplatVector(Op0.getNode(), C) && C.isAllOnes()) {
+ SetCCOpcode = ISD::SETGE;
+ Op0 = Op1;
+ Op1 = DAG.getConstant(0, dl, OpVT);
+ }
// Prefer SETGT over SETLT.
if (SetCCOpcode == ISD::SETLT) {
@@ -58307,38 +58331,6 @@ static SDValue combineAVX512SetCCToKMOV(EVT VT, SDValue Op0, ISD::CondCode CC,
return Bitcast;
}
-/// AVX512 VPCMP encodes all integer condcodes. Prefer a compare against zero
-/// over the canonical slt/sgt vs splat(±1), which otherwise becomes a
-/// constant-pool load. Analogous to TranslateX86CC for scalars.
-static SDValue combineAVX512SetCCPreferZero(EVT VT, SDValue LHS, SDValue RHS,
- ISD::CondCode CC, const SDLoc &DL,
- SelectionDAG &DAG,
- const X86Subtarget &Subtarget) {
- if (!Subtarget.hasAVX512() || !VT.isVectorOf(MVT::i1) ||
- !LHS.getValueType().isInteger())
- return SDValue();
-
- EVT OpVT = LHS.getValueType();
- APInt C;
- // slt x, 1 / sgt 1, x -> sle x, 0
- if ((CC == ISD::SETLT && ISD::isConstantSplatVector(RHS.getNode(), C) &&
- C.isOne()) ||
- (CC == ISD::SETGT && ISD::isConstantSplatVector(LHS.getNode(), C) &&
- C.isOne())) {
- SDValue X = CC == ISD::SETLT ? LHS : RHS;
- return DAG.getSetCC(DL, VT, X, DAG.getConstant(0, DL, OpVT), ISD::SETLE);
- }
- // sgt x, -1 / slt -1, x -> sge x, 0
- if ((CC == ISD::SETGT && ISD::isConstantSplatVector(RHS.getNode(), C) &&
- C.isAllOnes()) ||
- (CC == ISD::SETLT && ISD::isConstantSplatVector(LHS.getNode(), C) &&
- C.isAllOnes())) {
- SDValue X = CC == ISD::SETGT ? LHS : RHS;
- return DAG.getSetCC(DL, VT, X, DAG.getConstant(0, DL, OpVT), ISD::SETGE);
- }
- return SDValue();
-}
-
static SDValue combineSetCC(SDNode *N, SelectionDAG &DAG,
TargetLowering::DAGCombinerInfo &DCI,
const X86Subtarget &Subtarget) {
@@ -58471,10 +58463,6 @@ static SDValue combineSetCC(SDNode *N, SelectionDAG &DAG,
// use `PCMPGT` if the result is mean to stay in a vector (and if its going to
// a mask, there are signed AVX512 comparisons).
if (VT.isVector() && OpVT.isVector() && OpVT.isInteger()) {
- if (SDValue V = combineAVX512SetCCPreferZero(VT, LHS, RHS, CC, DL, DAG,
- Subtarget))
- return V;
-
bool CanMakeSigned = false;
if (ISD::isUnsignedIntSetCC(CC)) {
KnownBits CmpKnown =
@@ -58493,8 +58481,7 @@ static SDValue combineSetCC(SDNode *N, SelectionDAG &DAG,
if (CanMakeSigned || ISD::isSignedIntSetCC(CC)) {
// AVX512 can encode LE/GE against zero; do not turn that into LT/GT
// vs ±1.
- const bool KeepZeroCmp =
- Subtarget.hasAVX512() && VT.isVectorOf(MVT::i1);
+ const bool KeepZeroCmp = Subtarget.hasAVX512() && VT.isVectorOf(MVT::i1);
SDValue LHSOut = LHS;
SDValue RHSOut = RHS;
ISD::CondCode NewCC = CC;
@@ -58522,8 +58509,8 @@ static SDValue combineSetCC(SDNode *N, SelectionDAG &DAG,
if (SDValue NewLHS = incDecVectorConstant(LHS, DAG, /*IsInc*/ false,
/*NSW*/ true))
LHSOut = NewLHS;
- else if (SDValue NewRHS = incDecVectorConstant(
- RHS, DAG, /*IsInc*/ true, /*NSW*/ true)) {
+ else if (SDValue NewRHS = incDecVectorConstant(RHS, DAG, /*IsInc*/ true,
+ /*NSW*/ true)) {
if (KeepZeroCmp && ISD::isConstantSplatVectorAllZeros(RHS.getNode()))
break;
RHSOut = NewRHS;
diff --git a/llvm/test/CodeGen/X86/avx512-broadcast-unfold.ll b/llvm/test/CodeGen/X86/avx512-broadcast-unfold.ll
index 2f86499a2df9e..32dce4042d55e 100644
--- a/llvm/test/CodeGen/X86/avx512-broadcast-unfold.ll
+++ b/llvm/test/CodeGen/X86/avx512-broadcast-unfold.ll
@@ -3552,12 +3552,12 @@ define void @bcast_unfold_pcmp_v4i32(ptr %arg) {
; CHECK-LABEL: bcast_unfold_pcmp_v4i32:
; CHECK: # %bb.0: # %bb
; CHECK-NEXT: xorl %eax, %eax
-; CHECK-NEXT: vpbroadcastd {{.*#+}} xmm0 = [1,1,1,1]
+; CHECK-NEXT: vpxor %xmm0, %xmm0, %xmm0
; CHECK-NEXT: vpbroadcastd {{.*#+}} xmm1 = [3,3,3,3]
; CHECK-NEXT: .p2align 4
; CHECK-NEXT: .LBB108_1: # %bb1
; CHECK-NEXT: # =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: vpcmpgtd (%rdi,%rax,4), %xmm0, %k1
+; CHECK-NEXT: vpcmpnltd (%rdi,%rax,4), %xmm0, %k1
; CHECK-NEXT: vmovdqu32 %xmm1, (%rdi,%rax,4) {%k1}
; CHECK-NEXT: addq $4, %rax
; CHECK-NEXT: cmpq $1023, %rax # imm = 0x3FF
@@ -3586,12 +3586,12 @@ define void @bcast_unfold_pcmp_v8i32(ptr %arg) {
; CHECK-LABEL: bcast_unfold_pcmp_v8i32:
; CHECK: # %bb.0: # %bb
; CHECK-NEXT: xorl %eax, %eax
-; CHECK-NEXT: vpbroadcastd {{.*#+}} ymm0 = [1,1,1,1,1,1,1,1]
+; CHECK-NEXT: vpxor %xmm0, %xmm0, %xmm0
; CHECK-NEXT: vpbroadcastd {{.*#+}} ymm1 = [3,3,3,3,3,3,3,3]
; CHECK-NEXT: .p2align 4
; CHECK-NEXT: .LBB109_1: # %bb1
; CHECK-NEXT: # =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: vpcmpgtd (%rdi,%rax,4), %ymm0, %k1
+; CHECK-NEXT: vpcmpnltd (%rdi,%rax,4), %ymm0, %k1
; CHECK-NEXT: vmovdqu32 %ymm1, (%rdi,%rax,4) {%k1}
; CHECK-NEXT: addq $8, %rax
; CHECK-NEXT: cmpq $1023, %rax # imm = 0x3FF
@@ -3621,12 +3621,12 @@ define void @bcast_unfold_pcmp_v16i32(ptr %arg) {
; CHECK-LABEL: bcast_unfold_pcmp_v16i32:
; CHECK: # %bb.0: # %bb
; CHECK-NEXT: xorl %eax, %eax
-; CHECK-NEXT: vpbroadcastd {{.*#+}} zmm0 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
+; CHECK-NEXT: vpxor %xmm0, %xmm0, %xmm0
; CHECK-NEXT: vpbroadcastd {{.*#+}} zmm1 = [3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3]
; CHECK-NEXT: .p2align 4
; CHECK-NEXT: .LBB110_1: # %bb1
; CHECK-NEXT: # =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: vpcmpgtd (%rdi,%rax,4), %zmm0, %k1
+; CHECK-NEXT: vpcmpnltd (%rdi,%rax,4), %zmm0, %k1
; CHECK-NEXT: vmovdqu32 %zmm1, (%rdi,%rax,4) {%k1}
; CHECK-NEXT: addq $16, %rax
; CHECK-NEXT: cmpq $1023, %rax # imm = 0x3FF
@@ -3656,12 +3656,12 @@ define void @bcast_unfold_pcmp_v2i64(ptr %arg) {
; CHECK-LABEL: bcast_unfold_pcmp_v2i64:
; CHECK: # %bb.0: # %bb
; CHECK-NEXT: xorl %eax, %eax
-; CHECK-NEXT: vpbroadcastq {{.*#+}} xmm0 = [1,1]
+; CHECK-NEXT: vpxor %xmm0, %xmm0, %xmm0
; CHECK-NEXT: vpbroadcastq {{.*#+}} xmm1 = [3,3]
; CHECK-NEXT: .p2align 4
; CHECK-NEXT: .LBB111_1: # %bb1
; CHECK-NEXT: # =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: vpcmpgtq (%rdi,%rax,8), %xmm0, %k1
+; CHECK-NEXT: vpcmpnltq (%rdi,%rax,8), %xmm0, %k1
; CHECK-NEXT: vmovdqu64 %xmm1, (%rdi,%rax,8) {%k1}
; CHECK-NEXT: addq $2, %rax
; CHECK-NEXT: cmpq $1023, %rax # imm = 0x3FF
@@ -3690,12 +3690,12 @@ define void @bcast_unfold_pcmp_v4i64(ptr %arg) {
; CHECK-LABEL: bcast_unfold_pcmp_v4i64:
; CHECK: # %bb.0: # %bb
; CHECK-NEXT: xorl %eax, %eax
-; CHECK-NEXT: vpbroadcastq {{.*#+}} ymm0 = [1,1,1,1]
+; CHECK-NEXT: vpxor %xmm0, %xmm0, %xmm0
; CHECK-NEXT: vpbroadcastq {{.*#+}} ymm1 = [3,3,3,3]
; CHECK-NEXT: .p2align 4
; CHECK-NEXT: .LBB112_1: # %bb1
; CHECK-NEXT: # =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: vpcmpgtq (%rdi,%rax,8), %ymm0, %k1
+; CHECK-NEXT: vpcmpnltq (%rdi,%rax,8), %ymm0, %k1
; CHECK-NEXT: vmovdqu64 %ymm1, (%rdi,%rax,8) {%k1}
; CHECK-NEXT: addq $4, %rax
; CHECK-NEXT: cmpq $1023, %rax # imm = 0x3FF
@@ -3725,12 +3725,12 @@ define void @bcast_unfold_pcmp_v8i64(ptr %arg) {
; CHECK-LABEL: bcast_unfold_pcmp_v8i64:
; CHECK: # %bb.0: # %bb
; CHECK-NEXT: xorl %eax, %eax
-; CHECK-NEXT: vpbroadcastq {{.*#+}} zmm0 = [1,1,1,1,1,1,1,1]
+; CHECK-NEXT: vpxor %xmm0, %xmm0, %xmm0
; CHECK-NEXT: vpbroadcastq {{.*#+}} zmm1 = [3,3,3,3,3,3,3,3]
; CHECK-NEXT: .p2align 4
; CHECK-NEXT: .LBB113_1: # %bb1
; CHECK-NEXT: # =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: vpcmpgtq (%rdi,%rax,8), %zmm0, %k1
+; CHECK-NEXT: vpcmpnltq (%rdi,%rax,8), %zmm0, %k1
; CHECK-NEXT: vmovdqu64 %zmm1, (%rdi,%rax,8) {%k1}
; CHECK-NEXT: addq $8, %rax
; CHECK-NEXT: cmpq $1023, %rax # imm = 0x3FF
diff --git a/llvm/test/CodeGen/X86/cmp-select-sign.ll b/llvm/test/CodeGen/X86/cmp-select-sign.ll
index be6f2a6d05192..5093e5d33f654 100644
--- a/llvm/test/CodeGen/X86/cmp-select-sign.ll
+++ b/llvm/test/CodeGen/X86/cmp-select-sign.ll
@@ -559,8 +559,8 @@ define <4 x i32> @not_sign_4xi32_select_false_breaks_pattern(<4 x i32> %a) {
;
; CHECK-AVX512-LABEL: not_sign_4xi32_select_false_breaks_pattern:
; CHECK-AVX512: # %bb.0:
-; CHECK-AVX512-NEXT: vpcmpeqd %xmm1, %xmm1, %xmm1
-; CHECK-AVX512-NEXT: vpcmpgtd %xmm1, %xmm0, %k1
+; CHECK-AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; CHECK-AVX512-NEXT: vpcmpnltd %xmm1, %xmm0, %k1
; CHECK-AVX512-NEXT: vpmovsxbd {{.*#+}} xmm0 = [4294967295,4294967295,4294967295,1]
; CHECK-AVX512-NEXT: vpbroadcastd {{.*#+}} xmm0 {%k1} = [1,1,1,1]
; CHECK-AVX512-NEXT: retq
diff --git a/llvm/test/CodeGen/X86/vector-pcmp.ll b/llvm/test/CodeGen/X86/vector-pcmp.ll
index 69785909c434f..c9396ce13e58b 100644
--- a/llvm/test/CodeGen/X86/vector-pcmp.ll
+++ b/llvm/test/CodeGen/X86/vector-pcmp.ll
@@ -1447,18 +1447,18 @@ define <2 x i1> @ispositive_mask_v2i64_v2i1(<2 x i64> %x, <2 x i1> %y) {
; AVX512F-LABEL: ispositive_mask_v2i64_v2i1:
; AVX512F: # %bb.0:
; AVX512F-NEXT: vpsllq $63, %xmm1, %xmm1
-; AVX512F-NEXT: vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT: vpcmpgtq %xmm2, %xmm0, %k1
+; AVX512F-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX512F-NEXT: vpcmpnltq %xmm2, %xmm0, %k1
+; AVX512F-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
; AVX512F-NEXT: vptestmq %xmm1, %xmm1, %k1 {%k1}
-; AVX512F-NEXT: vmovdqa64 %xmm2, %xmm0 {%k1} {z}
+; AVX512F-NEXT: vmovdqa64 %xmm0, %xmm0 {%k1} {z}
; AVX512F-NEXT: retq
;
; AVX512DQBW-LABEL: ispositive_mask_v2i64_v2i1:
; AVX512DQBW: # %bb.0:
; AVX512DQBW-NEXT: vpsllq $63, %xmm1, %xmm1
; AVX512DQBW-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVX512DQBW-NEXT: vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX512DQBW-NEXT: vpcmpgtq %xmm3, %xmm0, %k1
+; AVX512DQBW-NEXT: vpcmpnltq %xmm2, %xmm0, %k1
; AVX512DQBW-NEXT: vpcmpgtq %xmm1, %xmm2, %k0 {%k1}
; AVX512DQBW-NEXT: vpmovm2q %k0, %xmm0
; AVX512DQBW-NEXT: retq
@@ -1489,18 +1489,19 @@ define <4 x i1> @is_positive_mask_v4i32_v4i1(<4 x i32> %x, <4 x i1> %y) {
; AVX512F-LABEL: is_positive_mask_v4i32_v4i1:
; AVX512F: # %bb.0:
; AVX512F-NEXT: vpslld $31, %xmm1, %xmm1
-; AVX512F-NEXT: vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT: vpcmpgtd %xmm2, %xmm0, %k1
+; AVX512F-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX512F-NEXT: vpcmpnltd %xmm2, %xmm0, %k1
+; AVX512F-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
; AVX512F-NEXT: vptestmd %xmm1, %xmm1, %k1 {%k1}
-; AVX512F-NEXT: vmovdqa32 %xmm2, %xmm0 {%k1} {z}
+; AVX512F-NEXT: vmovdqa32 %xmm0, %xmm0 {%k1} {z}
; AVX512F-NEXT: retq
;
; AVX512DQBW-LABEL: is_positive_mask_v4i32_v4i1:
; AVX512DQBW: # %bb.0:
; AVX512DQBW-NEXT: vpslld $31, %xmm1, %xmm1
; AVX512DQBW-NEXT: vpmovd2m %xmm1, %k1
-; AVX512DQBW-NEXT: vpcmpeqd %xmm1, %xmm1, %xmm1
-; AVX512DQBW-NEXT: vpcmpgtd %xmm1, %xmm0, %k0 {%k1}
+; AVX512DQBW-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512DQBW-NEXT: vpcmpnltd %xmm1, %xmm0, %k0 {%k1}
; AVX512DQBW-NEXT: vpmovm2d %k0, %xmm0
; AVX512DQBW-NEXT: retq
%cmp = icmp sgt <4 x i32> %x, <i32 -1, i32 -1, i32 -1, i32 -1>
@@ -1537,8 +1538,7 @@ define <8 x i1> @is_positive_mask_v8i16_v8i1(<8 x i16> %x, <8 x i1> %y) {
; AVX512DQBW: # %bb.0:
; AVX512DQBW-NEXT: vpsllw $15, %xmm1, %xmm1
; AVX512DQBW-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVX512DQBW-NEXT: vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX512DQBW-NEXT: vpcmpgtw %xmm3, %xmm0, %k1
+; AVX512DQBW-NEXT: vpcmpnltw %xmm2, %xmm0, %k1
; AVX512DQBW-NEXT: vpcmpgtw %xmm1, %xmm2, %k0 {%k1}
; AVX512DQBW-NEXT: vpmovm2w %k0, %xmm0
; AVX512DQBW-NEXT: retq
@@ -1580,8 +1580,8 @@ define <16 x i1> @is_positive_mask_v16i8_v16i1(<16 x i8> %x, <16 x i1> %y) {
; AVX512DQBW: # %bb.0:
; AVX512DQBW-NEXT: vpsllw $7, %xmm1, %xmm1
; AVX512DQBW-NEXT: vpmovb2m %xmm1, %k1
-; AVX512DQBW-NEXT: vpcmpeqd %xmm1, %xmm1, %xmm1
-; AVX512DQBW-NEXT: vpcmpgtb %xmm1, %xmm0, %k0 {%k1}
+; AVX512DQBW-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512DQBW-NEXT: vpcmpnltb %xmm1, %xmm0, %k0 {%k1}
; AVX512DQBW-NEXT: vpmovm2b %k0, %xmm0
; AVX512DQBW-NEXT: retq
%cmp = icmp sgt <16 x i8> %x, <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>
@@ -1630,8 +1630,8 @@ define <4 x i1> @is_positive_mask_v4i64_v4i1(<4 x i64> %x, <4 x i1> %y) {
; AVX512F-LABEL: is_positive_mask_v4i64_v4i1:
; AVX512F: # %bb.0:
; AVX512F-NEXT: vpslld $31, %xmm1, %xmm1
-; AVX512F-NEXT: vpcmpeqd %ymm2, %ymm2, %ymm2
-; AVX512F-NEXT: vpcmpgtq %ymm2, %ymm0, %k1
+; AVX512F-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX512F-NEXT: vpcmpnltq %ymm2, %ymm0, %k1
; AVX512F-NEXT: vptestmd %xmm1, %xmm1, %k1 {%k1}
; AVX512F-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0
; AVX512F-NEXT: vmovdqa32 %xmm0, %xmm0 {%k1} {z}
@@ -1642,8 +1642,7 @@ define <4 x i1> @is_positive_mask_v4i64_v4i1(<4 x i64> %x, <4 x i1> %y) {
; AVX512DQBW: # %bb.0:
; AVX512DQBW-NEXT: vpslld $31, %xmm1, %xmm1
; AVX512DQBW-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVX512DQBW-NEXT: vpcmpeqd %ymm3, %ymm3, %ymm3
-; AVX512DQBW-NEXT: vpcmpgtq %ymm3, %ymm0, %k1
+; AVX512DQBW-NEXT: vpcmpnltq %ymm2, %ymm0, %k1
; AVX512DQBW-NEXT: vpcmpgtd %xmm1, %xmm2, %k0 {%k1}
; AVX512DQBW-NEXT: vpmovm2d %k0, %xmm0
; AVX512DQBW-NEXT: vzeroupper
@@ -1688,10 +1687,11 @@ define <8 x i1> @is_positive_mask_v8i32_v8i1(<8 x i32> %x, <8 x i1> %y) {
; AVX512F: # %bb.0:
; AVX512F-NEXT: vpmovsxwd %xmm1, %ymm1
; AVX512F-NEXT: vpslld $31, %ymm1, %ymm1
-; AVX512F-NEXT: vpcmpeqd %ymm2, %ymm2, %ymm2
-; AVX512F-NEXT: vpcmpgtd %ymm2, %ymm0, %k1
+; AVX512F-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX512F-NEXT: vpcmpnltd %ymm2, %ymm0, %k1
+; AVX512F-NEXT: vpcmpeqd %ymm0, %ymm0, %ymm0
; AVX512F-NEXT: vptestmd %ymm1, %ymm1, %k1 {%k1}
-; AVX512F-NEXT: vmovdqa32 %ymm2, %ymm0 {%k1} {z}
+; AVX512F-NEXT: vmovdqa32 %ymm0, %ymm0 {%k1} {z}
; AVX512F-NEXT: vpmovdw %ymm0, %xmm0
; AVX512F-NEXT: vzeroupper
; AVX512F-NEXT: retq
@@ -1700,8 +1700,8 @@ define <8 x i1> @is_positive_mask_v8i32_v8i1(<8 x i32> %x, <8 x i1> %y) {
; AVX512DQBW: # %bb.0:
; AVX512DQBW-NEXT: vpsllw $15, %xmm1, %xmm1
; AVX512DQBW-NEXT: vpmovw2m %xmm1, %k1
-; AVX512DQBW-NEXT: vpcmpeqd %ymm1, %ymm1, %ymm1
-; AVX512DQBW-NEXT: vpcmpgtd %ymm1, %ymm0, %k0 {%k1}
+; AVX512DQBW-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512DQBW-NEXT: vpcmpnltd %ymm1, %ymm0, %k0 {%k1}
; AVX512DQBW-NEXT: vpmovm2w %k0, %xmm0
; AVX512DQBW-NEXT: vzeroupper
; AVX512DQBW-NEXT: retq
@@ -1759,8 +1759,8 @@ define <16 x i1> @is_positive_mask_v16i16_v16i1(<16 x i16> %x, <16 x i1> %y) {
; AVX512DQBW: # %bb.0:
; AVX512DQBW-NEXT: vpsllw $7, %xmm1, %xmm1
; AVX512DQBW-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVX512DQBW-NEXT: vpcmpeqd %ymm3, %ymm3, %ymm3
-; AVX512DQBW-NEXT: vpcmpgtw %ymm3, %ymm0, %k1
+; AVX512DQBW-NEXT: vpxor %xmm3, %xmm3, %xmm3
+; AVX512DQBW-NEXT: vpcmpnltw %ymm3, %ymm0, %k1
; AVX512DQBW-NEXT: vpcmpgtb %xmm1, %xmm2, %k0 {%k1}
; AVX512DQBW-NEXT: vpmovm2b %k0, %xmm0
; AVX512DQBW-NEXT: vzeroupper
@@ -1925,10 +1925,10 @@ define <32 x i1> @is_positive_mask_v32i8_v32i1(<32 x i8> %x, <32 x i1> %y) {
;
; AVX512DQBW-LABEL: is_positive_mask_v32i8_v32i1:
; AVX512DQBW: # %bb.0:
+; AVX512DQBW-NEXT: vpxor %xmm2, %xmm2, %xmm2
; AVX512DQBW-NEXT: vpsllw $7, %ymm1, %ymm1
; AVX512DQBW-NEXT: vpmovb2m %ymm1, %k1
-; AVX512DQBW-NEXT: vpcmpeqd %ymm1, %ymm1, %ymm1
-; AVX512DQBW-NEXT: vpcmpgtb %ymm1, %ymm0, %k0 {%k1}
+; AVX512DQBW-NEXT: vpcmpnltb %ymm2, %ymm0, %k0 {%k1}
; AVX512DQBW-NEXT: vpmovm2b %k0, %ymm0
; AVX512DQBW-NEXT: retq
%cmp = icmp sgt <32 x i8> %x, <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>
More information about the llvm-commits
mailing list