[llvm] 6997f4d - [X86] combineSetCCMOVMSK - fold allof(cmpeq(x, y)) -> ptest(sub(x,y)) (PR53379)
Simon Pilgrim via llvm-commits
llvm-commits at lists.llvm.org
Mon Jan 24 08:54:49 PST 2022
Author: Simon Pilgrim
Date: 2022-01-24T16:44:37Z
New Revision: 6997f4d07fa4b462dd3a02838a2cfed45db9c8a0
URL: https://github.com/llvm/llvm-project/commit/6997f4d07fa4b462dd3a02838a2cfed45db9c8a0
DIFF: https://github.com/llvm/llvm-project/commit/6997f4d07fa4b462dd3a02838a2cfed45db9c8a0.diff
LOG: [X86] combineSetCCMOVMSK - fold allof(cmpeq(x,y)) -> ptest(sub(x,y)) (PR53379)
As suggested on PR53379, for all-of icmp-eq patterns, we can use ptest(sub(x,y)) on SSE41+ targets
This is a generalization of the existing allof(cmpeq(x,0)) -> ptest(x) pattern
We can probably extend this further, in particularly to handle 256-bit cases on pre-AVX2 targets, but this part of the generalization is pretty trivial
Fixes Issue #53379
Added:
Modified:
llvm/lib/Target/X86/X86ISelLowering.cpp
llvm/test/CodeGen/X86/movmsk-cmp.ll
llvm/test/CodeGen/X86/vector-compare-all_of.ll
llvm/test/CodeGen/X86/vector-compare-any_of.ll
llvm/test/CodeGen/X86/vector-reduce-and-bool.ll
Removed:
################################################################################
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 10fdcc3d6ca81..2f8e97e63fd49 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -44457,12 +44457,15 @@ static SDValue combineSetCCMOVMSK(SDValue EFLAGS, X86::CondCode &CC,
// MOVMSK(PCMPEQ(X,0)) == -1 -> PTESTZ(X,X).
// MOVMSK(PCMPEQ(X,0)) != -1 -> !PTESTZ(X,X).
+ // MOVMSK(PCMPEQ(X,Y)) == -1 -> PTESTZ(SUB(X,Y),SUB(X,Y)).
+ // MOVMSK(PCMPEQ(X,Y)) != -1 -> !PTESTZ(SUB(X,Y),SUB(X,Y)).
if (IsAllOf && Subtarget.hasSSE41()) {
SDValue BC = peekThroughBitcasts(Vec);
- if (BC.getOpcode() == X86ISD::PCMPEQ &&
- ISD::isBuildVectorAllZeros(BC.getOperand(1).getNode())) {
+ if (BC.getOpcode() == X86ISD::PCMPEQ) {
MVT TestVT = VecVT.is128BitVector() ? MVT::v2i64 : MVT::v4i64;
- SDValue V = DAG.getBitcast(TestVT, BC.getOperand(0));
+ SDValue V = DAG.getNode(ISD::SUB, SDLoc(BC), BC.getValueType(),
+ BC.getOperand(0), BC.getOperand(1));
+ V = DAG.getBitcast(TestVT, V);
return DAG.getNode(X86ISD::PTEST, SDLoc(EFLAGS), MVT::i32, V, V);
}
}
@@ -44500,7 +44503,14 @@ static SDValue combineSetCCMOVMSK(SDValue EFLAGS, X86::CondCode &CC,
VecOp1.getConstantOperandAPInt(1) == 8 &&
(IsAnyOf || (SignExt0 && SignExt1))) {
SDLoc DL(EFLAGS);
- SDValue Result = DAG.getBitcast(MVT::v32i8, VecOp0.getOperand(0));
+ SDValue Result = peekThroughBitcasts(VecOp0.getOperand(0));
+ if (IsAllOf && Result.getOpcode() == X86ISD::PCMPEQ) {
+ SDValue V = DAG.getNode(ISD::SUB, DL, Result.getValueType(),
+ Result.getOperand(0), Result.getOperand(1));
+ V = DAG.getBitcast(MVT::v4i64, V);
+ return DAG.getNode(X86ISD::PTEST, SDLoc(EFLAGS), MVT::i32, V, V);
+ }
+ Result = DAG.getBitcast(MVT::v32i8, Result);
Result = DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, Result);
unsigned CmpMask = IsAnyOf ? 0 : 0xFFFFFFFF;
if (!SignExt0 || !SignExt1) {
diff --git a/llvm/test/CodeGen/X86/movmsk-cmp.ll b/llvm/test/CodeGen/X86/movmsk-cmp.ll
index 4488aeb273f70..68adeda025d7c 100644
--- a/llvm/test/CodeGen/X86/movmsk-cmp.ll
+++ b/llvm/test/CodeGen/X86/movmsk-cmp.ll
@@ -4162,17 +4162,15 @@ define i1 @movmsk_or_v2i64(<2 x i64> %x, <2 x i64> %y) {
;
; SSE41-LABEL: movmsk_or_v2i64:
; SSE41: # %bb.0:
-; SSE41-NEXT: pcmpeqq %xmm1, %xmm0
-; SSE41-NEXT: movmskpd %xmm0, %eax
-; SSE41-NEXT: cmpl $3, %eax
+; SSE41-NEXT: psubq %xmm1, %xmm0
+; SSE41-NEXT: ptest %xmm0, %xmm0
; SSE41-NEXT: setne %al
; SSE41-NEXT: retq
;
; AVX1OR2-LABEL: movmsk_or_v2i64:
; AVX1OR2: # %bb.0:
-; AVX1OR2-NEXT: vpcmpeqq %xmm1, %xmm0, %xmm0
-; AVX1OR2-NEXT: vmovmskpd %xmm0, %eax
-; AVX1OR2-NEXT: cmpl $3, %eax
+; AVX1OR2-NEXT: vpsubq %xmm1, %xmm0, %xmm0
+; AVX1OR2-NEXT: vptest %xmm0, %xmm0
; AVX1OR2-NEXT: setne %al
; AVX1OR2-NEXT: retq
;
diff --git a/llvm/test/CodeGen/X86/vector-compare-all_of.ll b/llvm/test/CodeGen/X86/vector-compare-all_of.ll
index a4a8367f93294..b399712553606 100644
--- a/llvm/test/CodeGen/X86/vector-compare-all_of.ll
+++ b/llvm/test/CodeGen/X86/vector-compare-all_of.ll
@@ -1249,9 +1249,8 @@ define i1 @bool_reduction_v8i32(<8 x i32> %x, <8 x i32> %y) {
; AVX2-LABEL: bool_reduction_v8i32:
; AVX2: # %bb.0:
; AVX2-NEXT: vpminud %ymm1, %ymm0, %ymm1
-; AVX2-NEXT: vpcmpeqd %ymm1, %ymm0, %ymm0
-; AVX2-NEXT: vmovmskps %ymm0, %eax
-; AVX2-NEXT: cmpb $-1, %al
+; AVX2-NEXT: vpsubd %ymm1, %ymm0, %ymm0
+; AVX2-NEXT: vptest %ymm0, %ymm0
; AVX2-NEXT: sete %al
; AVX2-NEXT: vzeroupper
; AVX2-NEXT: retq
@@ -1301,9 +1300,8 @@ define i1 @bool_reduction_v16i16(<16 x i16> %x, <16 x i16> %y) {
;
; AVX2-LABEL: bool_reduction_v16i16:
; AVX2: # %bb.0:
-; AVX2-NEXT: vpcmpeqw %ymm1, %ymm0, %ymm0
-; AVX2-NEXT: vpmovmskb %ymm0, %eax
-; AVX2-NEXT: cmpl $-1, %eax
+; AVX2-NEXT: vpsubw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT: vptest %ymm0, %ymm0
; AVX2-NEXT: sete %al
; AVX2-NEXT: vzeroupper
; AVX2-NEXT: retq
@@ -1354,9 +1352,8 @@ define i1 @bool_reduction_v32i8(<32 x i8> %x, <32 x i8> %y) {
;
; AVX2-LABEL: bool_reduction_v32i8:
; AVX2: # %bb.0:
-; AVX2-NEXT: vpcmpeqb %ymm1, %ymm0, %ymm0
-; AVX2-NEXT: vpmovmskb %ymm0, %eax
-; AVX2-NEXT: cmpl $-1, %eax
+; AVX2-NEXT: vpsubb %ymm1, %ymm0, %ymm0
+; AVX2-NEXT: vptest %ymm0, %ymm0
; AVX2-NEXT: sete %al
; AVX2-NEXT: vzeroupper
; AVX2-NEXT: retq
diff --git a/llvm/test/CodeGen/X86/vector-compare-any_of.ll b/llvm/test/CodeGen/X86/vector-compare-any_of.ll
index ee7e5f983c582..9af3d5e4969e8 100644
--- a/llvm/test/CodeGen/X86/vector-compare-any_of.ll
+++ b/llvm/test/CodeGen/X86/vector-compare-any_of.ll
@@ -959,17 +959,15 @@ define i1 @bool_reduction_v2i64(<2 x i64> %x, <2 x i64> %y) {
define i1 @bool_reduction_v4i32(<4 x i32> %x, <4 x i32> %y) {
; SSE-LABEL: bool_reduction_v4i32:
; SSE: # %bb.0:
-; SSE-NEXT: pcmpeqd %xmm1, %xmm0
-; SSE-NEXT: movmskps %xmm0, %eax
-; SSE-NEXT: cmpl $15, %eax
+; SSE-NEXT: psubd %xmm1, %xmm0
+; SSE-NEXT: ptest %xmm0, %xmm0
; SSE-NEXT: setne %al
; SSE-NEXT: retq
;
; AVX-LABEL: bool_reduction_v4i32:
; AVX: # %bb.0:
-; AVX-NEXT: vpcmpeqd %xmm1, %xmm0, %xmm0
-; AVX-NEXT: vmovmskps %xmm0, %eax
-; AVX-NEXT: cmpl $15, %eax
+; AVX-NEXT: vpsubd %xmm1, %xmm0, %xmm0
+; AVX-NEXT: vptest %xmm0, %xmm0
; AVX-NEXT: setne %al
; AVX-NEXT: retq
;
diff --git a/llvm/test/CodeGen/X86/vector-reduce-and-bool.ll b/llvm/test/CodeGen/X86/vector-reduce-and-bool.ll
index c9aef7b8e7404..be9ebf466eb0d 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-and-bool.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-and-bool.ll
@@ -1210,10 +1210,7 @@ define i1 @icmp0_v16i16_v16i1(<16 x i16>) {
;
; AVX2-LABEL: icmp0_v16i16_v16i1:
; AVX2: # %bb.0:
-; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT: vpcmpeqw %ymm1, %ymm0, %ymm0
-; AVX2-NEXT: vpmovmskb %ymm0, %eax
-; AVX2-NEXT: cmpl $-1, %eax
+; AVX2-NEXT: vptest %ymm0, %ymm0
; AVX2-NEXT: sete %al
; AVX2-NEXT: vzeroupper
; AVX2-NEXT: retq
@@ -1679,17 +1676,15 @@ define i1 @icmp_v2i64_v2i1(<2 x i64>, <2 x i64>) {
;
; SSE41-LABEL: icmp_v2i64_v2i1:
; SSE41: # %bb.0:
-; SSE41-NEXT: pcmpeqq %xmm1, %xmm0
-; SSE41-NEXT: movmskpd %xmm0, %eax
-; SSE41-NEXT: cmpb $3, %al
+; SSE41-NEXT: psubq %xmm1, %xmm0
+; SSE41-NEXT: ptest %xmm0, %xmm0
; SSE41-NEXT: sete %al
; SSE41-NEXT: retq
;
; AVX-LABEL: icmp_v2i64_v2i1:
; AVX: # %bb.0:
-; AVX-NEXT: vpcmpeqq %xmm1, %xmm0, %xmm0
-; AVX-NEXT: vmovmskpd %xmm0, %eax
-; AVX-NEXT: cmpb $3, %al
+; AVX-NEXT: vpsubq %xmm1, %xmm0, %xmm0
+; AVX-NEXT: vptest %xmm0, %xmm0
; AVX-NEXT: sete %al
; AVX-NEXT: retq
;
@@ -1728,19 +1723,25 @@ define i1 @icmp_v2i64_v2i1(<2 x i64>, <2 x i64>) {
}
define i1 @icmp_v4i32_v4i1(<4 x i32>, <4 x i32>) {
-; SSE-LABEL: icmp_v4i32_v4i1:
-; SSE: # %bb.0:
-; SSE-NEXT: pcmpeqd %xmm1, %xmm0
-; SSE-NEXT: movmskps %xmm0, %eax
-; SSE-NEXT: cmpb $15, %al
-; SSE-NEXT: sete %al
-; SSE-NEXT: retq
+; SSE2-LABEL: icmp_v4i32_v4i1:
+; SSE2: # %bb.0:
+; SSE2-NEXT: pcmpeqd %xmm1, %xmm0
+; SSE2-NEXT: movmskps %xmm0, %eax
+; SSE2-NEXT: cmpb $15, %al
+; SSE2-NEXT: sete %al
+; SSE2-NEXT: retq
+;
+; SSE41-LABEL: icmp_v4i32_v4i1:
+; SSE41: # %bb.0:
+; SSE41-NEXT: psubd %xmm1, %xmm0
+; SSE41-NEXT: ptest %xmm0, %xmm0
+; SSE41-NEXT: sete %al
+; SSE41-NEXT: retq
;
; AVX-LABEL: icmp_v4i32_v4i1:
; AVX: # %bb.0:
-; AVX-NEXT: vpcmpeqd %xmm1, %xmm0, %xmm0
-; AVX-NEXT: vmovmskps %xmm0, %eax
-; AVX-NEXT: cmpb $15, %al
+; AVX-NEXT: vpsubd %xmm1, %xmm0, %xmm0
+; AVX-NEXT: vptest %xmm0, %xmm0
; AVX-NEXT: sete %al
; AVX-NEXT: retq
;
@@ -1832,27 +1833,32 @@ define i1 @icmp_v8i16_v8i1(<8 x i16>, <8 x i16>) {
}
define i1 @icmp_v16i8_v16i1(<16 x i8>, <16 x i8>) {
-; SSE-LABEL: icmp_v16i8_v16i1:
-; SSE: # %bb.0:
-; SSE-NEXT: pcmpeqb %xmm1, %xmm0
-; SSE-NEXT: pmovmskb %xmm0, %eax
-; SSE-NEXT: cmpw $-1, %ax
-; SSE-NEXT: sete %al
-; SSE-NEXT: retq
+; SSE2-LABEL: icmp_v16i8_v16i1:
+; SSE2: # %bb.0:
+; SSE2-NEXT: pcmpeqb %xmm1, %xmm0
+; SSE2-NEXT: pmovmskb %xmm0, %eax
+; SSE2-NEXT: cmpw $-1, %ax
+; SSE2-NEXT: sete %al
+; SSE2-NEXT: retq
+;
+; SSE41-LABEL: icmp_v16i8_v16i1:
+; SSE41: # %bb.0:
+; SSE41-NEXT: psubb %xmm1, %xmm0
+; SSE41-NEXT: ptest %xmm0, %xmm0
+; SSE41-NEXT: sete %al
+; SSE41-NEXT: retq
;
; AVX-LABEL: icmp_v16i8_v16i1:
; AVX: # %bb.0:
-; AVX-NEXT: vpcmpeqb %xmm1, %xmm0, %xmm0
-; AVX-NEXT: vpmovmskb %xmm0, %eax
-; AVX-NEXT: cmpw $-1, %ax
+; AVX-NEXT: vpsubb %xmm1, %xmm0, %xmm0
+; AVX-NEXT: vptest %xmm0, %xmm0
; AVX-NEXT: sete %al
; AVX-NEXT: retq
;
; AVX512F-LABEL: icmp_v16i8_v16i1:
; AVX512F: # %bb.0:
-; AVX512F-NEXT: vpcmpeqb %xmm1, %xmm0, %xmm0
-; AVX512F-NEXT: vpmovmskb %xmm0, %eax
-; AVX512F-NEXT: cmpw $-1, %ax
+; AVX512F-NEXT: vpsubb %xmm1, %xmm0, %xmm0
+; AVX512F-NEXT: vptest %xmm0, %xmm0
; AVX512F-NEXT: sete %al
; AVX512F-NEXT: retq
;
@@ -1917,9 +1923,8 @@ define i1 @icmp_v4i64_v4i1(<4 x i64>, <4 x i64>) {
;
; AVX2-LABEL: icmp_v4i64_v4i1:
; AVX2: # %bb.0:
-; AVX2-NEXT: vpcmpeqq %ymm1, %ymm0, %ymm0
-; AVX2-NEXT: vmovmskpd %ymm0, %eax
-; AVX2-NEXT: cmpb $15, %al
+; AVX2-NEXT: vpsubq %ymm1, %ymm0, %ymm0
+; AVX2-NEXT: vptest %ymm0, %ymm0
; AVX2-NEXT: sete %al
; AVX2-NEXT: vzeroupper
; AVX2-NEXT: retq
@@ -1986,9 +1991,8 @@ define i1 @icmp_v8i32_v8i1(<8 x i32>, <8 x i32>) {
;
; AVX2-LABEL: icmp_v8i32_v8i1:
; AVX2: # %bb.0:
-; AVX2-NEXT: vpcmpeqd %ymm1, %ymm0, %ymm0
-; AVX2-NEXT: vmovmskps %ymm0, %eax
-; AVX2-NEXT: cmpb $-1, %al
+; AVX2-NEXT: vpsubd %ymm1, %ymm0, %ymm0
+; AVX2-NEXT: vptest %ymm0, %ymm0
; AVX2-NEXT: sete %al
; AVX2-NEXT: vzeroupper
; AVX2-NEXT: retq
@@ -2054,9 +2058,8 @@ define i1 @icmp_v16i16_v16i1(<16 x i16>, <16 x i16>) {
;
; AVX2-LABEL: icmp_v16i16_v16i1:
; AVX2: # %bb.0:
-; AVX2-NEXT: vpcmpeqw %ymm1, %ymm0, %ymm0
-; AVX2-NEXT: vpmovmskb %ymm0, %eax
-; AVX2-NEXT: cmpl $-1, %eax
+; AVX2-NEXT: vpsubw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT: vptest %ymm0, %ymm0
; AVX2-NEXT: sete %al
; AVX2-NEXT: vzeroupper
; AVX2-NEXT: retq
@@ -2119,9 +2122,8 @@ define i1 @icmp_v32i8_v32i1(<32 x i8>, <32 x i8>) {
;
; AVX2-LABEL: icmp_v32i8_v32i1:
; AVX2: # %bb.0:
-; AVX2-NEXT: vpcmpeqb %ymm1, %ymm0, %ymm0
-; AVX2-NEXT: vpmovmskb %ymm0, %eax
-; AVX2-NEXT: cmpl $-1, %eax
+; AVX2-NEXT: vpsubb %ymm1, %ymm0, %ymm0
+; AVX2-NEXT: vptest %ymm0, %ymm0
; AVX2-NEXT: sete %al
; AVX2-NEXT: vzeroupper
; AVX2-NEXT: retq
More information about the llvm-commits
mailing list