[llvm] [X86] combineConcatVectorOps - add 512-bit PCMPEQ/PCMPGT handling (PR #202928)
Simon Pilgrim via llvm-commits
llvm-commits at lists.llvm.org
Wed Jun 10 03:57:35 PDT 2026
https://github.com/RKSimon created https://github.com/llvm/llvm-project/pull/202928
If we can freely concatenate both operands, then its worth replacing with a VPCMP+VPMOVM2 pair
Managed to notice this while triaging #198162 - and the AVX512DQ SGT test shows another vpmovq2m+vpmovm2q pair codegen issue :(
>From 57f098c7a03f8934fae2c59a20658880b35a6e2d Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev at redking.me.uk>
Date: Wed, 10 Jun 2026 11:56:20 +0100
Subject: [PATCH] [X86] combineConcatVectorOps - add 512-bit PCMPEQ/PCMPGT
handling
If we can freely concatenate both operands, then its worth replacing with a VPCMP+VPMOVM2 pair
Managed to notice this while triaging #198162 - and the AVX512DQ SGT test shows another vpmovq2m+vpmovm2q pair codegen issue :(
---
llvm/lib/Target/X86/X86ISelLowering.cpp | 14 ++-
llvm/test/CodeGen/X86/pr53842.ll | 121 +++++++++++++++---------
2 files changed, 87 insertions(+), 48 deletions(-)
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 3299023860af0..eb65d5d6dcc66 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -60974,7 +60974,6 @@ static SDValue combineConcatVectorOps(const SDLoc &DL, MVT VT,
break;
case X86ISD::PCMPEQ:
case X86ISD::PCMPGT:
- // TODO: 512-bit PCMPEQ/PCMPGT -> VPCMP+VPMOVM2 handling.
if (!IsSplat && VT.is256BitVector() && Subtarget.hasInt256()) {
SDValue Concat0 = CombineSubOperand(VT, Ops, 0);
SDValue Concat1 = CombineSubOperand(VT, Ops, 1);
@@ -60984,6 +60983,19 @@ static SDValue combineConcatVectorOps(const SDLoc &DL, MVT VT,
Concat1 ? Concat1 : ConcatSubOperand(VT, Ops, 1));
break;
}
+ if (!IsSplat && VT.is512BitVector() && Subtarget.useAVX512Regs() &&
+ (EltSizeInBits >= 32 || Subtarget.useBWIRegs())) {
+ if (IsConcatFree(VT, Ops, 0) && IsConcatFree(VT, Ops, 1)) {
+ MVT BoolVT = VT.changeVectorElementType(MVT::i1);
+ SDValue Cmp =
+ DAG.getSetCC(DL, BoolVT, ConcatSubOperand(VT, Ops, 0),
+ ConcatSubOperand(VT, Ops, 1),
+ Opcode == X86ISD::PCMPEQ ? ISD::CondCode::SETEQ
+ : ISD::CondCode::SETGT);
+ return DAG.getNode(ISD::SIGN_EXTEND, DL, VT, Cmp);
+ }
+ break;
+ }
if (!IsSplat && VT == MVT::v8i32) {
// Without AVX2, see if we can cast the values to v8f32 and use fcmp.
diff --git a/llvm/test/CodeGen/X86/pr53842.ll b/llvm/test/CodeGen/X86/pr53842.ll
index 28f3af8202a20..0c81d03ea62b5 100644
--- a/llvm/test/CodeGen/X86/pr53842.ll
+++ b/llvm/test/CodeGen/X86/pr53842.ll
@@ -1,24 +1,33 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f | FileCheck %s
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512vl | FileCheck %s
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw | FileCheck %s
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512dq | FileCheck %s
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefixes=AVX512F
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512vl | FileCheck %s --check-prefixes=AVX512F
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw | FileCheck %s --check-prefixes=AVX512F
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512dq | FileCheck %s --check-prefixes=AVX512DQ
define void @PR53842_eq() {
-; CHECK-LABEL: PR53842_eq:
-; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; CHECK-NEXT: vpmovzxbq {{.*#+}} zmm1 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero,mem[2],zero,zero,zero,zero,zero,zero,zero,mem[3],zero,zero,zero,zero,zero,zero,zero,mem[4],zero,zero,zero,zero,zero,zero,zero,mem[5],zero,zero,zero,zero,zero,zero,zero,mem[6],zero,zero,zero,zero,zero,zero,zero,mem[7],zero,zero,zero,zero,zero,zero,zero
-; CHECK-NEXT: vextracti64x4 $1, %zmm1, %ymm2
-; CHECK-NEXT: vpxor %xmm3, %xmm3, %xmm3
-; CHECK-NEXT: vpcmpeqq %ymm3, %ymm2, %ymm2
-; CHECK-NEXT: vpcmpeqq %ymm3, %ymm1, %ymm1
-; CHECK-NEXT: vinserti64x4 $1, %ymm2, %zmm1, %zmm1
-; CHECK-NEXT: .p2align 4
-; CHECK-NEXT: .LBB0_1: # %vector.body
-; CHECK-NEXT: # =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: vpsubq %zmm1, %zmm0, %zmm0
-; CHECK-NEXT: jmp .LBB0_1
+; AVX512F-LABEL: PR53842_eq:
+; AVX512F: # %bb.0: # %entry
+; AVX512F-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX512F-NEXT: vpmovzxbq {{.*#+}} zmm1 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero,mem[2],zero,zero,zero,zero,zero,zero,zero,mem[3],zero,zero,zero,zero,zero,zero,zero,mem[4],zero,zero,zero,zero,zero,zero,zero,mem[5],zero,zero,zero,zero,zero,zero,zero,mem[6],zero,zero,zero,zero,zero,zero,zero,mem[7],zero,zero,zero,zero,zero,zero,zero
+; AVX512F-NEXT: vptestnmq %zmm1, %zmm1, %k1
+; AVX512F-NEXT: vpternlogd {{.*#+}} zmm1 = -1
+; AVX512F-NEXT: .p2align 4
+; AVX512F-NEXT: .LBB0_1: # %vector.body
+; AVX512F-NEXT: # =>This Inner Loop Header: Depth=1
+; AVX512F-NEXT: vpsubq %zmm1, %zmm0, %zmm0 {%k1}
+; AVX512F-NEXT: jmp .LBB0_1
+;
+; AVX512DQ-LABEL: PR53842_eq:
+; AVX512DQ: # %bb.0: # %entry
+; AVX512DQ-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX512DQ-NEXT: vpmovzxbq {{.*#+}} zmm1 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero,mem[2],zero,zero,zero,zero,zero,zero,zero,mem[3],zero,zero,zero,zero,zero,zero,zero,mem[4],zero,zero,zero,zero,zero,zero,zero,mem[5],zero,zero,zero,zero,zero,zero,zero,mem[6],zero,zero,zero,zero,zero,zero,zero,mem[7],zero,zero,zero,zero,zero,zero,zero
+; AVX512DQ-NEXT: vptestnmq %zmm1, %zmm1, %k0
+; AVX512DQ-NEXT: vpmovm2q %k0, %zmm1
+; AVX512DQ-NEXT: .p2align 4
+; AVX512DQ-NEXT: .LBB0_1: # %vector.body
+; AVX512DQ-NEXT: # =>This Inner Loop Header: Depth=1
+; AVX512DQ-NEXT: vpsubq %zmm1, %zmm0, %zmm0
+; AVX512DQ-NEXT: jmp .LBB0_1
entry:
br label %vector.body
@@ -37,20 +46,29 @@ middle.block:
}
define void @PR53842_sgt() {
-; CHECK-LABEL: PR53842_sgt:
-; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; CHECK-NEXT: vpmovsxbq (%rax), %zmm1
-; CHECK-NEXT: vextracti64x4 $1, %zmm1, %ymm2
-; CHECK-NEXT: vpxor %xmm3, %xmm3, %xmm3
-; CHECK-NEXT: vpcmpgtq %ymm2, %ymm3, %ymm2
-; CHECK-NEXT: vpcmpgtq %ymm1, %ymm3, %ymm1
-; CHECK-NEXT: vinserti64x4 $1, %ymm2, %zmm1, %zmm1
-; CHECK-NEXT: .p2align 4
-; CHECK-NEXT: .LBB1_1: # %vector.body
-; CHECK-NEXT: # =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: vpsubq %zmm1, %zmm0, %zmm0
-; CHECK-NEXT: jmp .LBB1_1
+; AVX512F-LABEL: PR53842_sgt:
+; AVX512F: # %bb.0: # %entry
+; AVX512F-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX512F-NEXT: vpmovsxbq (%rax), %zmm1
+; AVX512F-NEXT: vpcmpgtq %zmm1, %zmm0, %k1
+; AVX512F-NEXT: vpternlogd {{.*#+}} zmm1 = -1
+; AVX512F-NEXT: .p2align 4
+; AVX512F-NEXT: .LBB1_1: # %vector.body
+; AVX512F-NEXT: # =>This Inner Loop Header: Depth=1
+; AVX512F-NEXT: vpsubq %zmm1, %zmm0, %zmm0 {%k1}
+; AVX512F-NEXT: jmp .LBB1_1
+;
+; AVX512DQ-LABEL: PR53842_sgt:
+; AVX512DQ: # %bb.0: # %entry
+; AVX512DQ-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX512DQ-NEXT: vpmovsxbq (%rax), %zmm1
+; AVX512DQ-NEXT: vpmovq2m %zmm1, %k0
+; AVX512DQ-NEXT: vpmovm2q %k0, %zmm1
+; AVX512DQ-NEXT: .p2align 4
+; AVX512DQ-NEXT: .LBB1_1: # %vector.body
+; AVX512DQ-NEXT: # =>This Inner Loop Header: Depth=1
+; AVX512DQ-NEXT: vpsubq %zmm1, %zmm0, %zmm0
+; AVX512DQ-NEXT: jmp .LBB1_1
entry:
br label %vector.body
@@ -69,20 +87,29 @@ middle.block:
}
define void @PR53842_slt() {
-; CHECK-LABEL: PR53842_slt:
-; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; CHECK-NEXT: vpmovsxbq (%rax), %zmm1
-; CHECK-NEXT: vextracti64x4 $1, %zmm1, %ymm2
-; CHECK-NEXT: vpxor %xmm3, %xmm3, %xmm3
-; CHECK-NEXT: vpcmpgtq %ymm3, %ymm2, %ymm2
-; CHECK-NEXT: vpcmpgtq %ymm3, %ymm1, %ymm1
-; CHECK-NEXT: vinserti64x4 $1, %ymm2, %zmm1, %zmm1
-; CHECK-NEXT: .p2align 4
-; CHECK-NEXT: .LBB2_1: # %vector.body
-; CHECK-NEXT: # =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: vpsubq %zmm1, %zmm0, %zmm0
-; CHECK-NEXT: jmp .LBB2_1
+; AVX512F-LABEL: PR53842_slt:
+; AVX512F: # %bb.0: # %entry
+; AVX512F-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX512F-NEXT: vpmovsxbq (%rax), %zmm1
+; AVX512F-NEXT: vpcmpgtq %zmm0, %zmm1, %k1
+; AVX512F-NEXT: vpternlogd {{.*#+}} zmm1 = -1
+; AVX512F-NEXT: .p2align 4
+; AVX512F-NEXT: .LBB2_1: # %vector.body
+; AVX512F-NEXT: # =>This Inner Loop Header: Depth=1
+; AVX512F-NEXT: vpsubq %zmm1, %zmm0, %zmm0 {%k1}
+; AVX512F-NEXT: jmp .LBB2_1
+;
+; AVX512DQ-LABEL: PR53842_slt:
+; AVX512DQ: # %bb.0: # %entry
+; AVX512DQ-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX512DQ-NEXT: vpmovsxbq (%rax), %zmm1
+; AVX512DQ-NEXT: vpcmpgtq %zmm0, %zmm1, %k0
+; AVX512DQ-NEXT: vpmovm2q %k0, %zmm1
+; AVX512DQ-NEXT: .p2align 4
+; AVX512DQ-NEXT: .LBB2_1: # %vector.body
+; AVX512DQ-NEXT: # =>This Inner Loop Header: Depth=1
+; AVX512DQ-NEXT: vpsubq %zmm1, %zmm0, %zmm0
+; AVX512DQ-NEXT: jmp .LBB2_1
entry:
br label %vector.body
More information about the llvm-commits
mailing list