[llvm] [X86] X86CompressEVEX - Fold AVX-512 sign-mask compares to MOVMSK (PR #216884)
Simon Pilgrim via llvm-commits
llvm-commits at lists.llvm.org
Mon Sep 7 23:08:15 PDT 2026
https://github.com/RKSimon updated https://github.com/llvm/llvm-project/pull/216884
>From 9af97f8a80c07ec3d3560ae5f873307823943c39 Mon Sep 17 00:00:00 2001
From: Oscar Priego Verdugo <oscar.priegov at gmail.com>
Date: Mon, 17 Aug 2026 17:52:18 -0600
Subject: [PATCH 1/4] [X86] Fold AVX-512 sign-mask compares to MOVMSK
AVX-512 lowers complementary signed sign tests such as X >= 0 and
X > -1 to VPCMP* followed by KMOV.
Extend X86CompressEVEX to recognize the 128-bit and 256-bit B/D/Q
forms and replace them with the corresponding MOVMSK instruction:
VPCMPB -> VPMOVMSKB
VPCMPD -> VMOVMSKPS
VPCMPQ -> VMOVMSKPD
Complement only the meaningful lane-mask bits so that unused upper
bits preserve the zero-extension semantics of KMOV. Use NOT32r for
the 32-lane byte case, where all result bits are significant.
Add MIR coverage for the compression and negative cases involving
multiple uses, clobbers, unsupported predicates and constants,
narrowing KMOVs, extended registers, live EFLAGS, masked-move uses,
and incorrect all-ones widths.
Fixes #216774
Signed-off-by: Oscar Priego Verdugo <oscar.priegov at gmail.com>
---
llvm/lib/Target/X86/X86CompressEVEX.cpp | 162 ++++++++++--
llvm/test/CodeGen/X86/bitcast-sign-mask.ll | 229 +++++++++++++++++
.../CodeGen/X86/compress-evex-sign-mask.mir | 240 ++++++++++++++++++
.../CodeGen/X86/compress-evex-vpmov-kill.mir | 2 +-
4 files changed, 606 insertions(+), 27 deletions(-)
create mode 100644 llvm/test/CodeGen/X86/bitcast-sign-mask.ll
create mode 100644 llvm/test/CodeGen/X86/compress-evex-sign-mask.mir
diff --git a/llvm/lib/Target/X86/X86CompressEVEX.cpp b/llvm/lib/Target/X86/X86CompressEVEX.cpp
index 2c408851975e8..e34590a3ecfb2 100644
--- a/llvm/lib/Target/X86/X86CompressEVEX.cpp
+++ b/llvm/lib/Target/X86/X86CompressEVEX.cpp
@@ -181,29 +181,31 @@ static bool performCustomAdjustments(MachineInstr &MI, unsigned NewOpc) {
return true;
}
-static bool isKMovNarrowing(unsigned VPMOVOpc, unsigned KMOVOpc) {
- unsigned VPMOVBits = 0;
- switch (VPMOVOpc) {
+static unsigned getMovMskBits(unsigned Opc) {
+ switch (Opc) {
case X86::VPMOVQ2MZ128kr:
- VPMOVBits = 2;
- break;
+ case X86::VPCMPQZ128rri:
+ return 2;
case X86::VPMOVQ2MZ256kr:
case X86::VPMOVD2MZ128kr:
- VPMOVBits = 4;
- break;
+ case X86::VPCMPQZ256rri:
+ case X86::VPCMPDZ128rri:
+ return 4;
case X86::VPMOVD2MZ256kr:
- VPMOVBits = 8;
- break;
+ case X86::VPCMPDZ256rri:
+ return 8;
case X86::VPMOVB2MZ128kr:
- VPMOVBits = 16;
- break;
+ case X86::VPCMPBZ128rri:
+ return 16;
case X86::VPMOVB2MZ256kr:
- VPMOVBits = 32;
- break;
+ case X86::VPCMPBZ256rri:
+ return 32;
default:
- llvm_unreachable("Unknown VPMOV opcode");
+ llvm_unreachable("Unknown MOVMSK source opcode");
}
+}
+static bool isKMovNarrowing(unsigned MaskBits, unsigned KMOVOpc) {
unsigned KMOVSize = 0;
switch (KMOVOpc) {
case X86::KMOVBrk:
@@ -219,7 +221,44 @@ static bool isKMovNarrowing(unsigned VPMOVOpc, unsigned KMOVOpc) {
llvm_unreachable("Unknown KMOV opcode");
}
- return KMOVSize < VPMOVBits;
+ return KMOVSize < MaskBits;
+}
+
+static bool isZeroVector(const MachineInstr &MI) {
+ switch (MI.getOpcode()) {
+ case X86::VPXORrr:
+ case X86::VPXORYrr:
+ case X86::VXORPSrr:
+ case X86::VXORPSYrr:
+ return MI.getOperand(1).getReg() == MI.getOperand(2).getReg();
+ default:
+ return false;
+ }
+}
+
+static bool isAllOnesVector(const MachineInstr &MI, bool Is256Bit) {
+ switch (MI.getOpcode()) {
+ case X86::VPCMPEQDrr:
+ return !Is256Bit && MI.getOperand(1).getReg() == MI.getOperand(2).getReg();
+ case X86::VPCMPEQDYrr:
+ return MI.getOperand(1).getReg() == MI.getOperand(2).getReg();
+ default:
+ return false;
+ }
+}
+
+static MachineInstr *getSignMaskConstantDef(MachineInstr &MI, Register Reg,
+ bool IsZero, bool Is256Bit,
+ const TargetRegisterInfo *TRI) {
+ for (MachineInstr &DefMI : llvm::reverse(llvm::make_range(
+ MI.getParent()->begin(), MachineBasicBlock::iterator(MI)))) {
+ if (!DefMI.modifiesRegister(Reg, TRI))
+ continue;
+ return (IsZero ? isZeroVector(DefMI) : isAllOnesVector(DefMI, Is256Bit))
+ ? &DefMI
+ : nullptr;
+ }
+ return nullptr;
}
static bool isCompressibleBlendVUse(unsigned BlendOpc, unsigned UseOpc) {
@@ -273,23 +312,31 @@ static bool isCompressibleBlendVUse(unsigned BlendOpc, unsigned UseOpc) {
}
}
-// Try to compress VPMOV*2M chain patterns:
+// Try to compress VPMOV*2M and complementary signed sign-test patterns:
// vpmov*2m %xmm0, %k0 -> (erase this)
// kmov* %k0, %eax -> vmovmskp* %xmm0, %eax
+// vpcmpge* $0, %xmm0, %k0 -> (erase this) (X >= 0)
+// vpcmpgt* $-1, %xmm0, %k0 -> (erase this) (X > -1)
+// kmov* %k0, %eax -> vmovmskp* %xmm0, %eax
+// bounded complement of %eax
// and:
// vpmov*2m %xmm0, %k1 -> (erase this)
// vmov* %xmm1, %xmm2 {%k1} -> vblendv* %xmm0, %xmm2, %xmm1, %xmm2
-static bool tryCompressVPMOVPattern(MachineInstr &MI, MachineBasicBlock &MBB,
- const X86Subtarget &ST,
- SmallVectorImpl<MachineInstr *> &ToErase) {
+static bool tryCompressMovMskPattern(MachineInstr &MI, MachineBasicBlock &MBB,
+ const X86Subtarget &ST,
+ SmallVectorImpl<MachineInstr *> &ToErase) {
const X86InstrInfo *TII = ST.getInstrInfo();
const TargetRegisterInfo *TRI = ST.getRegisterInfo();
MachineRegisterInfo *MRI = &MBB.getParent()->getRegInfo();
unsigned Opc = MI.getOpcode();
- if (Opc != X86::VPMOVD2MZ128kr && Opc != X86::VPMOVD2MZ256kr &&
- Opc != X86::VPMOVQ2MZ128kr && Opc != X86::VPMOVQ2MZ256kr &&
- Opc != X86::VPMOVB2MZ128kr && Opc != X86::VPMOVB2MZ256kr)
+ bool IsSignMaskCmp = Opc == X86::VPCMPBZ128rri || Opc == X86::VPCMPBZ256rri ||
+ Opc == X86::VPCMPDZ128rri || Opc == X86::VPCMPDZ256rri ||
+ Opc == X86::VPCMPQZ128rri || Opc == X86::VPCMPQZ256rri;
+ if (!IsSignMaskCmp && Opc != X86::VPMOVD2MZ128kr &&
+ Opc != X86::VPMOVD2MZ256kr && Opc != X86::VPMOVQ2MZ128kr &&
+ Opc != X86::VPMOVQ2MZ256kr && Opc != X86::VPMOVB2MZ128kr &&
+ Opc != X86::VPMOVB2MZ256kr)
return false;
if (usesExtendedRegister(MI))
@@ -297,30 +344,59 @@ static bool tryCompressVPMOVPattern(MachineInstr &MI, MachineBasicBlock &MBB,
Register MaskReg = MI.getOperand(0).getReg();
Register SrcVecReg = MI.getOperand(1).getReg();
+ MachineInstr *ConstantDef = nullptr;
+ bool ConstantDefOnlyFeedsCmp = false;
+
+ if (IsSignMaskCmp) {
+ int64_t Pred = MI.getOperand(3).getImm();
+ if (Pred != 5 && Pred != 6)
+ return false;
+ Register ConstantReg = MI.getOperand(2).getReg();
+ bool Is256Bit = Opc == X86::VPCMPBZ256rri || Opc == X86::VPCMPDZ256rri ||
+ Opc == X86::VPCMPQZ256rri;
+ ConstantDef =
+ getSignMaskConstantDef(MI, ConstantReg, Pred == 5, Is256Bit, TRI);
+ if (!ConstantDef)
+ return false;
+ ConstantDefOnlyFeedsCmp = !TRI->regsOverlap(ConstantReg, SrcVecReg);
+ for (MachineInstr &UseMI :
+ llvm::make_range(std::next(MachineBasicBlock::iterator(*ConstantDef)),
+ MachineBasicBlock::iterator(MI)))
+ if (UseMI.readsRegister(ConstantReg, TRI)) {
+ ConstantDefOnlyFeedsCmp = false;
+ break;
+ }
+ }
unsigned MovMskOpc = 0;
unsigned BlendOpc = 0;
switch (Opc) {
+ case X86::VPCMPDZ128rri:
case X86::VPMOVD2MZ128kr:
MovMskOpc = X86::VMOVMSKPSrr;
BlendOpc = X86::VBLENDVPSrrr;
break;
+ case X86::VPCMPDZ256rri:
case X86::VPMOVD2MZ256kr:
MovMskOpc = X86::VMOVMSKPSYrr;
BlendOpc = X86::VBLENDVPSYrrr;
break;
+ case X86::VPCMPQZ128rri:
case X86::VPMOVQ2MZ128kr:
MovMskOpc = X86::VMOVMSKPDrr;
BlendOpc = X86::VBLENDVPDrrr;
break;
+ case X86::VPCMPQZ256rri:
case X86::VPMOVQ2MZ256kr:
MovMskOpc = X86::VMOVMSKPDYrr;
BlendOpc = X86::VBLENDVPDYrrr;
break;
+ case X86::VPCMPBZ128rri:
case X86::VPMOVB2MZ128kr:
MovMskOpc = X86::VPMOVMSKBrr;
BlendOpc = X86::VPBLENDVBrrr;
break;
+ case X86::VPCMPBZ256rri:
case X86::VPMOVB2MZ256kr:
MovMskOpc = X86::VPMOVMSKBYrr;
BlendOpc = X86::VPBLENDVBYrrr;
@@ -343,11 +419,12 @@ static bool tryCompressVPMOVPattern(MachineInstr &MI, MachineBasicBlock &MBB,
UseOpc == X86::KMOVDrk;
// Only allow non-narrowing KMOV uses of the mask.
if (IsKMOV && CurMI.getOperand(1).getReg() == MaskReg &&
- !isKMovNarrowing(Opc, UseOpc)) {
+ !usesExtendedRegister(CurMI) &&
+ !isKMovNarrowing(getMovMskBits(Opc), UseOpc)) {
KMovMI = &CurMI;
// continue scanning to ensure
// there are no *other* uses of the mask later in the block.
- } else if (isCompressibleBlendVUse(BlendOpc, UseOpc) &&
+ } else if (!IsSignMaskCmp && isCompressibleBlendVUse(BlendOpc, UseOpc) &&
CurMI.getOperand(2).getReg() == MaskReg &&
!usesExtendedRegister(CurMI) &&
checkPredicate(BlendOpc, &ST)) {
@@ -371,6 +448,19 @@ static bool tryCompressVPMOVPattern(MachineInstr &MI, MachineBasicBlock &MBB,
if (!KMovMI && !BlendMI)
return false;
+ unsigned MovMskBits = getMovMskBits(Opc);
+ // Bounded complements define EFLAGS, unlike VPCMP + KMOV. A 32-bit
+ // complement uses NOT, which does not modify EFLAGS.
+ if (IsSignMaskCmp && KMovMI) {
+ if (KMovMI->getOperand(0).isDead() ||
+ (MovMskBits != 32 &&
+ MBB.computeRegisterLiveness(
+ TRI, X86::EFLAGS,
+ std::next(MachineBasicBlock::const_iterator(*KMovMI)),
+ MBB.size()) != MachineBasicBlock::LQR_Dead))
+ return false;
+ }
+
// Check if MaskReg is used in any other basic blocks
for (const MachineInstr &UseMI : MRI->use_instructions(MaskReg))
if (UseMI.getParent() != &MBB)
@@ -379,6 +469,7 @@ static bool tryCompressVPMOVPattern(MachineInstr &MI, MachineBasicBlock &MBB,
// Apply the transformation
MachineInstr *NewMI = nullptr;
if (KMovMI) {
+ MachineOperand OldDst = KMovMI->getOperand(0);
KMovMI->setDesc(TII->get(MovMskOpc));
MachineOperand &NewSrc = KMovMI->getOperand(1);
NewSrc.setReg(SrcVecReg);
@@ -386,6 +477,23 @@ static bool tryCompressVPMOVPattern(MachineInstr &MI, MachineBasicBlock &MBB,
// state from the VPMOV instead.
NewSrc.setIsKill(MI.getOperand(1).isKill());
NewMI = KMovMI;
+ if (IsSignMaskCmp) {
+ Register DstReg = OldDst.getReg();
+ int64_t ComplementMask =
+ APInt::getLowBitsSet(32, MovMskBits).getSExtValue();
+ unsigned ComplementOpc =
+ MovMskBits == 32
+ ? X86::NOT32r
+ : (isInt<8>(ComplementMask) ? X86::XOR32ri8 : X86::XOR32ri);
+ auto MIB = BuildMI(MBB, std::next(MachineBasicBlock::iterator(*KMovMI)),
+ KMovMI->getDebugLoc(), TII->get(ComplementOpc), DstReg)
+ .addReg(DstReg, RegState::Kill);
+ if (MovMskBits != 32) {
+ MIB.addImm(ComplementMask);
+ MIB->findRegisterDefOperand(X86::EFLAGS, TRI)->setIsDead();
+ }
+ MIB->getOperand(0).setIsRenamable(OldDst.isRenamable());
+ }
} else if (BlendMI) {
const MachineOperand &MaskVec = MI.getOperand(1);
const MachineOperand &Dst = BlendMI->getOperand(0);
@@ -407,6 +515,8 @@ static bool tryCompressVPMOVPattern(MachineInstr &MI, MachineBasicBlock &MBB,
assert(NewMI && "Expected a compressed instruction");
NewMI->setAsmPrinterFlag(X86::AC_EVEX_2_VEX);
ToErase.push_back(&MI);
+ if (ConstantDefOnlyFeedsCmp && MI.getOperand(2).isKill())
+ ToErase.push_back(ConstantDef);
return true;
}
@@ -423,8 +533,8 @@ static bool CompressEVEXImpl(MachineInstr &MI, MachineBasicBlock &MBB,
if (TSFlags & (X86II::EVEX_K | X86II::EVEX_L2))
return false;
- // Specialized VPMOVD2M + KMOV -> MOVMSK fold first.
- if (tryCompressVPMOVPattern(MI, MBB, ST, ToErase))
+ // Specialized mask-producing instruction + KMOV -> MOVMSK folds first.
+ if (tryCompressMovMskPattern(MI, MBB, ST, ToErase))
return true;
auto IsRedundantNewDataDest = [&](unsigned &Opc) {
diff --git a/llvm/test/CodeGen/X86/bitcast-sign-mask.ll b/llvm/test/CodeGen/X86/bitcast-sign-mask.ll
new file mode 100644
index 0000000000000..67a85669885ea
--- /dev/null
+++ b/llvm/test/CodeGen/X86/bitcast-sign-mask.ll
@@ -0,0 +1,229 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 | FileCheck %s --check-prefix=SSE
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64-v3 | FileCheck %s --check-prefix=AVX
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64-v4 | FileCheck %s --check-prefix=AVX512
+
+; Prefer MOVMSK when bitcasting complementary signed integer sign tests.
+
+define i16 @cmp_sge_v16i8(<16 x i8> %x) {
+; SSE-LABEL: cmp_sge_v16i8:
+; SSE: # %bb.0:
+; SSE-NEXT: pmovmskb %xmm0, %eax
+; SSE-NEXT: notl %eax
+; SSE-NEXT: # kill: def $ax killed $ax killed $eax
+; SSE-NEXT: retq
+;
+; AVX-LABEL: cmp_sge_v16i8:
+; AVX: # %bb.0:
+; AVX-NEXT: vpmovmskb %xmm0, %eax
+; AVX-NEXT: notl %eax
+; AVX-NEXT: # kill: def $ax killed $ax killed $eax
+; AVX-NEXT: retq
+;
+; AVX512-LABEL: cmp_sge_v16i8:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpmovmskb %xmm0, %eax
+; AVX512-NEXT: xorl $65535, %eax # imm = 0xFFFF
+; AVX512-NEXT: # kill: def $ax killed $ax killed $eax
+; AVX512-NEXT: retq
+ %c = icmp sge <16 x i8> %x, zeroinitializer
+ %r = bitcast <16 x i1> %c to i16
+ ret i16 %r
+}
+
+define i16 @cmp_sgt_neg1_v16i8(<16 x i8> %x) {
+; SSE-LABEL: cmp_sgt_neg1_v16i8:
+; SSE: # %bb.0:
+; SSE-NEXT: pmovmskb %xmm0, %eax
+; SSE-NEXT: notl %eax
+; SSE-NEXT: # kill: def $ax killed $ax killed $eax
+; SSE-NEXT: retq
+;
+; AVX-LABEL: cmp_sgt_neg1_v16i8:
+; AVX: # %bb.0:
+; AVX-NEXT: vpmovmskb %xmm0, %eax
+; AVX-NEXT: notl %eax
+; AVX-NEXT: # kill: def $ax killed $ax killed $eax
+; AVX-NEXT: retq
+;
+; AVX512-LABEL: cmp_sgt_neg1_v16i8:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpmovmskb %xmm0, %eax
+; AVX512-NEXT: xorl $65535, %eax # imm = 0xFFFF
+; AVX512-NEXT: # kill: def $ax killed $ax killed $eax
+; AVX512-NEXT: retq
+ %c = icmp sgt <16 x i8> %x, splat (i8 -1)
+ %r = bitcast <16 x i1> %c to i16
+ ret i16 %r
+}
+
+define i32 @cmp_sge_v32i8(<32 x i8> %x) {
+; SSE-LABEL: cmp_sge_v32i8:
+; SSE: # %bb.0:
+; SSE-NEXT: pmovmskb %xmm0, %ecx
+; SSE-NEXT: xorl $65535, %ecx # imm = 0xFFFF
+; SSE-NEXT: pmovmskb %xmm1, %eax
+; SSE-NEXT: notl %eax
+; SSE-NEXT: shll $16, %eax
+; SSE-NEXT: orl %ecx, %eax
+; SSE-NEXT: retq
+;
+; AVX-LABEL: cmp_sge_v32i8:
+; AVX: # %bb.0:
+; AVX-NEXT: vpmovmskb %ymm0, %eax
+; AVX-NEXT: notl %eax
+; AVX-NEXT: vzeroupper
+; AVX-NEXT: retq
+;
+; AVX512-LABEL: cmp_sge_v32i8:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpmovmskb %ymm0, %eax
+; AVX512-NEXT: notl %eax
+; AVX512-NEXT: vzeroupper
+; AVX512-NEXT: retq
+ %c = icmp sge <32 x i8> %x, zeroinitializer
+ %r = bitcast <32 x i1> %c to i32
+ ret i32 %r
+}
+
+define i32 @cmp_sgt_neg1_v32i8(<32 x i8> %x) {
+; SSE-LABEL: cmp_sgt_neg1_v32i8:
+; SSE: # %bb.0:
+; SSE-NEXT: pmovmskb %xmm0, %ecx
+; SSE-NEXT: xorl $65535, %ecx # imm = 0xFFFF
+; SSE-NEXT: pmovmskb %xmm1, %eax
+; SSE-NEXT: notl %eax
+; SSE-NEXT: shll $16, %eax
+; SSE-NEXT: orl %ecx, %eax
+; SSE-NEXT: retq
+;
+; AVX-LABEL: cmp_sgt_neg1_v32i8:
+; AVX: # %bb.0:
+; AVX-NEXT: vpmovmskb %ymm0, %eax
+; AVX-NEXT: notl %eax
+; AVX-NEXT: vzeroupper
+; AVX-NEXT: retq
+;
+; AVX512-LABEL: cmp_sgt_neg1_v32i8:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpmovmskb %ymm0, %eax
+; AVX512-NEXT: notl %eax
+; AVX512-NEXT: vzeroupper
+; AVX512-NEXT: retq
+ %c = icmp sgt <32 x i8> %x, splat (i8 -1)
+ %r = bitcast <32 x i1> %c to i32
+ ret i32 %r
+}
+
+define i4 @cmp_sge_v4i32(<4 x i32> %x) {
+; SSE-LABEL: cmp_sge_v4i32:
+; SSE: # %bb.0:
+; SSE-NEXT: movmskps %xmm0, %eax
+; SSE-NEXT: xorb $15, %al
+; SSE-NEXT: # kill: def $al killed $al killed $eax
+; SSE-NEXT: retq
+;
+; AVX-LABEL: cmp_sge_v4i32:
+; AVX: # %bb.0:
+; AVX-NEXT: vmovmskps %xmm0, %eax
+; AVX-NEXT: xorb $15, %al
+; AVX-NEXT: # kill: def $al killed $al killed $eax
+; AVX-NEXT: retq
+;
+; AVX512-LABEL: cmp_sge_v4i32:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vmovmskps %xmm0, %eax
+; AVX512-NEXT: xorl $15, %eax
+; AVX512-NEXT: # kill: def $al killed $al killed $eax
+; AVX512-NEXT: retq
+ %c = icmp sge <4 x i32> %x, zeroinitializer
+ %r = bitcast <4 x i1> %c to i4
+ ret i4 %r
+}
+
+define i8 @cmp_sge_v8i32(<8 x i32> %x) {
+; SSE-LABEL: cmp_sge_v8i32:
+; SSE: # %bb.0:
+; SSE-NEXT: pcmpeqd %xmm2, %xmm2
+; SSE-NEXT: pxor %xmm2, %xmm1
+; SSE-NEXT: pxor %xmm2, %xmm0
+; SSE-NEXT: packssdw %xmm1, %xmm0
+; SSE-NEXT: packsswb %xmm0, %xmm0
+; SSE-NEXT: pmovmskb %xmm0, %eax
+; SSE-NEXT: # kill: def $al killed $al killed $eax
+; SSE-NEXT: retq
+;
+; AVX-LABEL: cmp_sge_v8i32:
+; AVX: # %bb.0:
+; AVX-NEXT: vmovmskps %ymm0, %eax
+; AVX-NEXT: notb %al
+; AVX-NEXT: # kill: def $al killed $al killed $eax
+; AVX-NEXT: vzeroupper
+; AVX-NEXT: retq
+;
+; AVX512-LABEL: cmp_sge_v8i32:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vmovmskps %ymm0, %eax
+; AVX512-NEXT: xorl $255, %eax
+; AVX512-NEXT: # kill: def $al killed $al killed $eax
+; AVX512-NEXT: vzeroupper
+; AVX512-NEXT: retq
+ %c = icmp sge <8 x i32> %x, zeroinitializer
+ %r = bitcast <8 x i1> %c to i8
+ ret i8 %r
+}
+
+define i2 @cmp_sge_v2i64(<2 x i64> %x) {
+; SSE-LABEL: cmp_sge_v2i64:
+; SSE: # %bb.0:
+; SSE-NEXT: movmskpd %xmm0, %eax
+; SSE-NEXT: xorb $3, %al
+; SSE-NEXT: # kill: def $al killed $al killed $eax
+; SSE-NEXT: retq
+;
+; AVX-LABEL: cmp_sge_v2i64:
+; AVX: # %bb.0:
+; AVX-NEXT: vmovmskpd %xmm0, %eax
+; AVX-NEXT: xorb $3, %al
+; AVX-NEXT: # kill: def $al killed $al killed $eax
+; AVX-NEXT: retq
+;
+; AVX512-LABEL: cmp_sge_v2i64:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vmovmskpd %xmm0, %eax
+; AVX512-NEXT: xorl $3, %eax
+; AVX512-NEXT: # kill: def $al killed $al killed $eax
+; AVX512-NEXT: retq
+ %c = icmp sge <2 x i64> %x, zeroinitializer
+ %r = bitcast <2 x i1> %c to i2
+ ret i2 %r
+}
+
+define i4 @cmp_sge_v4i64(<4 x i64> %x) {
+; SSE-LABEL: cmp_sge_v4i64:
+; SSE: # %bb.0:
+; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[1,3],xmm1[1,3]
+; SSE-NEXT: movmskps %xmm0, %eax
+; SSE-NEXT: xorl $15, %eax
+; SSE-NEXT: # kill: def $al killed $al killed $eax
+; SSE-NEXT: retq
+;
+; AVX-LABEL: cmp_sge_v4i64:
+; AVX: # %bb.0:
+; AVX-NEXT: vmovmskpd %ymm0, %eax
+; AVX-NEXT: xorb $15, %al
+; AVX-NEXT: # kill: def $al killed $al killed $eax
+; AVX-NEXT: vzeroupper
+; AVX-NEXT: retq
+;
+; AVX512-LABEL: cmp_sge_v4i64:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vmovmskpd %ymm0, %eax
+; AVX512-NEXT: xorl $15, %eax
+; AVX512-NEXT: # kill: def $al killed $al killed $eax
+; AVX512-NEXT: vzeroupper
+; AVX512-NEXT: retq
+ %c = icmp sge <4 x i64> %x, zeroinitializer
+ %r = bitcast <4 x i1> %c to i4
+ ret i4 %r
+}
diff --git a/llvm/test/CodeGen/X86/compress-evex-sign-mask.mir b/llvm/test/CodeGen/X86/compress-evex-sign-mask.mir
new file mode 100644
index 0000000000000..60c7e65fe9eb0
--- /dev/null
+++ b/llvm/test/CodeGen/X86/compress-evex-sign-mask.mir
@@ -0,0 +1,240 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -mtriple=x86_64-- -run-pass=x86-compress-evex \
+# RUN: -verify-machineinstrs -mcpu=skx -o - %s | FileCheck %s
+
+---
+name: sign_mask_positive
+body: |
+ bb.0:
+ liveins: $xmm0, $xmm2, $xmm4, $ymm6, $ymm8, $ymm10, $xmm12
+
+ ; CHECK-LABEL: name: sign_mask_positive
+ ; CHECK: liveins: $xmm0, $xmm2, $xmm4, $ymm6, $ymm8, $ymm10, $xmm12
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: $eax = VPMOVMSKBrr killed $xmm0
+ ; CHECK-NEXT: $eax = XOR32ri killed $eax, 65535, implicit-def dead $eflags
+ ; CHECK-NEXT: $ecx = VPMOVMSKBYrr killed $ymm2
+ ; CHECK-NEXT: $ecx = NOT32r killed $ecx
+ ; CHECK-NEXT: $edx = VMOVMSKPSrr killed $xmm4
+ ; CHECK-NEXT: $edx = XOR32ri8 killed $edx, 15, implicit-def dead $eflags
+ ; CHECK-NEXT: $esi = VMOVMSKPSYrr killed $ymm6
+ ; CHECK-NEXT: $esi = XOR32ri killed $esi, 255, implicit-def dead $eflags
+ ; CHECK-NEXT: $edi = VMOVMSKPDrr killed $xmm8
+ ; CHECK-NEXT: $edi = XOR32ri8 killed $edi, 3, implicit-def dead $eflags
+ ; CHECK-NEXT: $r8d = VMOVMSKPDYrr killed $ymm10
+ ; CHECK-NEXT: $r8d = XOR32ri8 killed $r8d, 15, implicit-def dead $eflags
+ ; CHECK-NEXT: $r9d = VPMOVMSKBrr killed $xmm12
+ ; CHECK-NEXT: $r9d = XOR32ri killed $r9d, 65535, implicit-def dead $eflags
+ ; CHECK-NEXT: RET64
+ $xmm1 = VPXORrr undef $xmm1, undef $xmm1
+ $k0 = VPCMPBZ128rri killed $xmm0, killed $xmm1, 5
+ $eax = KMOVDrk killed $k0
+
+ $xmm3 = VPXORrr undef $xmm3, undef $xmm3, implicit-def $ymm3
+ $k1 = VPCMPBZ256rri killed $ymm2, killed $ymm3, 5
+ $ecx = KMOVDrk killed $k1
+
+ $xmm5 = VPXORrr undef $xmm5, undef $xmm5
+ $k2 = VPCMPDZ128rri killed $xmm4, killed $xmm5, 5
+ $edx = KMOVBrk killed $k2
+
+ $xmm7 = VPXORrr undef $xmm7, undef $xmm7, implicit-def $ymm7
+ $k3 = VPCMPDZ256rri killed $ymm6, killed $ymm7, 5
+ $esi = KMOVBrk killed $k3
+
+ $xmm9 = VPXORrr undef $xmm9, undef $xmm9
+ $k4 = VPCMPQZ128rri killed $xmm8, killed $xmm9, 5
+ $edi = KMOVBrk killed $k4
+
+ $xmm11 = VPXORrr undef $xmm11, undef $xmm11, implicit-def $ymm11
+ $k5 = VPCMPQZ256rri killed $ymm10, killed $ymm11, 5
+ $r8d = KMOVBrk killed $k5
+
+ ; Also recognize the existing SETGT(X, -1) canonical form.
+ $xmm13 = VPCMPEQDrr undef $xmm13, undef $xmm13
+ $k6 = VPCMPBZ128rri killed $xmm12, killed $xmm13, 6
+ $r9d = KMOVDrk killed $k6
+ RET64
+...
+
+---
+name: sign_mask_256_allones_positive
+body: |
+ bb.0:
+ liveins: $ymm0
+
+ ; A full-width all-ones producer makes predicate 6 equivalent to X >= 0.
+ ; CHECK-LABEL: name: sign_mask_256_allones_positive
+ ; CHECK: liveins: $ymm0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: $eax = VMOVMSKPSYrr killed $ymm0
+ ; CHECK-NEXT: $eax = XOR32ri killed $eax, 255, implicit-def dead $eflags
+ ; CHECK-NEXT: RET64
+ $ymm1 = VPCMPEQDYrr undef $ymm1, undef $ymm1
+ $k0 = VPCMPDZ256rri killed $ymm0, killed $ymm1, 6
+ $eax = KMOVBrk killed $k0
+ RET64
+...
+
+---
+name: sign_mask_negative
+body: |
+ bb.0:
+ liveins: $xmm0, $xmm2, $xmm4, $ymm6, $zmm8, $xmm16, $xmm12, $r10d, $r11d, $eflags
+
+ ; The mask has another use.
+ ; CHECK-LABEL: name: sign_mask_negative
+ ; CHECK: liveins: $xmm0, $xmm2, $xmm4, $ymm6, $zmm8, $xmm16, $xmm12, $r10d, $r11d, $eflags
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: $xmm1 = VPXORrr undef $xmm1, undef $xmm1
+ ; CHECK-NEXT: $k0 = VPCMPBZ128rri killed $xmm0, killed $xmm1, 5
+ ; CHECK-NEXT: $eax = KMOVDrk $k0
+ ; CHECK-NEXT: $ecx = KMOVDrk killed $k0
+ ; CHECK-NEXT: $xmm3 = VPXORrr undef $xmm3, undef $xmm3
+ ; CHECK-NEXT: $k1 = VPCMPDZ128rri $xmm2, killed $xmm3, 5
+ ; CHECK-NEXT: $xmm2 = VMOVAPSrr $xmm4
+ ; CHECK-NEXT: $edx = KMOVBrk killed $k1
+ ; CHECK-NEXT: $xmm5 = VPXORrr undef $xmm5, undef $xmm5
+ ; CHECK-NEXT: $k2 = VPCMPQZ128rri killed $xmm4, killed $xmm5, 4
+ ; CHECK-NEXT: $esi = KMOVBrk killed $k2
+ ; CHECK-NEXT: $xmm7 = VPXORrr undef $xmm7, undef $xmm7, implicit-def $ymm7
+ ; CHECK-NEXT: $k3 = VPCMPBZ256rri killed $ymm6, killed $ymm7, 5
+ ; CHECK-NEXT: $edi = KMOVWrk killed $k3
+ ; CHECK-NEXT: $zmm9 = VPXORDZrr undef $zmm9, undef $zmm9
+ ; CHECK-NEXT: $k4 = VPCMPDZrri killed $zmm8, killed $zmm9, 5
+ ; CHECK-NEXT: $r8d = KMOVWrk killed $k4
+ ; CHECK-NEXT: $xmm10 = VPXORrr undef $xmm10, undef $xmm10
+ ; CHECK-NEXT: $k5 = VPCMPBZ128rri killed $xmm16, killed $xmm10, 5
+ ; CHECK-NEXT: $r9d = KMOVDrk killed $k5
+ ; CHECK-NEXT: $xmm13 = VPXORrr undef $xmm13, undef $xmm13
+ ; CHECK-NEXT: $k6 = VPCMPDZ128rri killed $xmm12, killed $xmm13, 5
+ ; CHECK-NEXT: $r10d = KMOVBrk killed $k6
+ ; CHECK-NEXT: $r10d = ADC32rr $r10d, $r11d, implicit-def dead $eflags, implicit killed $eflags
+ ; CHECK-NEXT: RET64
+ $xmm1 = VPXORrr undef $xmm1, undef $xmm1
+ $k0 = VPCMPBZ128rri killed $xmm0, killed $xmm1, 5
+ $eax = KMOVDrk $k0
+ $ecx = KMOVDrk killed $k0
+
+ ; The vector source is clobbered before the KMOV.
+ $xmm3 = VPXORrr undef $xmm3, undef $xmm3
+ $k1 = VPCMPDZ128rri $xmm2, killed $xmm3, 5
+ $xmm2 = VMOVAPSrr $xmm4
+ $edx = KMOVBrk killed $k1
+
+ ; Predicate 4 is not a sign-bit-clear test.
+ $xmm5 = VPXORrr undef $xmm5, undef $xmm5
+ $k2 = VPCMPQZ128rri killed $xmm4, killed $xmm5, 4
+ $esi = KMOVBrk killed $k2
+
+ ; A narrowing KMOV cannot preserve every byte lane.
+ $xmm7 = VPXORrr undef $xmm7, undef $xmm7, implicit-def $ymm7
+ $k3 = VPCMPBZ256rri killed $ymm6, killed $ymm7, 5
+ $edi = KMOVWrk killed $k3
+
+ ; There is no 512-bit MOVMSK replacement.
+ $zmm9 = VPXORDZrr undef $zmm9, undef $zmm9
+ $k4 = VPCMPDZrri killed $zmm8, killed $zmm9, 5
+ $r8d = KMOVWrk killed $k4
+
+ ; VEX MOVMSK cannot encode an extended vector register.
+ $xmm10 = VPXORrr undef $xmm10, undef $xmm10
+ $k5 = VPCMPBZ128rri killed $xmm16, killed $xmm10, 5
+ $r9d = KMOVDrk killed $k5
+
+ ; The bounded XOR would clobber live EFLAGS.
+ $xmm13 = VPXORrr undef $xmm13, undef $xmm13
+ $k6 = VPCMPDZ128rri killed $xmm12, killed $xmm13, 5
+ $r10d = KMOVBrk killed $k6
+ $r10d = ADC32rr $r10d, $r11d, implicit-def dead $eflags, implicit killed $eflags
+ RET64
+...
+
+---
+name: sign_mask_blend_not_compressed
+body: |
+ bb.0:
+ liveins: $xmm0, $xmm2, $xmm3
+
+ ; The compare mask has the opposite polarity from the source sign bits, so
+ ; it cannot be replaced by a VBLENDV using the source as its mask.
+ ; CHECK-LABEL: name: sign_mask_blend_not_compressed
+ ; CHECK: liveins: $xmm0, $xmm2, $xmm3
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: $xmm1 = VPXORrr undef $xmm1, undef $xmm1
+ ; CHECK-NEXT: $k1 = VPCMPDZ128rri killed $xmm0, killed $xmm1, 5
+ ; CHECK-NEXT: $xmm2 = VMOVAPSZ128rrk $xmm2, killed $k1, $xmm3
+ ; CHECK-NEXT: RET64 implicit killed $xmm2
+ $xmm1 = VPXORrr undef $xmm1, undef $xmm1
+ $k1 = VPCMPDZ128rri killed $xmm0, killed $xmm1, 5
+ $xmm2 = VMOVAPSZ128rrk $xmm2, killed $k1, $xmm3
+ RET64 implicit killed $xmm2
+...
+
+---
+name: sign_mask_wrong_constants
+body: |
+ bb.0:
+ liveins: $xmm0, $xmm2
+
+ ; Predicate 5 requires zero, not all ones.
+ ; CHECK-LABEL: name: sign_mask_wrong_constants
+ ; CHECK: liveins: $xmm0, $xmm2
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: $xmm1 = VPCMPEQDrr undef $xmm1, undef $xmm1
+ ; CHECK-NEXT: $k0 = VPCMPBZ128rri killed $xmm0, killed $xmm1, 5
+ ; CHECK-NEXT: $eax = KMOVDrk killed $k0
+ ; CHECK-NEXT: $xmm3 = VPXORrr undef $xmm3, undef $xmm3
+ ; CHECK-NEXT: $k1 = VPCMPQZ128rri killed $xmm2, killed $xmm3, 6
+ ; CHECK-NEXT: $ecx = KMOVBrk killed $k1
+ ; CHECK-NEXT: RET64
+ $xmm1 = VPCMPEQDrr undef $xmm1, undef $xmm1
+ $k0 = VPCMPBZ128rri killed $xmm0, killed $xmm1, 5
+ $eax = KMOVDrk killed $k0
+
+ ; Predicate 6 requires all ones, not zero.
+ $xmm3 = VPXORrr undef $xmm3, undef $xmm3
+ $k1 = VPCMPQZ128rri killed $xmm2, killed $xmm3, 6
+ $ecx = KMOVBrk killed $k1
+ RET64
+...
+
+---
+name: sign_mask_256_allones_width
+body: |
+ bb.0:
+ liveins: $ymm0
+
+ ; A 128-bit all-ones idiom zeroes, rather than sets, the upper YMM lanes.
+ ; CHECK-LABEL: name: sign_mask_256_allones_width
+ ; CHECK: liveins: $ymm0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: $xmm1 = VPCMPEQDrr undef $xmm1, undef $xmm1, implicit-def $ymm1
+ ; CHECK-NEXT: $k0 = VPCMPDZ256rri killed $ymm0, killed $ymm1, 6
+ ; CHECK-NEXT: $eax = KMOVBrk killed $k0
+ ; CHECK-NEXT: RET64
+ $xmm1 = VPCMPEQDrr undef $xmm1, undef $xmm1, implicit-def $ymm1
+ $k0 = VPCMPDZ256rri killed $ymm0, killed $ymm1, 6
+ $eax = KMOVBrk killed $k0
+ RET64
+...
+
+---
+name: sign_mask_dead_kmov_destination
+body: |
+ bb.0:
+ liveins: $xmm0
+
+ ; The new XOR would consume a destination that was originally dead.
+ ; CHECK-LABEL: name: sign_mask_dead_kmov_destination
+ ; CHECK: liveins: $xmm0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: $xmm1 = VPXORrr undef $xmm1, undef $xmm1
+ ; CHECK-NEXT: $k0 = VPCMPBZ128rri killed $xmm0, killed $xmm1, 5
+ ; CHECK-NEXT: dead $eax = KMOVDrk killed $k0
+ ; CHECK-NEXT: RET64
+ $xmm1 = VPXORrr undef $xmm1, undef $xmm1
+ $k0 = VPCMPBZ128rri killed $xmm0, killed $xmm1, 5
+ dead $eax = KMOVDrk killed $k0
+ RET64
+...
diff --git a/llvm/test/CodeGen/X86/compress-evex-vpmov-kill.mir b/llvm/test/CodeGen/X86/compress-evex-vpmov-kill.mir
index 052e466599a9d..c0aab50ad8bb4 100644
--- a/llvm/test/CodeGen/X86/compress-evex-vpmov-kill.mir
+++ b/llvm/test/CodeGen/X86/compress-evex-vpmov-kill.mir
@@ -1,7 +1,7 @@
# RUN: llc %s -mtriple=x86_64-unknown -mattr=+avx512vl,+avx512dq \
# RUN: -run-pass=x86-compress-evex -o - | FileCheck %s
#
-# tryCompressVPMOVPattern folds VPMOV*2M + KMOV into VMOVMSK. The kill flag on
+# tryCompressMovMskPattern folds VPMOV*2M + KMOV into VMOVMSK. The kill flag on
# the XMM source must be derived from the VPMOV. Otherwise it inherits the kill
# flag from the KMOV's mask operand, which can mark a still-live XMM register as
# killed and produce incorrect liveness.
>From f26a5cd7946abc2090138fde28ce7c73f5accd3c Mon Sep 17 00:00:00 2001
From: Oscar Priego Verdugo <oscar.priegov at gmail.com>
Date: Fri, 4 Sep 2026 13:10:21 -0600
Subject: [PATCH 2/4] [X86] Address review feedback for sign-mask MOVMSK fold
---
llvm/lib/Target/X86/X86CompressEVEX.cpp | 45 +++++++++++--------
.../CodeGen/X86/compress-evex-vpmov-kill.mir | 2 +-
2 files changed, 27 insertions(+), 20 deletions(-)
diff --git a/llvm/lib/Target/X86/X86CompressEVEX.cpp b/llvm/lib/Target/X86/X86CompressEVEX.cpp
index e34590a3ecfb2..18d655288f400 100644
--- a/llvm/lib/Target/X86/X86CompressEVEX.cpp
+++ b/llvm/lib/Target/X86/X86CompressEVEX.cpp
@@ -201,7 +201,7 @@ static unsigned getMovMskBits(unsigned Opc) {
case X86::VPCMPBZ256rri:
return 32;
default:
- llvm_unreachable("Unknown MOVMSK source opcode");
+ llvm_unreachable("Unknown opcode");
}
}
@@ -254,9 +254,11 @@ static MachineInstr *getSignMaskConstantDef(MachineInstr &MI, Register Reg,
MI.getParent()->begin(), MachineBasicBlock::iterator(MI)))) {
if (!DefMI.modifiesRegister(Reg, TRI))
continue;
- return (IsZero ? isZeroVector(DefMI) : isAllOnesVector(DefMI, Is256Bit))
- ? &DefMI
- : nullptr;
+ // Stop at the nearest def/clobber; an older matching constant may no
+ // longer be the reaching definition.
+ if (IsZero ? isZeroVector(DefMI) : isAllOnesVector(DefMI, Is256Bit))
+ return &DefMI;
+ break;
}
return nullptr;
}
@@ -312,19 +314,20 @@ static bool isCompressibleBlendVUse(unsigned BlendOpc, unsigned UseOpc) {
}
}
-// Try to compress VPMOV*2M and complementary signed sign-test patterns:
-// vpmov*2m %xmm0, %k0 -> (erase this)
-// kmov* %k0, %eax -> vmovmskp* %xmm0, %eax
-// vpcmpge* $0, %xmm0, %k0 -> (erase this) (X >= 0)
-// vpcmpgt* $-1, %xmm0, %k0 -> (erase this) (X > -1)
-// kmov* %k0, %eax -> vmovmskp* %xmm0, %eax
-// bounded complement of %eax
-// and:
-// vpmov*2m %xmm0, %k1 -> (erase this)
-// vmov* %xmm1, %xmm2 {%k1} -> vblendv* %xmm0, %xmm2, %xmm1, %xmm2
-static bool tryCompressMovMskPattern(MachineInstr &MI, MachineBasicBlock &MBB,
- const X86Subtarget &ST,
- SmallVectorImpl<MachineInstr *> &ToErase) {
+// Try to compress mask producer chains:
+// vpmov*2m %xmm0, %k0 -> (erase this)
+// kmov* %k0, %eax -> vmovmskp* %xmm0, %eax
+//
+// vpcmpge* $0, %xmm0, %k0 -> (erase this) (X >= 0)
+// vpcmpgt* $-1, %xmm0, %k0 -> (erase this) (X > -1)
+// kmov* %k0, %eax -> vmovmskp* %xmm0, %eax
+// bounded complement of %eax
+//
+// vpmov*2m %xmm0, %k1 -> (erase this)
+// vmov* %xmm1, %xmm2 {%k1} -> vblendv* %xmm0, %xmm2, %xmm1, %xmm2
+static bool tryCompressMaskProducer(MachineInstr &MI, MachineBasicBlock &MBB,
+ const X86Subtarget &ST,
+ SmallVectorImpl<MachineInstr *> &ToErase) {
const X86InstrInfo *TII = ST.getInstrInfo();
const TargetRegisterInfo *TRI = ST.getRegisterInfo();
MachineRegisterInfo *MRI = &MBB.getParent()->getRegInfo();
@@ -349,15 +352,19 @@ static bool tryCompressMovMskPattern(MachineInstr &MI, MachineBasicBlock &MBB,
if (IsSignMaskCmp) {
int64_t Pred = MI.getOperand(3).getImm();
+ // VPCMP signed predicates: nlt (5) folds X >= 0, nle (6) folds X > -1.
if (Pred != 5 && Pred != 6)
return false;
Register ConstantReg = MI.getOperand(2).getReg();
bool Is256Bit = Opc == X86::VPCMPBZ256rri || Opc == X86::VPCMPDZ256rri ||
Opc == X86::VPCMPQZ256rri;
+ // The sign-mask fold is valid only for compares against the reaching
+ // zero/all-ones vector definition.
ConstantDef =
getSignMaskConstantDef(MI, ConstantReg, Pred == 5, Is256Bit, TRI);
if (!ConstantDef)
return false;
+ // If the constant feeds only this compare, erase it with the compare.
ConstantDefOnlyFeedsCmp = !TRI->regsOverlap(ConstantReg, SrcVecReg);
for (MachineInstr &UseMI :
llvm::make_range(std::next(MachineBasicBlock::iterator(*ConstantDef)),
@@ -533,8 +540,8 @@ static bool CompressEVEXImpl(MachineInstr &MI, MachineBasicBlock &MBB,
if (TSFlags & (X86II::EVEX_K | X86II::EVEX_L2))
return false;
- // Specialized mask-producing instruction + KMOV -> MOVMSK folds first.
- if (tryCompressMovMskPattern(MI, MBB, ST, ToErase))
+ // Specialized mask-producing folds to MOVMSK/VBLENDV first.
+ if (tryCompressMaskProducer(MI, MBB, ST, ToErase))
return true;
auto IsRedundantNewDataDest = [&](unsigned &Opc) {
diff --git a/llvm/test/CodeGen/X86/compress-evex-vpmov-kill.mir b/llvm/test/CodeGen/X86/compress-evex-vpmov-kill.mir
index c0aab50ad8bb4..bb6cb7e9c42d4 100644
--- a/llvm/test/CodeGen/X86/compress-evex-vpmov-kill.mir
+++ b/llvm/test/CodeGen/X86/compress-evex-vpmov-kill.mir
@@ -1,7 +1,7 @@
# RUN: llc %s -mtriple=x86_64-unknown -mattr=+avx512vl,+avx512dq \
# RUN: -run-pass=x86-compress-evex -o - | FileCheck %s
#
-# tryCompressMovMskPattern folds VPMOV*2M + KMOV into VMOVMSK. The kill flag on
+# tryCompressMaskProducer folds VPMOV*2M + KMOV into VMOVMSK. The kill flag on
# the XMM source must be derived from the VPMOV. Otherwise it inherits the kill
# flag from the KMOV's mask operand, which can mark a still-live XMM register as
# killed and produce incorrect liveness.
>From 832fec3126a2b53b0d396180d1653d9b353b6e81 Mon Sep 17 00:00:00 2001
From: Oscar Priego Verdugo <oscar.priegov at gmail.com>
Date: Fri, 4 Sep 2026 16:31:58 -0600
Subject: [PATCH 3/4] [X86] Update sign-mask checks after AVX-512 compare
changes
---
llvm/test/CodeGen/X86/avx512-icmp-sle-zero.ll | 10 ++++------
1 file changed, 4 insertions(+), 6 deletions(-)
diff --git a/llvm/test/CodeGen/X86/avx512-icmp-sle-zero.ll b/llvm/test/CodeGen/X86/avx512-icmp-sle-zero.ll
index ef8da7d206ac0..5b71fbfcb4847 100644
--- a/llvm/test/CodeGen/X86/avx512-icmp-sle-zero.ll
+++ b/llvm/test/CodeGen/X86/avx512-icmp-sle-zero.ll
@@ -81,9 +81,8 @@ define i16 @cmp_sge_v16i8(<16 x i8> %x) {
;
; AVX512-LABEL: cmp_sge_v16i8:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512-NEXT: vpcmpnltb %xmm1, %xmm0, %k0
-; AVX512-NEXT: kmovd %k0, %eax
+; AVX512-NEXT: vpmovmskb %xmm0, %eax
+; AVX512-NEXT: xorl $65535, %eax # imm = 0xFFFF
; AVX512-NEXT: # kill: def $ax killed $ax killed $eax
; AVX512-NEXT: retq
%c = icmp sge <16 x i8> %x, splat (i8 0)
@@ -101,9 +100,8 @@ define i16 @cmp_sgt_neg1_v16i8(<16 x i8> %x) {
;
; AVX512-LABEL: cmp_sgt_neg1_v16i8:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512-NEXT: vpcmpnltb %xmm1, %xmm0, %k0
-; AVX512-NEXT: kmovd %k0, %eax
+; AVX512-NEXT: vpmovmskb %xmm0, %eax
+; AVX512-NEXT: xorl $65535, %eax # imm = 0xFFFF
; AVX512-NEXT: # kill: def $ax killed $ax killed $eax
; AVX512-NEXT: retq
%c = icmp sgt <16 x i8> %x, splat (i8 -1)
>From 2598788e0f9bb6dc783fdd0b9b4d167063b96f2d Mon Sep 17 00:00:00 2001
From: Oscar Priego Verdugo <oscar.priegov at gmail.com>
Date: Mon, 7 Sep 2026 11:54:52 -0600
Subject: [PATCH 4/4] [X86] Share AVX checks in bitcast sign-mask tests
Signed-off-by: Oscar Priego Verdugo <oscar.priegov at gmail.com>
---
llvm/test/CodeGen/X86/bitcast-sign-mask.ll | 94 +++++++++-------------
1 file changed, 40 insertions(+), 54 deletions(-)
diff --git a/llvm/test/CodeGen/X86/bitcast-sign-mask.ll b/llvm/test/CodeGen/X86/bitcast-sign-mask.ll
index 67a85669885ea..e4a06df9a0a1a 100644
--- a/llvm/test/CodeGen/X86/bitcast-sign-mask.ll
+++ b/llvm/test/CodeGen/X86/bitcast-sign-mask.ll
@@ -1,7 +1,7 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 | FileCheck %s --check-prefix=SSE
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64-v3 | FileCheck %s --check-prefix=AVX
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64-v4 | FileCheck %s --check-prefix=AVX512
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64-v3 | FileCheck %s --check-prefixes=AVX,AVX2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64-v4 | FileCheck %s --check-prefixes=AVX,AVX512
; Prefer MOVMSK when bitcasting complementary signed integer sign tests.
@@ -13,12 +13,12 @@ define i16 @cmp_sge_v16i8(<16 x i8> %x) {
; SSE-NEXT: # kill: def $ax killed $ax killed $eax
; SSE-NEXT: retq
;
-; AVX-LABEL: cmp_sge_v16i8:
-; AVX: # %bb.0:
-; AVX-NEXT: vpmovmskb %xmm0, %eax
-; AVX-NEXT: notl %eax
-; AVX-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX-NEXT: retq
+; AVX2-LABEL: cmp_sge_v16i8:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vpmovmskb %xmm0, %eax
+; AVX2-NEXT: notl %eax
+; AVX2-NEXT: # kill: def $ax killed $ax killed $eax
+; AVX2-NEXT: retq
;
; AVX512-LABEL: cmp_sge_v16i8:
; AVX512: # %bb.0:
@@ -39,12 +39,12 @@ define i16 @cmp_sgt_neg1_v16i8(<16 x i8> %x) {
; SSE-NEXT: # kill: def $ax killed $ax killed $eax
; SSE-NEXT: retq
;
-; AVX-LABEL: cmp_sgt_neg1_v16i8:
-; AVX: # %bb.0:
-; AVX-NEXT: vpmovmskb %xmm0, %eax
-; AVX-NEXT: notl %eax
-; AVX-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX-NEXT: retq
+; AVX2-LABEL: cmp_sgt_neg1_v16i8:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vpmovmskb %xmm0, %eax
+; AVX2-NEXT: notl %eax
+; AVX2-NEXT: # kill: def $ax killed $ax killed $eax
+; AVX2-NEXT: retq
;
; AVX512-LABEL: cmp_sgt_neg1_v16i8:
; AVX512: # %bb.0:
@@ -74,13 +74,6 @@ define i32 @cmp_sge_v32i8(<32 x i8> %x) {
; AVX-NEXT: notl %eax
; AVX-NEXT: vzeroupper
; AVX-NEXT: retq
-;
-; AVX512-LABEL: cmp_sge_v32i8:
-; AVX512: # %bb.0:
-; AVX512-NEXT: vpmovmskb %ymm0, %eax
-; AVX512-NEXT: notl %eax
-; AVX512-NEXT: vzeroupper
-; AVX512-NEXT: retq
%c = icmp sge <32 x i8> %x, zeroinitializer
%r = bitcast <32 x i1> %c to i32
ret i32 %r
@@ -103,13 +96,6 @@ define i32 @cmp_sgt_neg1_v32i8(<32 x i8> %x) {
; AVX-NEXT: notl %eax
; AVX-NEXT: vzeroupper
; AVX-NEXT: retq
-;
-; AVX512-LABEL: cmp_sgt_neg1_v32i8:
-; AVX512: # %bb.0:
-; AVX512-NEXT: vpmovmskb %ymm0, %eax
-; AVX512-NEXT: notl %eax
-; AVX512-NEXT: vzeroupper
-; AVX512-NEXT: retq
%c = icmp sgt <32 x i8> %x, splat (i8 -1)
%r = bitcast <32 x i1> %c to i32
ret i32 %r
@@ -123,12 +109,12 @@ define i4 @cmp_sge_v4i32(<4 x i32> %x) {
; SSE-NEXT: # kill: def $al killed $al killed $eax
; SSE-NEXT: retq
;
-; AVX-LABEL: cmp_sge_v4i32:
-; AVX: # %bb.0:
-; AVX-NEXT: vmovmskps %xmm0, %eax
-; AVX-NEXT: xorb $15, %al
-; AVX-NEXT: # kill: def $al killed $al killed $eax
-; AVX-NEXT: retq
+; AVX2-LABEL: cmp_sge_v4i32:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vmovmskps %xmm0, %eax
+; AVX2-NEXT: xorb $15, %al
+; AVX2-NEXT: # kill: def $al killed $al killed $eax
+; AVX2-NEXT: retq
;
; AVX512-LABEL: cmp_sge_v4i32:
; AVX512: # %bb.0:
@@ -153,13 +139,13 @@ define i8 @cmp_sge_v8i32(<8 x i32> %x) {
; SSE-NEXT: # kill: def $al killed $al killed $eax
; SSE-NEXT: retq
;
-; AVX-LABEL: cmp_sge_v8i32:
-; AVX: # %bb.0:
-; AVX-NEXT: vmovmskps %ymm0, %eax
-; AVX-NEXT: notb %al
-; AVX-NEXT: # kill: def $al killed $al killed $eax
-; AVX-NEXT: vzeroupper
-; AVX-NEXT: retq
+; AVX2-LABEL: cmp_sge_v8i32:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vmovmskps %ymm0, %eax
+; AVX2-NEXT: notb %al
+; AVX2-NEXT: # kill: def $al killed $al killed $eax
+; AVX2-NEXT: vzeroupper
+; AVX2-NEXT: retq
;
; AVX512-LABEL: cmp_sge_v8i32:
; AVX512: # %bb.0:
@@ -181,12 +167,12 @@ define i2 @cmp_sge_v2i64(<2 x i64> %x) {
; SSE-NEXT: # kill: def $al killed $al killed $eax
; SSE-NEXT: retq
;
-; AVX-LABEL: cmp_sge_v2i64:
-; AVX: # %bb.0:
-; AVX-NEXT: vmovmskpd %xmm0, %eax
-; AVX-NEXT: xorb $3, %al
-; AVX-NEXT: # kill: def $al killed $al killed $eax
-; AVX-NEXT: retq
+; AVX2-LABEL: cmp_sge_v2i64:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vmovmskpd %xmm0, %eax
+; AVX2-NEXT: xorb $3, %al
+; AVX2-NEXT: # kill: def $al killed $al killed $eax
+; AVX2-NEXT: retq
;
; AVX512-LABEL: cmp_sge_v2i64:
; AVX512: # %bb.0:
@@ -208,13 +194,13 @@ define i4 @cmp_sge_v4i64(<4 x i64> %x) {
; SSE-NEXT: # kill: def $al killed $al killed $eax
; SSE-NEXT: retq
;
-; AVX-LABEL: cmp_sge_v4i64:
-; AVX: # %bb.0:
-; AVX-NEXT: vmovmskpd %ymm0, %eax
-; AVX-NEXT: xorb $15, %al
-; AVX-NEXT: # kill: def $al killed $al killed $eax
-; AVX-NEXT: vzeroupper
-; AVX-NEXT: retq
+; AVX2-LABEL: cmp_sge_v4i64:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vmovmskpd %ymm0, %eax
+; AVX2-NEXT: xorb $15, %al
+; AVX2-NEXT: # kill: def $al killed $al killed $eax
+; AVX2-NEXT: vzeroupper
+; AVX2-NEXT: retq
;
; AVX512-LABEL: cmp_sge_v4i64:
; AVX512: # %bb.0:
More information about the llvm-commits
mailing list