[llvm] 586cfbc - [X86] X86CompressEVEX - Fold AVX-512 sign-mask compares to MOVMSK (#216884)
via llvm-commits
llvm-commits at lists.llvm.org
Tue Sep 8 00:25:47 PDT 2026
Author: Oscar Priego
Date: 2026-09-08T07:25:41Z
New Revision: 586cfbcd9d59af33894e79192fdcad6057467465
URL: https://github.com/llvm/llvm-project/commit/586cfbcd9d59af33894e79192fdcad6057467465
DIFF: https://github.com/llvm/llvm-project/commit/586cfbcd9d59af33894e79192fdcad6057467465.diff
LOG: [X86] X86CompressEVEX - Fold AVX-512 sign-mask compares to MOVMSK (#216884)
## Summary
Fold AVX-512 complementary signed sign-test masks to the corresponding
MOVMSK instructions in `X86CompressEVEX`.
AVX-512 lowers comparisons such as:
- `X >= 0`
- `X > -1`
to `VPCMP*` producing a k-mask followed by `KMOV`.
For 128-bit and 256-bit B/D/Q forms, these masks can instead be obtained
from the source sign bits using:
- `VPCMPB` -> `VPMOVMSKB`
- `VPCMPD` -> `VMOVMSKPS`
- `VPCMPQ` -> `VMOVMSKPD`
followed by the appropriate complement.
## Details
The complement is restricted to the meaningful lane-mask bits so that
unused upper bits retain the zero-extension semantics of `KMOV`.
For the 32-lane byte case, all 32 bits are meaningful, so `NOT32r` can
be used directly without clobbering EFLAGS.
The transformation is limited to 128-bit and 256-bit B/D/Q forms with a
single non-narrowing `KMOV` consumer.
The matcher also verifies the expected zero/all-ones producer for the
comparison constant and avoids the fold when required register or
EFLAGS liveness constraints are not satisfied.
## Tests
Added:
- end-to-end CodeGen coverage for the affected sign-mask cases;
- dedicated MIR coverage for `X86CompressEVEX`;
- negative MIR cases covering:
- multiple mask uses;
- source clobbers;
- unsupported predicates/constants;
- narrowing `KMOV`;
- 512-bit vectors;
- extended registers;
- live EFLAGS;
- masked-move consumers with incompatible mask polarity;
- incorrect all-ones producer width;
- dead `KMOV` destinations.
Existing `VPMOV*2M + KMOV -> MOVMSK` coverage is preserved.
Fixes #216774
Signed-off-by: Oscar Priego Verdugo <oscar.priegov at gmail.com>
AI-assisted development tools were used during this contribution; I
personally modified, reviewed, and validated the final patch
Added:
llvm/test/CodeGen/X86/bitcast-sign-mask.ll
llvm/test/CodeGen/X86/compress-evex-sign-mask.mir
Modified:
llvm/lib/Target/X86/X86CompressEVEX.cpp
llvm/test/CodeGen/X86/avx512-icmp-sle-zero.ll
llvm/test/CodeGen/X86/compress-evex-vpmov-kill.mir
Removed:
################################################################################
diff --git a/llvm/lib/Target/X86/X86CompressEVEX.cpp b/llvm/lib/Target/X86/X86CompressEVEX.cpp
index 2c408851975e8..18d655288f400 100644
--- a/llvm/lib/Target/X86/X86CompressEVEX.cpp
+++ b/llvm/lib/Target/X86/X86CompressEVEX.cpp
@@ -181,29 +181,31 @@ static bool performCustomAdjustments(MachineInstr &MI, unsigned NewOpc) {
return true;
}
-static bool isKMovNarrowing(unsigned VPMOVOpc, unsigned KMOVOpc) {
- unsigned VPMOVBits = 0;
- switch (VPMOVOpc) {
+static unsigned getMovMskBits(unsigned Opc) {
+ switch (Opc) {
case X86::VPMOVQ2MZ128kr:
- VPMOVBits = 2;
- break;
+ case X86::VPCMPQZ128rri:
+ return 2;
case X86::VPMOVQ2MZ256kr:
case X86::VPMOVD2MZ128kr:
- VPMOVBits = 4;
- break;
+ case X86::VPCMPQZ256rri:
+ case X86::VPCMPDZ128rri:
+ return 4;
case X86::VPMOVD2MZ256kr:
- VPMOVBits = 8;
- break;
+ case X86::VPCMPDZ256rri:
+ return 8;
case X86::VPMOVB2MZ128kr:
- VPMOVBits = 16;
- break;
+ case X86::VPCMPBZ128rri:
+ return 16;
case X86::VPMOVB2MZ256kr:
- VPMOVBits = 32;
- break;
+ case X86::VPCMPBZ256rri:
+ return 32;
default:
- llvm_unreachable("Unknown VPMOV opcode");
+ llvm_unreachable("Unknown opcode");
}
+}
+static bool isKMovNarrowing(unsigned MaskBits, unsigned KMOVOpc) {
unsigned KMOVSize = 0;
switch (KMOVOpc) {
case X86::KMOVBrk:
@@ -219,7 +221,46 @@ static bool isKMovNarrowing(unsigned VPMOVOpc, unsigned KMOVOpc) {
llvm_unreachable("Unknown KMOV opcode");
}
- return KMOVSize < VPMOVBits;
+ return KMOVSize < MaskBits;
+}
+
+static bool isZeroVector(const MachineInstr &MI) {
+ switch (MI.getOpcode()) {
+ case X86::VPXORrr:
+ case X86::VPXORYrr:
+ case X86::VXORPSrr:
+ case X86::VXORPSYrr:
+ return MI.getOperand(1).getReg() == MI.getOperand(2).getReg();
+ default:
+ return false;
+ }
+}
+
+static bool isAllOnesVector(const MachineInstr &MI, bool Is256Bit) {
+ switch (MI.getOpcode()) {
+ case X86::VPCMPEQDrr:
+ return !Is256Bit && MI.getOperand(1).getReg() == MI.getOperand(2).getReg();
+ case X86::VPCMPEQDYrr:
+ return MI.getOperand(1).getReg() == MI.getOperand(2).getReg();
+ default:
+ return false;
+ }
+}
+
+static MachineInstr *getSignMaskConstantDef(MachineInstr &MI, Register Reg,
+ bool IsZero, bool Is256Bit,
+ const TargetRegisterInfo *TRI) {
+ for (MachineInstr &DefMI : llvm::reverse(llvm::make_range(
+ MI.getParent()->begin(), MachineBasicBlock::iterator(MI)))) {
+ if (!DefMI.modifiesRegister(Reg, TRI))
+ continue;
+ // Stop at the nearest def/clobber; an older matching constant may no
+ // longer be the reaching definition.
+ if (IsZero ? isZeroVector(DefMI) : isAllOnesVector(DefMI, Is256Bit))
+ return &DefMI;
+ break;
+ }
+ return nullptr;
}
static bool isCompressibleBlendVUse(unsigned BlendOpc, unsigned UseOpc) {
@@ -273,13 +314,18 @@ static bool isCompressibleBlendVUse(unsigned BlendOpc, unsigned UseOpc) {
}
}
-// Try to compress VPMOV*2M chain patterns:
-// vpmov*2m %xmm0, %k0 -> (erase this)
-// kmov* %k0, %eax -> vmovmskp* %xmm0, %eax
-// and:
-// vpmov*2m %xmm0, %k1 -> (erase this)
-// vmov* %xmm1, %xmm2 {%k1} -> vblendv* %xmm0, %xmm2, %xmm1, %xmm2
-static bool tryCompressVPMOVPattern(MachineInstr &MI, MachineBasicBlock &MBB,
+// Try to compress mask producer chains:
+// vpmov*2m %xmm0, %k0 -> (erase this)
+// kmov* %k0, %eax -> vmovmskp* %xmm0, %eax
+//
+// vpcmpge* $0, %xmm0, %k0 -> (erase this) (X >= 0)
+// vpcmpgt* $-1, %xmm0, %k0 -> (erase this) (X > -1)
+// kmov* %k0, %eax -> vmovmskp* %xmm0, %eax
+// bounded complement of %eax
+//
+// vpmov*2m %xmm0, %k1 -> (erase this)
+// vmov* %xmm1, %xmm2 {%k1} -> vblendv* %xmm0, %xmm2, %xmm1, %xmm2
+static bool tryCompressMaskProducer(MachineInstr &MI, MachineBasicBlock &MBB,
const X86Subtarget &ST,
SmallVectorImpl<MachineInstr *> &ToErase) {
const X86InstrInfo *TII = ST.getInstrInfo();
@@ -287,9 +333,13 @@ static bool tryCompressVPMOVPattern(MachineInstr &MI, MachineBasicBlock &MBB,
MachineRegisterInfo *MRI = &MBB.getParent()->getRegInfo();
unsigned Opc = MI.getOpcode();
- if (Opc != X86::VPMOVD2MZ128kr && Opc != X86::VPMOVD2MZ256kr &&
- Opc != X86::VPMOVQ2MZ128kr && Opc != X86::VPMOVQ2MZ256kr &&
- Opc != X86::VPMOVB2MZ128kr && Opc != X86::VPMOVB2MZ256kr)
+ bool IsSignMaskCmp = Opc == X86::VPCMPBZ128rri || Opc == X86::VPCMPBZ256rri ||
+ Opc == X86::VPCMPDZ128rri || Opc == X86::VPCMPDZ256rri ||
+ Opc == X86::VPCMPQZ128rri || Opc == X86::VPCMPQZ256rri;
+ if (!IsSignMaskCmp && Opc != X86::VPMOVD2MZ128kr &&
+ Opc != X86::VPMOVD2MZ256kr && Opc != X86::VPMOVQ2MZ128kr &&
+ Opc != X86::VPMOVQ2MZ256kr && Opc != X86::VPMOVB2MZ128kr &&
+ Opc != X86::VPMOVB2MZ256kr)
return false;
if (usesExtendedRegister(MI))
@@ -297,30 +347,63 @@ static bool tryCompressVPMOVPattern(MachineInstr &MI, MachineBasicBlock &MBB,
Register MaskReg = MI.getOperand(0).getReg();
Register SrcVecReg = MI.getOperand(1).getReg();
+ MachineInstr *ConstantDef = nullptr;
+ bool ConstantDefOnlyFeedsCmp = false;
+
+ if (IsSignMaskCmp) {
+ int64_t Pred = MI.getOperand(3).getImm();
+ // VPCMP signed predicates: nlt (5) folds X >= 0, nle (6) folds X > -1.
+ if (Pred != 5 && Pred != 6)
+ return false;
+ Register ConstantReg = MI.getOperand(2).getReg();
+ bool Is256Bit = Opc == X86::VPCMPBZ256rri || Opc == X86::VPCMPDZ256rri ||
+ Opc == X86::VPCMPQZ256rri;
+ // The sign-mask fold is valid only for compares against the reaching
+ // zero/all-ones vector definition.
+ ConstantDef =
+ getSignMaskConstantDef(MI, ConstantReg, Pred == 5, Is256Bit, TRI);
+ if (!ConstantDef)
+ return false;
+ // If the constant feeds only this compare, erase it with the compare.
+ ConstantDefOnlyFeedsCmp = !TRI->regsOverlap(ConstantReg, SrcVecReg);
+ for (MachineInstr &UseMI :
+ llvm::make_range(std::next(MachineBasicBlock::iterator(*ConstantDef)),
+ MachineBasicBlock::iterator(MI)))
+ if (UseMI.readsRegister(ConstantReg, TRI)) {
+ ConstantDefOnlyFeedsCmp = false;
+ break;
+ }
+ }
unsigned MovMskOpc = 0;
unsigned BlendOpc = 0;
switch (Opc) {
+ case X86::VPCMPDZ128rri:
case X86::VPMOVD2MZ128kr:
MovMskOpc = X86::VMOVMSKPSrr;
BlendOpc = X86::VBLENDVPSrrr;
break;
+ case X86::VPCMPDZ256rri:
case X86::VPMOVD2MZ256kr:
MovMskOpc = X86::VMOVMSKPSYrr;
BlendOpc = X86::VBLENDVPSYrrr;
break;
+ case X86::VPCMPQZ128rri:
case X86::VPMOVQ2MZ128kr:
MovMskOpc = X86::VMOVMSKPDrr;
BlendOpc = X86::VBLENDVPDrrr;
break;
+ case X86::VPCMPQZ256rri:
case X86::VPMOVQ2MZ256kr:
MovMskOpc = X86::VMOVMSKPDYrr;
BlendOpc = X86::VBLENDVPDYrrr;
break;
+ case X86::VPCMPBZ128rri:
case X86::VPMOVB2MZ128kr:
MovMskOpc = X86::VPMOVMSKBrr;
BlendOpc = X86::VPBLENDVBrrr;
break;
+ case X86::VPCMPBZ256rri:
case X86::VPMOVB2MZ256kr:
MovMskOpc = X86::VPMOVMSKBYrr;
BlendOpc = X86::VPBLENDVBYrrr;
@@ -343,11 +426,12 @@ static bool tryCompressVPMOVPattern(MachineInstr &MI, MachineBasicBlock &MBB,
UseOpc == X86::KMOVDrk;
// Only allow non-narrowing KMOV uses of the mask.
if (IsKMOV && CurMI.getOperand(1).getReg() == MaskReg &&
- !isKMovNarrowing(Opc, UseOpc)) {
+ !usesExtendedRegister(CurMI) &&
+ !isKMovNarrowing(getMovMskBits(Opc), UseOpc)) {
KMovMI = &CurMI;
// continue scanning to ensure
// there are no *other* uses of the mask later in the block.
- } else if (isCompressibleBlendVUse(BlendOpc, UseOpc) &&
+ } else if (!IsSignMaskCmp && isCompressibleBlendVUse(BlendOpc, UseOpc) &&
CurMI.getOperand(2).getReg() == MaskReg &&
!usesExtendedRegister(CurMI) &&
checkPredicate(BlendOpc, &ST)) {
@@ -371,6 +455,19 @@ static bool tryCompressVPMOVPattern(MachineInstr &MI, MachineBasicBlock &MBB,
if (!KMovMI && !BlendMI)
return false;
+ unsigned MovMskBits = getMovMskBits(Opc);
+ // Bounded complements define EFLAGS, unlike VPCMP + KMOV. A 32-bit
+ // complement uses NOT, which does not modify EFLAGS.
+ if (IsSignMaskCmp && KMovMI) {
+ if (KMovMI->getOperand(0).isDead() ||
+ (MovMskBits != 32 &&
+ MBB.computeRegisterLiveness(
+ TRI, X86::EFLAGS,
+ std::next(MachineBasicBlock::const_iterator(*KMovMI)),
+ MBB.size()) != MachineBasicBlock::LQR_Dead))
+ return false;
+ }
+
// Check if MaskReg is used in any other basic blocks
for (const MachineInstr &UseMI : MRI->use_instructions(MaskReg))
if (UseMI.getParent() != &MBB)
@@ -379,6 +476,7 @@ static bool tryCompressVPMOVPattern(MachineInstr &MI, MachineBasicBlock &MBB,
// Apply the transformation
MachineInstr *NewMI = nullptr;
if (KMovMI) {
+ MachineOperand OldDst = KMovMI->getOperand(0);
KMovMI->setDesc(TII->get(MovMskOpc));
MachineOperand &NewSrc = KMovMI->getOperand(1);
NewSrc.setReg(SrcVecReg);
@@ -386,6 +484,23 @@ static bool tryCompressVPMOVPattern(MachineInstr &MI, MachineBasicBlock &MBB,
// state from the VPMOV instead.
NewSrc.setIsKill(MI.getOperand(1).isKill());
NewMI = KMovMI;
+ if (IsSignMaskCmp) {
+ Register DstReg = OldDst.getReg();
+ int64_t ComplementMask =
+ APInt::getLowBitsSet(32, MovMskBits).getSExtValue();
+ unsigned ComplementOpc =
+ MovMskBits == 32
+ ? X86::NOT32r
+ : (isInt<8>(ComplementMask) ? X86::XOR32ri8 : X86::XOR32ri);
+ auto MIB = BuildMI(MBB, std::next(MachineBasicBlock::iterator(*KMovMI)),
+ KMovMI->getDebugLoc(), TII->get(ComplementOpc), DstReg)
+ .addReg(DstReg, RegState::Kill);
+ if (MovMskBits != 32) {
+ MIB.addImm(ComplementMask);
+ MIB->findRegisterDefOperand(X86::EFLAGS, TRI)->setIsDead();
+ }
+ MIB->getOperand(0).setIsRenamable(OldDst.isRenamable());
+ }
} else if (BlendMI) {
const MachineOperand &MaskVec = MI.getOperand(1);
const MachineOperand &Dst = BlendMI->getOperand(0);
@@ -407,6 +522,8 @@ static bool tryCompressVPMOVPattern(MachineInstr &MI, MachineBasicBlock &MBB,
assert(NewMI && "Expected a compressed instruction");
NewMI->setAsmPrinterFlag(X86::AC_EVEX_2_VEX);
ToErase.push_back(&MI);
+ if (ConstantDefOnlyFeedsCmp && MI.getOperand(2).isKill())
+ ToErase.push_back(ConstantDef);
return true;
}
@@ -423,8 +540,8 @@ static bool CompressEVEXImpl(MachineInstr &MI, MachineBasicBlock &MBB,
if (TSFlags & (X86II::EVEX_K | X86II::EVEX_L2))
return false;
- // Specialized VPMOVD2M + KMOV -> MOVMSK fold first.
- if (tryCompressVPMOVPattern(MI, MBB, ST, ToErase))
+ // Specialized mask-producing folds to MOVMSK/VBLENDV first.
+ if (tryCompressMaskProducer(MI, MBB, ST, ToErase))
return true;
auto IsRedundantNewDataDest = [&](unsigned &Opc) {
diff --git a/llvm/test/CodeGen/X86/avx512-icmp-sle-zero.ll b/llvm/test/CodeGen/X86/avx512-icmp-sle-zero.ll
index ef8da7d206ac0..5b71fbfcb4847 100644
--- a/llvm/test/CodeGen/X86/avx512-icmp-sle-zero.ll
+++ b/llvm/test/CodeGen/X86/avx512-icmp-sle-zero.ll
@@ -81,9 +81,8 @@ define i16 @cmp_sge_v16i8(<16 x i8> %x) {
;
; AVX512-LABEL: cmp_sge_v16i8:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512-NEXT: vpcmpnltb %xmm1, %xmm0, %k0
-; AVX512-NEXT: kmovd %k0, %eax
+; AVX512-NEXT: vpmovmskb %xmm0, %eax
+; AVX512-NEXT: xorl $65535, %eax # imm = 0xFFFF
; AVX512-NEXT: # kill: def $ax killed $ax killed $eax
; AVX512-NEXT: retq
%c = icmp sge <16 x i8> %x, splat (i8 0)
@@ -101,9 +100,8 @@ define i16 @cmp_sgt_neg1_v16i8(<16 x i8> %x) {
;
; AVX512-LABEL: cmp_sgt_neg1_v16i8:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512-NEXT: vpcmpnltb %xmm1, %xmm0, %k0
-; AVX512-NEXT: kmovd %k0, %eax
+; AVX512-NEXT: vpmovmskb %xmm0, %eax
+; AVX512-NEXT: xorl $65535, %eax # imm = 0xFFFF
; AVX512-NEXT: # kill: def $ax killed $ax killed $eax
; AVX512-NEXT: retq
%c = icmp sgt <16 x i8> %x, splat (i8 -1)
diff --git a/llvm/test/CodeGen/X86/bitcast-sign-mask.ll b/llvm/test/CodeGen/X86/bitcast-sign-mask.ll
new file mode 100644
index 0000000000000..e4a06df9a0a1a
--- /dev/null
+++ b/llvm/test/CodeGen/X86/bitcast-sign-mask.ll
@@ -0,0 +1,215 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 | FileCheck %s --check-prefix=SSE
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64-v3 | FileCheck %s --check-prefixes=AVX,AVX2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64-v4 | FileCheck %s --check-prefixes=AVX,AVX512
+
+; Prefer MOVMSK when bitcasting complementary signed integer sign tests.
+
+define i16 @cmp_sge_v16i8(<16 x i8> %x) {
+; SSE-LABEL: cmp_sge_v16i8:
+; SSE: # %bb.0:
+; SSE-NEXT: pmovmskb %xmm0, %eax
+; SSE-NEXT: notl %eax
+; SSE-NEXT: # kill: def $ax killed $ax killed $eax
+; SSE-NEXT: retq
+;
+; AVX2-LABEL: cmp_sge_v16i8:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vpmovmskb %xmm0, %eax
+; AVX2-NEXT: notl %eax
+; AVX2-NEXT: # kill: def $ax killed $ax killed $eax
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: cmp_sge_v16i8:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpmovmskb %xmm0, %eax
+; AVX512-NEXT: xorl $65535, %eax # imm = 0xFFFF
+; AVX512-NEXT: # kill: def $ax killed $ax killed $eax
+; AVX512-NEXT: retq
+ %c = icmp sge <16 x i8> %x, zeroinitializer
+ %r = bitcast <16 x i1> %c to i16
+ ret i16 %r
+}
+
+define i16 @cmp_sgt_neg1_v16i8(<16 x i8> %x) {
+; SSE-LABEL: cmp_sgt_neg1_v16i8:
+; SSE: # %bb.0:
+; SSE-NEXT: pmovmskb %xmm0, %eax
+; SSE-NEXT: notl %eax
+; SSE-NEXT: # kill: def $ax killed $ax killed $eax
+; SSE-NEXT: retq
+;
+; AVX2-LABEL: cmp_sgt_neg1_v16i8:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vpmovmskb %xmm0, %eax
+; AVX2-NEXT: notl %eax
+; AVX2-NEXT: # kill: def $ax killed $ax killed $eax
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: cmp_sgt_neg1_v16i8:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpmovmskb %xmm0, %eax
+; AVX512-NEXT: xorl $65535, %eax # imm = 0xFFFF
+; AVX512-NEXT: # kill: def $ax killed $ax killed $eax
+; AVX512-NEXT: retq
+ %c = icmp sgt <16 x i8> %x, splat (i8 -1)
+ %r = bitcast <16 x i1> %c to i16
+ ret i16 %r
+}
+
+define i32 @cmp_sge_v32i8(<32 x i8> %x) {
+; SSE-LABEL: cmp_sge_v32i8:
+; SSE: # %bb.0:
+; SSE-NEXT: pmovmskb %xmm0, %ecx
+; SSE-NEXT: xorl $65535, %ecx # imm = 0xFFFF
+; SSE-NEXT: pmovmskb %xmm1, %eax
+; SSE-NEXT: notl %eax
+; SSE-NEXT: shll $16, %eax
+; SSE-NEXT: orl %ecx, %eax
+; SSE-NEXT: retq
+;
+; AVX-LABEL: cmp_sge_v32i8:
+; AVX: # %bb.0:
+; AVX-NEXT: vpmovmskb %ymm0, %eax
+; AVX-NEXT: notl %eax
+; AVX-NEXT: vzeroupper
+; AVX-NEXT: retq
+ %c = icmp sge <32 x i8> %x, zeroinitializer
+ %r = bitcast <32 x i1> %c to i32
+ ret i32 %r
+}
+
+define i32 @cmp_sgt_neg1_v32i8(<32 x i8> %x) {
+; SSE-LABEL: cmp_sgt_neg1_v32i8:
+; SSE: # %bb.0:
+; SSE-NEXT: pmovmskb %xmm0, %ecx
+; SSE-NEXT: xorl $65535, %ecx # imm = 0xFFFF
+; SSE-NEXT: pmovmskb %xmm1, %eax
+; SSE-NEXT: notl %eax
+; SSE-NEXT: shll $16, %eax
+; SSE-NEXT: orl %ecx, %eax
+; SSE-NEXT: retq
+;
+; AVX-LABEL: cmp_sgt_neg1_v32i8:
+; AVX: # %bb.0:
+; AVX-NEXT: vpmovmskb %ymm0, %eax
+; AVX-NEXT: notl %eax
+; AVX-NEXT: vzeroupper
+; AVX-NEXT: retq
+ %c = icmp sgt <32 x i8> %x, splat (i8 -1)
+ %r = bitcast <32 x i1> %c to i32
+ ret i32 %r
+}
+
+define i4 @cmp_sge_v4i32(<4 x i32> %x) {
+; SSE-LABEL: cmp_sge_v4i32:
+; SSE: # %bb.0:
+; SSE-NEXT: movmskps %xmm0, %eax
+; SSE-NEXT: xorb $15, %al
+; SSE-NEXT: # kill: def $al killed $al killed $eax
+; SSE-NEXT: retq
+;
+; AVX2-LABEL: cmp_sge_v4i32:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vmovmskps %xmm0, %eax
+; AVX2-NEXT: xorb $15, %al
+; AVX2-NEXT: # kill: def $al killed $al killed $eax
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: cmp_sge_v4i32:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vmovmskps %xmm0, %eax
+; AVX512-NEXT: xorl $15, %eax
+; AVX512-NEXT: # kill: def $al killed $al killed $eax
+; AVX512-NEXT: retq
+ %c = icmp sge <4 x i32> %x, zeroinitializer
+ %r = bitcast <4 x i1> %c to i4
+ ret i4 %r
+}
+
+define i8 @cmp_sge_v8i32(<8 x i32> %x) {
+; SSE-LABEL: cmp_sge_v8i32:
+; SSE: # %bb.0:
+; SSE-NEXT: pcmpeqd %xmm2, %xmm2
+; SSE-NEXT: pxor %xmm2, %xmm1
+; SSE-NEXT: pxor %xmm2, %xmm0
+; SSE-NEXT: packssdw %xmm1, %xmm0
+; SSE-NEXT: packsswb %xmm0, %xmm0
+; SSE-NEXT: pmovmskb %xmm0, %eax
+; SSE-NEXT: # kill: def $al killed $al killed $eax
+; SSE-NEXT: retq
+;
+; AVX2-LABEL: cmp_sge_v8i32:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vmovmskps %ymm0, %eax
+; AVX2-NEXT: notb %al
+; AVX2-NEXT: # kill: def $al killed $al killed $eax
+; AVX2-NEXT: vzeroupper
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: cmp_sge_v8i32:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vmovmskps %ymm0, %eax
+; AVX512-NEXT: xorl $255, %eax
+; AVX512-NEXT: # kill: def $al killed $al killed $eax
+; AVX512-NEXT: vzeroupper
+; AVX512-NEXT: retq
+ %c = icmp sge <8 x i32> %x, zeroinitializer
+ %r = bitcast <8 x i1> %c to i8
+ ret i8 %r
+}
+
+define i2 @cmp_sge_v2i64(<2 x i64> %x) {
+; SSE-LABEL: cmp_sge_v2i64:
+; SSE: # %bb.0:
+; SSE-NEXT: movmskpd %xmm0, %eax
+; SSE-NEXT: xorb $3, %al
+; SSE-NEXT: # kill: def $al killed $al killed $eax
+; SSE-NEXT: retq
+;
+; AVX2-LABEL: cmp_sge_v2i64:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vmovmskpd %xmm0, %eax
+; AVX2-NEXT: xorb $3, %al
+; AVX2-NEXT: # kill: def $al killed $al killed $eax
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: cmp_sge_v2i64:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vmovmskpd %xmm0, %eax
+; AVX512-NEXT: xorl $3, %eax
+; AVX512-NEXT: # kill: def $al killed $al killed $eax
+; AVX512-NEXT: retq
+ %c = icmp sge <2 x i64> %x, zeroinitializer
+ %r = bitcast <2 x i1> %c to i2
+ ret i2 %r
+}
+
+define i4 @cmp_sge_v4i64(<4 x i64> %x) {
+; SSE-LABEL: cmp_sge_v4i64:
+; SSE: # %bb.0:
+; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[1,3],xmm1[1,3]
+; SSE-NEXT: movmskps %xmm0, %eax
+; SSE-NEXT: xorl $15, %eax
+; SSE-NEXT: # kill: def $al killed $al killed $eax
+; SSE-NEXT: retq
+;
+; AVX2-LABEL: cmp_sge_v4i64:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vmovmskpd %ymm0, %eax
+; AVX2-NEXT: xorb $15, %al
+; AVX2-NEXT: # kill: def $al killed $al killed $eax
+; AVX2-NEXT: vzeroupper
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: cmp_sge_v4i64:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vmovmskpd %ymm0, %eax
+; AVX512-NEXT: xorl $15, %eax
+; AVX512-NEXT: # kill: def $al killed $al killed $eax
+; AVX512-NEXT: vzeroupper
+; AVX512-NEXT: retq
+ %c = icmp sge <4 x i64> %x, zeroinitializer
+ %r = bitcast <4 x i1> %c to i4
+ ret i4 %r
+}
diff --git a/llvm/test/CodeGen/X86/compress-evex-sign-mask.mir b/llvm/test/CodeGen/X86/compress-evex-sign-mask.mir
new file mode 100644
index 0000000000000..60c7e65fe9eb0
--- /dev/null
+++ b/llvm/test/CodeGen/X86/compress-evex-sign-mask.mir
@@ -0,0 +1,240 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -mtriple=x86_64-- -run-pass=x86-compress-evex \
+# RUN: -verify-machineinstrs -mcpu=skx -o - %s | FileCheck %s
+
+---
+name: sign_mask_positive
+body: |
+ bb.0:
+ liveins: $xmm0, $xmm2, $xmm4, $ymm6, $ymm8, $ymm10, $xmm12
+
+ ; CHECK-LABEL: name: sign_mask_positive
+ ; CHECK: liveins: $xmm0, $xmm2, $xmm4, $ymm6, $ymm8, $ymm10, $xmm12
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: $eax = VPMOVMSKBrr killed $xmm0
+ ; CHECK-NEXT: $eax = XOR32ri killed $eax, 65535, implicit-def dead $eflags
+ ; CHECK-NEXT: $ecx = VPMOVMSKBYrr killed $ymm2
+ ; CHECK-NEXT: $ecx = NOT32r killed $ecx
+ ; CHECK-NEXT: $edx = VMOVMSKPSrr killed $xmm4
+ ; CHECK-NEXT: $edx = XOR32ri8 killed $edx, 15, implicit-def dead $eflags
+ ; CHECK-NEXT: $esi = VMOVMSKPSYrr killed $ymm6
+ ; CHECK-NEXT: $esi = XOR32ri killed $esi, 255, implicit-def dead $eflags
+ ; CHECK-NEXT: $edi = VMOVMSKPDrr killed $xmm8
+ ; CHECK-NEXT: $edi = XOR32ri8 killed $edi, 3, implicit-def dead $eflags
+ ; CHECK-NEXT: $r8d = VMOVMSKPDYrr killed $ymm10
+ ; CHECK-NEXT: $r8d = XOR32ri8 killed $r8d, 15, implicit-def dead $eflags
+ ; CHECK-NEXT: $r9d = VPMOVMSKBrr killed $xmm12
+ ; CHECK-NEXT: $r9d = XOR32ri killed $r9d, 65535, implicit-def dead $eflags
+ ; CHECK-NEXT: RET64
+ $xmm1 = VPXORrr undef $xmm1, undef $xmm1
+ $k0 = VPCMPBZ128rri killed $xmm0, killed $xmm1, 5
+ $eax = KMOVDrk killed $k0
+
+ $xmm3 = VPXORrr undef $xmm3, undef $xmm3, implicit-def $ymm3
+ $k1 = VPCMPBZ256rri killed $ymm2, killed $ymm3, 5
+ $ecx = KMOVDrk killed $k1
+
+ $xmm5 = VPXORrr undef $xmm5, undef $xmm5
+ $k2 = VPCMPDZ128rri killed $xmm4, killed $xmm5, 5
+ $edx = KMOVBrk killed $k2
+
+ $xmm7 = VPXORrr undef $xmm7, undef $xmm7, implicit-def $ymm7
+ $k3 = VPCMPDZ256rri killed $ymm6, killed $ymm7, 5
+ $esi = KMOVBrk killed $k3
+
+ $xmm9 = VPXORrr undef $xmm9, undef $xmm9
+ $k4 = VPCMPQZ128rri killed $xmm8, killed $xmm9, 5
+ $edi = KMOVBrk killed $k4
+
+ $xmm11 = VPXORrr undef $xmm11, undef $xmm11, implicit-def $ymm11
+ $k5 = VPCMPQZ256rri killed $ymm10, killed $ymm11, 5
+ $r8d = KMOVBrk killed $k5
+
+ ; Also recognize the existing SETGT(X, -1) canonical form.
+ $xmm13 = VPCMPEQDrr undef $xmm13, undef $xmm13
+ $k6 = VPCMPBZ128rri killed $xmm12, killed $xmm13, 6
+ $r9d = KMOVDrk killed $k6
+ RET64
+...
+
+---
+name: sign_mask_256_allones_positive
+body: |
+ bb.0:
+ liveins: $ymm0
+
+ ; A full-width all-ones producer makes predicate 6 equivalent to X >= 0.
+ ; CHECK-LABEL: name: sign_mask_256_allones_positive
+ ; CHECK: liveins: $ymm0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: $eax = VMOVMSKPSYrr killed $ymm0
+ ; CHECK-NEXT: $eax = XOR32ri killed $eax, 255, implicit-def dead $eflags
+ ; CHECK-NEXT: RET64
+ $ymm1 = VPCMPEQDYrr undef $ymm1, undef $ymm1
+ $k0 = VPCMPDZ256rri killed $ymm0, killed $ymm1, 6
+ $eax = KMOVBrk killed $k0
+ RET64
+...
+
+---
+name: sign_mask_negative
+body: |
+ bb.0:
+ liveins: $xmm0, $xmm2, $xmm4, $ymm6, $zmm8, $xmm16, $xmm12, $r10d, $r11d, $eflags
+
+ ; The mask has another use.
+ ; CHECK-LABEL: name: sign_mask_negative
+ ; CHECK: liveins: $xmm0, $xmm2, $xmm4, $ymm6, $zmm8, $xmm16, $xmm12, $r10d, $r11d, $eflags
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: $xmm1 = VPXORrr undef $xmm1, undef $xmm1
+ ; CHECK-NEXT: $k0 = VPCMPBZ128rri killed $xmm0, killed $xmm1, 5
+ ; CHECK-NEXT: $eax = KMOVDrk $k0
+ ; CHECK-NEXT: $ecx = KMOVDrk killed $k0
+ ; CHECK-NEXT: $xmm3 = VPXORrr undef $xmm3, undef $xmm3
+ ; CHECK-NEXT: $k1 = VPCMPDZ128rri $xmm2, killed $xmm3, 5
+ ; CHECK-NEXT: $xmm2 = VMOVAPSrr $xmm4
+ ; CHECK-NEXT: $edx = KMOVBrk killed $k1
+ ; CHECK-NEXT: $xmm5 = VPXORrr undef $xmm5, undef $xmm5
+ ; CHECK-NEXT: $k2 = VPCMPQZ128rri killed $xmm4, killed $xmm5, 4
+ ; CHECK-NEXT: $esi = KMOVBrk killed $k2
+ ; CHECK-NEXT: $xmm7 = VPXORrr undef $xmm7, undef $xmm7, implicit-def $ymm7
+ ; CHECK-NEXT: $k3 = VPCMPBZ256rri killed $ymm6, killed $ymm7, 5
+ ; CHECK-NEXT: $edi = KMOVWrk killed $k3
+ ; CHECK-NEXT: $zmm9 = VPXORDZrr undef $zmm9, undef $zmm9
+ ; CHECK-NEXT: $k4 = VPCMPDZrri killed $zmm8, killed $zmm9, 5
+ ; CHECK-NEXT: $r8d = KMOVWrk killed $k4
+ ; CHECK-NEXT: $xmm10 = VPXORrr undef $xmm10, undef $xmm10
+ ; CHECK-NEXT: $k5 = VPCMPBZ128rri killed $xmm16, killed $xmm10, 5
+ ; CHECK-NEXT: $r9d = KMOVDrk killed $k5
+ ; CHECK-NEXT: $xmm13 = VPXORrr undef $xmm13, undef $xmm13
+ ; CHECK-NEXT: $k6 = VPCMPDZ128rri killed $xmm12, killed $xmm13, 5
+ ; CHECK-NEXT: $r10d = KMOVBrk killed $k6
+ ; CHECK-NEXT: $r10d = ADC32rr $r10d, $r11d, implicit-def dead $eflags, implicit killed $eflags
+ ; CHECK-NEXT: RET64
+ $xmm1 = VPXORrr undef $xmm1, undef $xmm1
+ $k0 = VPCMPBZ128rri killed $xmm0, killed $xmm1, 5
+ $eax = KMOVDrk $k0
+ $ecx = KMOVDrk killed $k0
+
+ ; The vector source is clobbered before the KMOV.
+ $xmm3 = VPXORrr undef $xmm3, undef $xmm3
+ $k1 = VPCMPDZ128rri $xmm2, killed $xmm3, 5
+ $xmm2 = VMOVAPSrr $xmm4
+ $edx = KMOVBrk killed $k1
+
+ ; Predicate 4 is not a sign-bit-clear test.
+ $xmm5 = VPXORrr undef $xmm5, undef $xmm5
+ $k2 = VPCMPQZ128rri killed $xmm4, killed $xmm5, 4
+ $esi = KMOVBrk killed $k2
+
+ ; A narrowing KMOV cannot preserve every byte lane.
+ $xmm7 = VPXORrr undef $xmm7, undef $xmm7, implicit-def $ymm7
+ $k3 = VPCMPBZ256rri killed $ymm6, killed $ymm7, 5
+ $edi = KMOVWrk killed $k3
+
+ ; There is no 512-bit MOVMSK replacement.
+ $zmm9 = VPXORDZrr undef $zmm9, undef $zmm9
+ $k4 = VPCMPDZrri killed $zmm8, killed $zmm9, 5
+ $r8d = KMOVWrk killed $k4
+
+ ; VEX MOVMSK cannot encode an extended vector register.
+ $xmm10 = VPXORrr undef $xmm10, undef $xmm10
+ $k5 = VPCMPBZ128rri killed $xmm16, killed $xmm10, 5
+ $r9d = KMOVDrk killed $k5
+
+ ; The bounded XOR would clobber live EFLAGS.
+ $xmm13 = VPXORrr undef $xmm13, undef $xmm13
+ $k6 = VPCMPDZ128rri killed $xmm12, killed $xmm13, 5
+ $r10d = KMOVBrk killed $k6
+ $r10d = ADC32rr $r10d, $r11d, implicit-def dead $eflags, implicit killed $eflags
+ RET64
+...
+
+---
+name: sign_mask_blend_not_compressed
+body: |
+ bb.0:
+ liveins: $xmm0, $xmm2, $xmm3
+
+ ; The compare mask has the opposite polarity from the source sign bits, so
+ ; it cannot be replaced by a VBLENDV using the source as its mask.
+ ; CHECK-LABEL: name: sign_mask_blend_not_compressed
+ ; CHECK: liveins: $xmm0, $xmm2, $xmm3
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: $xmm1 = VPXORrr undef $xmm1, undef $xmm1
+ ; CHECK-NEXT: $k1 = VPCMPDZ128rri killed $xmm0, killed $xmm1, 5
+ ; CHECK-NEXT: $xmm2 = VMOVAPSZ128rrk $xmm2, killed $k1, $xmm3
+ ; CHECK-NEXT: RET64 implicit killed $xmm2
+ $xmm1 = VPXORrr undef $xmm1, undef $xmm1
+ $k1 = VPCMPDZ128rri killed $xmm0, killed $xmm1, 5
+ $xmm2 = VMOVAPSZ128rrk $xmm2, killed $k1, $xmm3
+ RET64 implicit killed $xmm2
+...
+
+---
+name: sign_mask_wrong_constants
+body: |
+ bb.0:
+ liveins: $xmm0, $xmm2
+
+ ; Predicate 5 requires zero, not all ones.
+ ; CHECK-LABEL: name: sign_mask_wrong_constants
+ ; CHECK: liveins: $xmm0, $xmm2
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: $xmm1 = VPCMPEQDrr undef $xmm1, undef $xmm1
+ ; CHECK-NEXT: $k0 = VPCMPBZ128rri killed $xmm0, killed $xmm1, 5
+ ; CHECK-NEXT: $eax = KMOVDrk killed $k0
+ ; CHECK-NEXT: $xmm3 = VPXORrr undef $xmm3, undef $xmm3
+ ; CHECK-NEXT: $k1 = VPCMPQZ128rri killed $xmm2, killed $xmm3, 6
+ ; CHECK-NEXT: $ecx = KMOVBrk killed $k1
+ ; CHECK-NEXT: RET64
+ $xmm1 = VPCMPEQDrr undef $xmm1, undef $xmm1
+ $k0 = VPCMPBZ128rri killed $xmm0, killed $xmm1, 5
+ $eax = KMOVDrk killed $k0
+
+ ; Predicate 6 requires all ones, not zero.
+ $xmm3 = VPXORrr undef $xmm3, undef $xmm3
+ $k1 = VPCMPQZ128rri killed $xmm2, killed $xmm3, 6
+ $ecx = KMOVBrk killed $k1
+ RET64
+...
+
+---
+name: sign_mask_256_allones_width
+body: |
+ bb.0:
+ liveins: $ymm0
+
+ ; A 128-bit all-ones idiom zeroes, rather than sets, the upper YMM lanes.
+ ; CHECK-LABEL: name: sign_mask_256_allones_width
+ ; CHECK: liveins: $ymm0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: $xmm1 = VPCMPEQDrr undef $xmm1, undef $xmm1, implicit-def $ymm1
+ ; CHECK-NEXT: $k0 = VPCMPDZ256rri killed $ymm0, killed $ymm1, 6
+ ; CHECK-NEXT: $eax = KMOVBrk killed $k0
+ ; CHECK-NEXT: RET64
+ $xmm1 = VPCMPEQDrr undef $xmm1, undef $xmm1, implicit-def $ymm1
+ $k0 = VPCMPDZ256rri killed $ymm0, killed $ymm1, 6
+ $eax = KMOVBrk killed $k0
+ RET64
+...
+
+---
+name: sign_mask_dead_kmov_destination
+body: |
+ bb.0:
+ liveins: $xmm0
+
+ ; The new XOR would consume a destination that was originally dead.
+ ; CHECK-LABEL: name: sign_mask_dead_kmov_destination
+ ; CHECK: liveins: $xmm0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: $xmm1 = VPXORrr undef $xmm1, undef $xmm1
+ ; CHECK-NEXT: $k0 = VPCMPBZ128rri killed $xmm0, killed $xmm1, 5
+ ; CHECK-NEXT: dead $eax = KMOVDrk killed $k0
+ ; CHECK-NEXT: RET64
+ $xmm1 = VPXORrr undef $xmm1, undef $xmm1
+ $k0 = VPCMPBZ128rri killed $xmm0, killed $xmm1, 5
+ dead $eax = KMOVDrk killed $k0
+ RET64
+...
diff --git a/llvm/test/CodeGen/X86/compress-evex-vpmov-kill.mir b/llvm/test/CodeGen/X86/compress-evex-vpmov-kill.mir
index 052e466599a9d..bb6cb7e9c42d4 100644
--- a/llvm/test/CodeGen/X86/compress-evex-vpmov-kill.mir
+++ b/llvm/test/CodeGen/X86/compress-evex-vpmov-kill.mir
@@ -1,7 +1,7 @@
# RUN: llc %s -mtriple=x86_64-unknown -mattr=+avx512vl,+avx512dq \
# RUN: -run-pass=x86-compress-evex -o - | FileCheck %s
#
-# tryCompressVPMOVPattern folds VPMOV*2M + KMOV into VMOVMSK. The kill flag on
+# tryCompressMaskProducer folds VPMOV*2M + KMOV into VMOVMSK. The kill flag on
# the XMM source must be derived from the VPMOV. Otherwise it inherits the kill
# flag from the KMOV's mask operand, which can mark a still-live XMM register as
# killed and produce incorrect liveness.
More information about the llvm-commits
mailing list