[llvm] [X86] Prefer VPMOVMSKB for nonnegative byte comparison masks (PR #216884)

Oscar Priego via llvm-commits llvm-commits at lists.llvm.org
Mon Aug 17 22:06:08 PDT 2026


https://github.com/Opriego updated https://github.com/llvm/llvm-project/pull/216884

>From dabf34ca6607944cba7d8c291475b2f1d73226fe Mon Sep 17 00:00:00 2001
From: Oscar Priego Verdugo <oscar.priegov at gmail.com>
Date: Mon, 17 Aug 2026 17:52:18 -0600
Subject: [PATCH] [X86] Prefer VPMOVMSKB for nonnegative byte comparison masks

For v16i8 and v32i8 comparisons, `icmp sge x, 0` and
`icmp sgt x, -1` are both checking whether the sign bit is clear.

When the comparison result is immediately bitcast to an integer mask,
we can use the existing MOVMSK path instead of generating an AVX-512
compare into a k-register and then moving that mask back to a GPR.

This produces VPMOVMSKB followed by NOT, which is the same sequence
already generated without AVX-512 for these cases.

Keep the existing `SETLT x, 0` handling unchanged and limit the new
cases to v16i8 and v32i8, where VPMOVMSKB directly provides the mask
we need.

Add x86-64-v3 and x86-64-v4 tests for both comparison forms.

Fixes #216774
Signed-off-by: Oscar Priego Verdugo <oscar.priegov at gmail.com>
---
 llvm/lib/Target/X86/X86ISelLowering.cpp    | 20 +++--
 llvm/test/CodeGen/X86/bitcast-sign-mask.ll | 88 ++++++++++++++++++++++
 2 files changed, 103 insertions(+), 5 deletions(-)
 create mode 100644 llvm/test/CodeGen/X86/bitcast-sign-mask.ll

diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index e89afce282602..a3a7990b4d55b 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -46518,14 +46518,24 @@ static SDValue combineBitcastvxi1(SelectionDAG &DAG, EVT VT, SDValue Src,
                        Src.getOperand(0).getValueType() == MVT::v32i8 ||
                        Src.getOperand(0).getValueType() == MVT::v64i8);
 
-  // Prefer movmsk for AVX512 for (bitcast (setlt X, 0)) which can be handled
+  // Prefer movmsk for AVX512 for (bitcast (setlt X, 0)), which can be handled
   // directly with vpmovmskb/vmovmskps/vmovmskpd.
-  if (Src.getOpcode() == ISD::SETCC && Src.hasOneUse() &&
-      cast<CondCodeSDNode>(Src.getOperand(2))->get() == ISD::SETLT &&
-      ISD::isBuildVectorAllZeros(Src.getOperand(1).getNode())) {
+  if (Src.getOpcode() == ISD::SETCC && Src.hasOneUse()) {
+    ISD::CondCode CC = cast<CondCodeSDNode>(Src.getOperand(2))->get();
     EVT CmpVT = Src.getOperand(0).getValueType();
     EVT EltVT = CmpVT.getVectorElementType();
-    if (CmpVT.getSizeInBits() <= 256 &&
+    SDNode *RHS = Src.getOperand(1).getNode();
+
+    bool IsZero = ISD::isBuildVectorAllZeros(RHS);
+    bool IsNegativeTest = CC == ISD::SETLT && IsZero;
+
+    // For signed i8, x >= 0 and x > -1 both test that the sign bit is clear.
+    bool IsNonNegativeByteTest =
+        (CmpVT == MVT::v16i8 || CmpVT == MVT::v32i8) &&
+        ((CC == ISD::SETGE && IsZero) ||
+         (CC == ISD::SETGT && ISD::isBuildVectorAllOnes(RHS)));
+    if ((IsNegativeTest || IsNonNegativeByteTest) &&
+        CmpVT.getSizeInBits() <= 256 &&
         (EltVT == MVT::i8 || EltVT == MVT::i32 || EltVT == MVT::i64))
       PreferMovMsk = true;
   }
diff --git a/llvm/test/CodeGen/X86/bitcast-sign-mask.ll b/llvm/test/CodeGen/X86/bitcast-sign-mask.ll
new file mode 100644
index 0000000000000..26de7487b6d32
--- /dev/null
+++ b/llvm/test/CodeGen/X86/bitcast-sign-mask.ll
@@ -0,0 +1,88 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 | FileCheck %s --check-prefix=SSE
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64-v3 | FileCheck %s --check-prefix=AVX
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64-v4 | FileCheck %s --check-prefix=AVX
+
+; Prefer MOVMSK when bitcasting complementary signed byte sign tests.
+
+define i16 @cmp_sge_v16i8(<16 x i8> %x) {
+; SSE-LABEL: cmp_sge_v16i8:
+; SSE:       # %bb.0:
+; SSE-NEXT:    pmovmskb %xmm0, %eax
+; SSE-NEXT:    notl %eax
+; SSE-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: cmp_sge_v16i8:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpmovmskb %xmm0, %eax
+; AVX-NEXT:    notl %eax
+; AVX-NEXT:    # kill: def $ax killed $ax killed $eax
+; AVX-NEXT:    retq
+  %c = icmp sge <16 x i8> %x, zeroinitializer
+  %r = bitcast <16 x i1> %c to i16
+  ret i16 %r
+}
+
+define i16 @cmp_sgt_neg1_v16i8(<16 x i8> %x) {
+; SSE-LABEL: cmp_sgt_neg1_v16i8:
+; SSE:       # %bb.0:
+; SSE-NEXT:    pmovmskb %xmm0, %eax
+; SSE-NEXT:    notl %eax
+; SSE-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: cmp_sgt_neg1_v16i8:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpmovmskb %xmm0, %eax
+; AVX-NEXT:    notl %eax
+; AVX-NEXT:    # kill: def $ax killed $ax killed $eax
+; AVX-NEXT:    retq
+  %c = icmp sgt <16 x i8> %x, splat (i8 -1)
+  %r = bitcast <16 x i1> %c to i16
+  ret i16 %r
+}
+
+define i32 @cmp_sge_v32i8(<32 x i8> %x) {
+; SSE-LABEL: cmp_sge_v32i8:
+; SSE:       # %bb.0:
+; SSE-NEXT:    pmovmskb %xmm0, %ecx
+; SSE-NEXT:    xorl $65535, %ecx # imm = 0xFFFF
+; SSE-NEXT:    pmovmskb %xmm1, %eax
+; SSE-NEXT:    notl %eax
+; SSE-NEXT:    shll $16, %eax
+; SSE-NEXT:    orl %ecx, %eax
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: cmp_sge_v32i8:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpmovmskb %ymm0, %eax
+; AVX-NEXT:    notl %eax
+; AVX-NEXT:    vzeroupper
+; AVX-NEXT:    retq
+  %c = icmp sge <32 x i8> %x, zeroinitializer
+  %r = bitcast <32 x i1> %c to i32
+  ret i32 %r
+}
+
+define i32 @cmp_sgt_neg1_v32i8(<32 x i8> %x) {
+; SSE-LABEL: cmp_sgt_neg1_v32i8:
+; SSE:       # %bb.0:
+; SSE-NEXT:    pmovmskb %xmm0, %ecx
+; SSE-NEXT:    xorl $65535, %ecx # imm = 0xFFFF
+; SSE-NEXT:    pmovmskb %xmm1, %eax
+; SSE-NEXT:    notl %eax
+; SSE-NEXT:    shll $16, %eax
+; SSE-NEXT:    orl %ecx, %eax
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: cmp_sgt_neg1_v32i8:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpmovmskb %ymm0, %eax
+; AVX-NEXT:    notl %eax
+; AVX-NEXT:    vzeroupper
+; AVX-NEXT:    retq
+  %c = icmp sgt <32 x i8> %x, splat (i8 -1)
+  %r = bitcast <32 x i1> %c to i32
+  ret i32 %r
+}



More information about the llvm-commits mailing list