[llvm] [X86] Compress masked VPBLENDM using VBLENDV (PR #221838)
Oscar Priego via llvm-commits
llvm-commits at lists.llvm.org
Tue Sep 8 02:44:43 PDT 2026
https://github.com/Opriego updated https://github.com/llvm/llvm-project/pull/221838
>From 3bfb4218fcb1c2f95d7c1b5f1510dabdb6d32c65 Mon Sep 17 00:00:00 2001
From: Oscar Priego Verdugo <oscar.priegov at gmail.com>
Date: Mon, 7 Sep 2026 15:15:32 -0600
Subject: [PATCH] [X86] Compress masked blends using VBLENDV
Signed-off-by: Oscar Priego Verdugo <oscar.priegov at gmail.com>
---
llvm/lib/Target/X86/X86CompressEVEX.cpp | 44 +++++++++++++++----
.../test/CodeGen/X86/avx512-regcall-NoMask.ll | 9 ++--
.../X86/compress-evex-vpmov-blendv.mir | 31 +++++++++++++
.../CodeGen/X86/compress-evex-vpmov-kill.mir | 30 +++++++++++++
4 files changed, 99 insertions(+), 15 deletions(-)
diff --git a/llvm/lib/Target/X86/X86CompressEVEX.cpp b/llvm/lib/Target/X86/X86CompressEVEX.cpp
index 18d655288f400..8e48ebf08733e 100644
--- a/llvm/lib/Target/X86/X86CompressEVEX.cpp
+++ b/llvm/lib/Target/X86/X86CompressEVEX.cpp
@@ -314,6 +314,21 @@ static bool isCompressibleBlendVUse(unsigned BlendOpc, unsigned UseOpc) {
}
}
+static bool isCompressibleMaskedBlendUse(unsigned BlendOpc, unsigned UseOpc) {
+ switch (BlendOpc) {
+ case X86::VBLENDVPSrrr:
+ return UseOpc == X86::VPBLENDMDZ128rrk;
+ case X86::VBLENDVPSYrrr:
+ return UseOpc == X86::VPBLENDMDZ256rrk;
+ case X86::VBLENDVPDrrr:
+ return UseOpc == X86::VPBLENDMQZ128rrk;
+ case X86::VBLENDVPDYrrr:
+ return UseOpc == X86::VPBLENDMQZ256rrk;
+ default:
+ return false;
+ }
+}
+
// Try to compress mask producer chains:
// vpmov*2m %xmm0, %k0 -> (erase this)
// kmov* %k0, %eax -> vmovmskp* %xmm0, %eax
@@ -414,6 +429,7 @@ static bool tryCompressMaskProducer(MachineInstr &MI, MachineBasicBlock &MBB,
MachineInstr *KMovMI = nullptr;
MachineInstr *BlendMI = nullptr;
+ bool BlendIsMaskedBlend = false;
for (MachineInstr &CurMI : llvm::make_range(
std::next(MachineBasicBlock::iterator(MI)), MBB.end())) {
@@ -431,13 +447,23 @@ static bool tryCompressMaskProducer(MachineInstr &MI, MachineBasicBlock &MBB,
KMovMI = &CurMI;
// continue scanning to ensure
// there are no *other* uses of the mask later in the block.
- } else if (!IsSignMaskCmp && isCompressibleBlendVUse(BlendOpc, UseOpc) &&
- CurMI.getOperand(2).getReg() == MaskReg &&
- !usesExtendedRegister(CurMI) &&
- checkPredicate(BlendOpc, &ST)) {
- BlendMI = &CurMI;
} else {
- return false;
+ bool IsMaskedMove =
+ !IsSignMaskCmp && isCompressibleBlendVUse(BlendOpc, UseOpc);
+ bool IsMaskedBlend =
+ !IsSignMaskCmp && isCompressibleMaskedBlendUse(BlendOpc, UseOpc);
+
+ if (!IsMaskedMove && !IsMaskedBlend)
+ return false;
+
+ unsigned MaskOpIdx = IsMaskedBlend ? 1 : 2;
+ if (CurMI.getOperand(MaskOpIdx).getReg() == MaskReg &&
+ !usesExtendedRegister(CurMI) && checkPredicate(BlendOpc, &ST)) {
+ BlendMI = &CurMI;
+ BlendIsMaskedBlend = IsMaskedBlend;
+ } else {
+ return false;
+ }
}
}
@@ -504,12 +530,12 @@ static bool tryCompressMaskProducer(MachineInstr &MI, MachineBasicBlock &MBB,
} else if (BlendMI) {
const MachineOperand &MaskVec = MI.getOperand(1);
const MachineOperand &Dst = BlendMI->getOperand(0);
- const MachineOperand &Passthru = BlendMI->getOperand(1);
+ const MachineOperand &Passthru =
+ BlendMI->getOperand(BlendIsMaskedBlend ? 2 : 1);
const MachineOperand &Src = BlendMI->getOperand(3);
// Build a replacement instead of changing BlendMI in place because
- // VMOV*rrk has a tied passthrough operand and a different operand order
- // than VBLENDV.
+ // masked VMOV and VPBLENDM have different operand layouts from VBLENDV.
auto MIB =
BuildMI(MBB, *BlendMI, BlendMI->getDebugLoc(), TII->get(BlendOpc))
.addReg(Dst.getReg(), getRegState(Dst))
diff --git a/llvm/test/CodeGen/X86/avx512-regcall-NoMask.ll b/llvm/test/CodeGen/X86/avx512-regcall-NoMask.ll
index a664cc7f17a5c..a72f18c086255 100644
--- a/llvm/test/CodeGen/X86/avx512-regcall-NoMask.ll
+++ b/llvm/test/CodeGen/X86/avx512-regcall-NoMask.ll
@@ -666,22 +666,19 @@ define dso_local x86_regcallcc <4 x i32> @test_argRet128Vector(<4 x i1> %x, <4 x
; X32-LABEL: test_argRet128Vector:
; X32: # %bb.0:
; X32-NEXT: vpslld $31, %xmm0, %xmm0
-; X32-NEXT: vpmovd2m %xmm0, %k1
-; X32-NEXT: vpblendmd %xmm1, %xmm2, %xmm0 {%k1}
+; X32-NEXT: vblendvps %xmm0, %xmm1, %xmm2, %xmm0
; X32-NEXT: retl
;
; WIN64-LABEL: test_argRet128Vector:
; WIN64: # %bb.0:
; WIN64-NEXT: vpslld $31, %xmm0, %xmm0
-; WIN64-NEXT: vpmovd2m %xmm0, %k1
-; WIN64-NEXT: vpblendmd %xmm1, %xmm2, %xmm0 {%k1}
+; WIN64-NEXT: vblendvps %xmm0, %xmm1, %xmm2, %xmm0
; WIN64-NEXT: retq
;
; LINUXOSX64-LABEL: test_argRet128Vector:
; LINUXOSX64: # %bb.0:
; LINUXOSX64-NEXT: vpslld $31, %xmm0, %xmm0
-; LINUXOSX64-NEXT: vpmovd2m %xmm0, %k1
-; LINUXOSX64-NEXT: vpblendmd %xmm1, %xmm2, %xmm0 {%k1}
+; LINUXOSX64-NEXT: vblendvps %xmm0, %xmm1, %xmm2, %xmm0
; LINUXOSX64-NEXT: retq
%d = select <4 x i1> %x, <4 x i32> %a, <4 x i32> %b
ret <4 x i32> %d
diff --git a/llvm/test/CodeGen/X86/compress-evex-vpmov-blendv.mir b/llvm/test/CodeGen/X86/compress-evex-vpmov-blendv.mir
index 3949fdbd8423f..67c1cc55fd24a 100644
--- a/llvm/test/CodeGen/X86/compress-evex-vpmov-blendv.mir
+++ b/llvm/test/CodeGen/X86/compress-evex-vpmov-blendv.mir
@@ -78,6 +78,22 @@ body: |
$k1 = VPMOVB2MZ256kr $ymm0
$ymm1 = VMOVDQU8Z256rrk $ymm1, $k1, $ymm2
+ ; CHECK: $xmm1 = VBLENDVPSrrr $xmm2, $xmm3, $xmm0
+ $k1 = VPMOVD2MZ128kr $xmm0
+ $xmm1 = VPBLENDMDZ128rrk $k1, $xmm2, $xmm3
+
+ ; CHECK: $ymm1 = VBLENDVPSYrrr $ymm2, $ymm3, $ymm0
+ $k1 = VPMOVD2MZ256kr $ymm0
+ $ymm1 = VPBLENDMDZ256rrk $k1, $ymm2, $ymm3
+
+ ; CHECK: $xmm1 = VBLENDVPDrrr $xmm2, $xmm3, $xmm0
+ $k1 = VPMOVQ2MZ128kr $xmm0
+ $xmm1 = VPBLENDMQZ128rrk $k1, $xmm2, $xmm3
+
+ ; CHECK: $ymm1 = VBLENDVPDYrrr $ymm2, $ymm3, $ymm0
+ $k1 = VPMOVQ2MZ256kr $ymm0
+ $ymm1 = VPBLENDMQZ256rrk $k1, $ymm2, $ymm3
+
RET64
...
@@ -138,3 +154,18 @@ body: |
$xmm16 = VMOVAPSZ128rrk $xmm16, $k1, $xmm2
RET64
...
+
+---
+name: blendv_incompatible_mask_width
+body: |
+ bb.0:
+ ; A dword mask extracted from XMM cannot directly become the vector mask
+ ; for a 256-bit qword VBLENDVPD.
+ ; CHECK-LABEL: name: blendv_incompatible_mask_width
+ ; CHECK: $k1 = VPMOVD2MZ128kr $xmm0
+ ; CHECK: $ymm1 = VPBLENDMQZ256rrk $k1, $ymm2, $ymm3
+ ; CHECK-NOT: VBLENDVPDYrrr
+ $k1 = VPMOVD2MZ128kr $xmm0
+ $ymm1 = VPBLENDMQZ256rrk $k1, $ymm2, $ymm3
+ RET64
+...
diff --git a/llvm/test/CodeGen/X86/compress-evex-vpmov-kill.mir b/llvm/test/CodeGen/X86/compress-evex-vpmov-kill.mir
index bb6cb7e9c42d4..86ae8c1379a8d 100644
--- a/llvm/test/CodeGen/X86/compress-evex-vpmov-kill.mir
+++ b/llvm/test/CodeGen/X86/compress-evex-vpmov-kill.mir
@@ -65,3 +65,33 @@ body: |
$xmm1 = VMOVAPSZ128rrk killed $xmm1, killed $k1, killed $xmm2
RET64 implicit killed $xmm1
...
+
+---
+name: masked_blend_live_mask_vec
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $xmm0, $xmm1, $xmm2
+ ; $xmm0 is used after the fold, so it must not be killed.
+ ; CHECK-LABEL: name: masked_blend_live_mask_vec
+ ; CHECK: $xmm1 = VBLENDVPSrrr killed $xmm2, killed $xmm1, $xmm0
+ ; CHECK-NOT: killed $xmm0
+ $k1 = VPMOVD2MZ128kr $xmm0
+ $xmm1 = VPBLENDMDZ128rrk killed $k1, killed $xmm2, killed $xmm1
+ $xmm3 = VMOVAPSrr $xmm0
+ RET64 implicit killed $xmm1, implicit killed $xmm3
+...
+
+---
+name: masked_blend_dead_mask_vec
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $xmm0, $xmm1, $xmm2
+ ; $xmm0 is dead after the fold, so the kill is preserved.
+ ; CHECK-LABEL: name: masked_blend_dead_mask_vec
+ ; CHECK: $xmm1 = VBLENDVPSrrr killed $xmm2, killed $xmm1, killed $xmm0
+ $k1 = VPMOVD2MZ128kr killed $xmm0
+ $xmm1 = VPBLENDMDZ128rrk killed $k1, killed $xmm2, killed $xmm1
+ RET64 implicit killed $xmm1
+...
More information about the llvm-commits
mailing list