[llvm] [X86] Handle multi-use setcc in commuteSelect (PR #213645)
Timur Golubovich via llvm-commits
llvm-commits at lists.llvm.org
Mon Aug 3 06:48:40 PDT 2026
https://github.com/timurgol007 updated https://github.com/llvm/llvm-project/pull/213645
>From 814d3868f8d0fddac1fdc0ec6e2a5b6729efa3f8 Mon Sep 17 00:00:00 2001
From: Timur Golubovich <timur.golubovich at intel.com>
Date: Sat, 1 Aug 2026 20:35:06 +0200
Subject: [PATCH 1/2] [X86] Handle multi-use setcc in commuteSelect
When a setcc condition is shared between multiple vselects,
commuteSelect previously bailed out due to the m_OneUse constraint.
Extend it to check all users of the setcc and, if they all benefit
from commuting, invert the condition once with ReplaceAllUsesOfValueWith
and swap all vselect operands.
This fixes a write-mask fusion regression where masked min/max
reductions produced unfused vminps + vmovaps {%k} instead of a single
write-masked vminps {%k}.
---
llvm/lib/Target/X86/X86ISelLowering.cpp | 35 ++++++++++--
.../CodeGen/X86/avx512-masked-op-fusion.ll | 57 +++++++++++++++++++
2 files changed, 86 insertions(+), 6 deletions(-)
create mode 100644 llvm/test/CodeGen/X86/avx512-masked-op-fusion.ll
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index dfda1157a720e..fc6dcce98d289 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -48564,21 +48564,44 @@ static SDValue commuteSelect(SDNode *N, SelectionDAG &DAG, const SDLoc &DL,
ISD::CondCode CC;
SDValue Cond, X, Y, LHS, RHS;
- if (!sd_match(N, m_VSelect(m_AllOf(m_Value(Cond),
- m_OneUse(m_SetCC(m_Value(X), m_Value(Y),
- m_CondCode(CC)))),
- m_Value(LHS), m_Value(RHS))))
+ if (!sd_match(
+ N, m_VSelect(m_AllOf(m_Value(Cond),
+ m_SetCC(m_Value(X), m_Value(Y), m_CondCode(CC))),
+ m_Value(LHS), m_Value(RHS))))
return SDValue();
if (canCombineAsMaskOperation(LHS, Subtarget) ||
!canCombineAsMaskOperation(RHS, Subtarget))
return SDValue();
+ // For multi-use setcc, check that all users are vselects that benefit.
+ if (!Cond.hasOneUse()) {
+ SDValue UserLHS, UserRHS;
+ for (SDNode *User : Cond->users()) {
+ if (!sd_match(User, m_VSelect(m_Specific(Cond), m_Value(UserLHS),
+ m_Value(UserRHS))))
+ return SDValue();
+ if (canCombineAsMaskOperation(UserLHS, Subtarget) ||
+ !canCombineAsMaskOperation(UserRHS, Subtarget))
+ return SDValue();
+ }
+ }
+
// Commute LHS and RHS to create opportunity to select mask instruction.
// (vselect M, L, R) -> (vselect ~M, R, L)
ISD::CondCode NewCC = ISD::getSetCCInverse(CC, X.getValueType());
- Cond = DAG.getSetCC(SDLoc(Cond), Cond.getValueType(), X, Y, NewCC);
- return DAG.getSelect(DL, LHS.getValueType(), Cond, RHS, LHS);
+ SDValue NewCond = DAG.getSetCC(SDLoc(Cond), Cond.getValueType(), X, Y, NewCC);
+ if (Cond.hasOneUse())
+ return DAG.getSelect(DL, LHS.getValueType(), NewCond, RHS, LHS);
+
+ // Invert the setcc for all users and commute all vselects.
+ DAG.ReplaceAllUsesOfValueWith(Cond, NewCond);
+ for (SDNode *User : NewCond->users()) {
+ SDValue UserLHS = User->getOperand(1);
+ SDValue UserRHS = User->getOperand(2);
+ DAG.UpdateNodeOperands(User, NewCond, UserRHS, UserLHS);
+ }
+ return SDValue(N, 0);
}
/// Do target-specific dag combines on SELECT and VSELECT nodes.
diff --git a/llvm/test/CodeGen/X86/avx512-masked-op-fusion.ll b/llvm/test/CodeGen/X86/avx512-masked-op-fusion.ll
new file mode 100644
index 0000000000000..8bf8a83b90ee0
--- /dev/null
+++ b/llvm/test/CodeGen/X86/avx512-masked-op-fusion.ll
@@ -0,0 +1,57 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mcpu=skylake-avx512 | FileCheck %s
+
+; Verify that commuteSelect handles multi-use setcc conditions shared between
+; min and max vselects. The setcc should be inverted once and both selects
+; commuted, enabling ISel to emit fused write-masked vminps/vmaxps {%k}.
+
+define void @masked_min_max(ptr %pSrc, ptr %pMsk, i64 %n, ptr %pMin, ptr %pMax) {
+; CHECK-LABEL: masked_min_max:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: vbroadcastss {{.*#+}} zmm0 = [-Inf,-Inf,-Inf,-Inf,-Inf,-Inf,-Inf,-Inf,-Inf,-Inf,-Inf,-Inf,-Inf,-Inf,-Inf,-Inf]
+; CHECK-NEXT: vbroadcastss {{.*#+}} zmm1 = [+Inf,+Inf,+Inf,+Inf,+Inf,+Inf,+Inf,+Inf,+Inf,+Inf,+Inf,+Inf,+Inf,+Inf,+Inf,+Inf]
+; CHECK-NEXT: xorl %eax, %eax
+; CHECK-NEXT: .p2align 4
+; CHECK-NEXT: .LBB0_1: # %loop
+; CHECK-NEXT: # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: vmovdqu (%rsi,%rax), %xmm2
+; CHECK-NEXT: vptestmb %xmm2, %xmm2, %k1
+; CHECK-NEXT: vmovups (%rdi,%rax,4), %zmm2
+; CHECK-NEXT: vminps %zmm2, %zmm1, %zmm1 {%k1}
+; CHECK-NEXT: vmaxps %zmm2, %zmm0, %zmm0 {%k1}
+; CHECK-NEXT: addq $16, %rax
+; CHECK-NEXT: cmpq %rdx, %rax
+; CHECK-NEXT: jb .LBB0_1
+; CHECK-NEXT: # %bb.2: # %exit
+; CHECK-NEXT: vmovaps %zmm1, (%rcx)
+; CHECK-NEXT: vmovaps %zmm0, (%r8)
+; CHECK-NEXT: vzeroupper
+; CHECK-NEXT: retq
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %acc_min = phi <16 x float> [ splat (float 0x7FF0000000000000), %entry ], [ %res_min, %loop ]
+ %acc_max = phi <16 x float> [ splat (float 0xFFF0000000000000), %entry ], [ %res_max, %loop ]
+ %msk_ptr = getelementptr inbounds i8, ptr %pMsk, i64 %iv
+ %msk_bytes = load <16 x i8>, ptr %msk_ptr, align 1
+ %cmp = icmp eq <16 x i8> %msk_bytes, zeroinitializer
+ %src_ptr = getelementptr inbounds float, ptr %pSrc, i64 %iv
+ %src = load <16 x float>, ptr %src_ptr, align 1
+ %min = tail call <16 x float> @llvm.x86.avx512.min.ps.512(<16 x float> %acc_min, <16 x float> %src, i32 4)
+ %res_min = select <16 x i1> %cmp, <16 x float> %acc_min, <16 x float> %min
+ %max = tail call <16 x float> @llvm.x86.avx512.max.ps.512(<16 x float> %acc_max, <16 x float> %src, i32 4)
+ %res_max = select <16 x i1> %cmp, <16 x float> %acc_max, <16 x float> %max
+ %iv.next = add nuw nsw i64 %iv, 16
+ %done = icmp uge i64 %iv.next, %n
+ br i1 %done, label %exit, label %loop
+
+exit:
+ store <16 x float> %res_min, ptr %pMin, align 64
+ store <16 x float> %res_max, ptr %pMax, align 64
+ ret void
+}
+
+declare <16 x float> @llvm.x86.avx512.min.ps.512(<16 x float>, <16 x float>, i32)
+declare <16 x float> @llvm.x86.avx512.max.ps.512(<16 x float>, <16 x float>, i32)
>From 0ddf95c37d5862f2594b5289d37ebe780f11651e Mon Sep 17 00:00:00 2001
From: Timur Golubovich <timur.golubovich at intel.com>
Date: Mon, 3 Aug 2026 15:48:25 +0200
Subject: [PATCH 2/2] added all_of algorithm
---
llvm/lib/Target/X86/X86ISelLowering.cpp | 17 ++++++++---------
1 file changed, 8 insertions(+), 9 deletions(-)
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index fc6dcce98d289..7922948bdbcaf 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -48576,15 +48576,14 @@ static SDValue commuteSelect(SDNode *N, SelectionDAG &DAG, const SDLoc &DL,
// For multi-use setcc, check that all users are vselects that benefit.
if (!Cond.hasOneUse()) {
- SDValue UserLHS, UserRHS;
- for (SDNode *User : Cond->users()) {
- if (!sd_match(User, m_VSelect(m_Specific(Cond), m_Value(UserLHS),
- m_Value(UserRHS))))
- return SDValue();
- if (canCombineAsMaskOperation(UserLHS, Subtarget) ||
- !canCombineAsMaskOperation(UserRHS, Subtarget))
- return SDValue();
- }
+ if (!llvm::all_of(Cond->users(), [&](SDNode *User) {
+ SDValue UserLHS, UserRHS;
+ return sd_match(User, m_VSelect(m_Specific(Cond), m_Value(UserLHS),
+ m_Value(UserRHS))) &&
+ !canCombineAsMaskOperation(UserLHS, Subtarget) &&
+ canCombineAsMaskOperation(UserRHS, Subtarget);
+ }))
+ return SDValue();
}
// Commute LHS and RHS to create opportunity to select mask instruction.
More information about the llvm-commits
mailing list