[llvm] [X86] Handle multi-use setcc in commuteSelect (PR #213645)
Timur Golubovich via llvm-commits
llvm-commits at lists.llvm.org
Tue Aug 4 07:33:29 PDT 2026
https://github.com/timurgol007 updated https://github.com/llvm/llvm-project/pull/213645
>From 3e9bfe66a78a26dc2eff75256a5cef8ed60a3aa6 Mon Sep 17 00:00:00 2001
From: Timur Golubovich <timur.golubovich at intel.com>
Date: Sat, 1 Aug 2026 20:35:06 +0200
Subject: [PATCH 1/3] [X86] Handle multi-use setcc in commuteSelect
When a setcc condition is shared between multiple vselects,
commuteSelect previously bailed out due to the m_OneUse constraint.
Extend it to check all users of the setcc and, if they all benefit
from commuting, invert the condition once with ReplaceAllUsesOfValueWith
and swap all vselect operands.
This fixes a write-mask fusion regression where masked min/max
reductions produced unfused vminps + vmovaps {%k} instead of a single
write-masked vminps {%k}.
---
llvm/lib/Target/X86/X86ISelLowering.cpp | 35 +++++++++++++++----
.../CodeGen/X86/avx512-masked-op-fusion.ll | 18 ++++------
2 files changed, 36 insertions(+), 17 deletions(-)
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index dfda1157a720e..fc6dcce98d289 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -48564,21 +48564,44 @@ static SDValue commuteSelect(SDNode *N, SelectionDAG &DAG, const SDLoc &DL,
ISD::CondCode CC;
SDValue Cond, X, Y, LHS, RHS;
- if (!sd_match(N, m_VSelect(m_AllOf(m_Value(Cond),
- m_OneUse(m_SetCC(m_Value(X), m_Value(Y),
- m_CondCode(CC)))),
- m_Value(LHS), m_Value(RHS))))
+ if (!sd_match(
+ N, m_VSelect(m_AllOf(m_Value(Cond),
+ m_SetCC(m_Value(X), m_Value(Y), m_CondCode(CC))),
+ m_Value(LHS), m_Value(RHS))))
return SDValue();
if (canCombineAsMaskOperation(LHS, Subtarget) ||
!canCombineAsMaskOperation(RHS, Subtarget))
return SDValue();
+ // For multi-use setcc, check that all users are vselects that benefit.
+ if (!Cond.hasOneUse()) {
+ SDValue UserLHS, UserRHS;
+ for (SDNode *User : Cond->users()) {
+ if (!sd_match(User, m_VSelect(m_Specific(Cond), m_Value(UserLHS),
+ m_Value(UserRHS))))
+ return SDValue();
+ if (canCombineAsMaskOperation(UserLHS, Subtarget) ||
+ !canCombineAsMaskOperation(UserRHS, Subtarget))
+ return SDValue();
+ }
+ }
+
// Commute LHS and RHS to create opportunity to select mask instruction.
// (vselect M, L, R) -> (vselect ~M, R, L)
ISD::CondCode NewCC = ISD::getSetCCInverse(CC, X.getValueType());
- Cond = DAG.getSetCC(SDLoc(Cond), Cond.getValueType(), X, Y, NewCC);
- return DAG.getSelect(DL, LHS.getValueType(), Cond, RHS, LHS);
+ SDValue NewCond = DAG.getSetCC(SDLoc(Cond), Cond.getValueType(), X, Y, NewCC);
+ if (Cond.hasOneUse())
+ return DAG.getSelect(DL, LHS.getValueType(), NewCond, RHS, LHS);
+
+ // Invert the setcc for all users and commute all vselects.
+ DAG.ReplaceAllUsesOfValueWith(Cond, NewCond);
+ for (SDNode *User : NewCond->users()) {
+ SDValue UserLHS = User->getOperand(1);
+ SDValue UserRHS = User->getOperand(2);
+ DAG.UpdateNodeOperands(User, NewCond, UserRHS, UserLHS);
+ }
+ return SDValue(N, 0);
}
/// Do target-specific dag combines on SELECT and VSELECT nodes.
diff --git a/llvm/test/CodeGen/X86/avx512-masked-op-fusion.ll b/llvm/test/CodeGen/X86/avx512-masked-op-fusion.ll
index 9280366c78b9c..a9af94c92a9f1 100644
--- a/llvm/test/CodeGen/X86/avx512-masked-op-fusion.ll
+++ b/llvm/test/CodeGen/X86/avx512-masked-op-fusion.ll
@@ -14,21 +14,17 @@ define void @masked_min_max(ptr %pSrc, ptr %pMsk, i64 %n, ptr %pMin, ptr %pMax)
; CHECK-NEXT: .p2align 4
; CHECK-NEXT: .LBB0_1: # %loop
; CHECK-NEXT: # =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: vmovaps %zmm1, %zmm2
-; CHECK-NEXT: vmovaps %zmm0, %zmm1
-; CHECK-NEXT: vmovdqu (%rsi,%rax), %xmm0
-; CHECK-NEXT: vptestnmb %xmm0, %xmm0, %k1
-; CHECK-NEXT: vmovups (%rdi,%rax,4), %zmm3
-; CHECK-NEXT: vminps %zmm3, %zmm1, %zmm0
-; CHECK-NEXT: vmovaps %zmm1, %zmm0 {%k1}
-; CHECK-NEXT: vmaxps %zmm3, %zmm2, %zmm1
-; CHECK-NEXT: vmovaps %zmm2, %zmm1 {%k1}
+; CHECK-NEXT: vmovdqu (%rsi,%rax), %xmm2
+; CHECK-NEXT: vptestmb %xmm2, %xmm2, %k1
+; CHECK-NEXT: vmovups (%rdi,%rax,4), %zmm2
+; CHECK-NEXT: vminps %zmm2, %zmm1, %zmm1 {%k1}
+; CHECK-NEXT: vmaxps %zmm2, %zmm0, %zmm0 {%k1}
; CHECK-NEXT: addq $16, %rax
; CHECK-NEXT: cmpq %rdx, %rax
; CHECK-NEXT: jb .LBB0_1
; CHECK-NEXT: # %bb.2: # %exit
-; CHECK-NEXT: vmovaps %zmm0, (%rcx)
-; CHECK-NEXT: vmovaps %zmm1, (%r8)
+; CHECK-NEXT: vmovaps %zmm1, (%rcx)
+; CHECK-NEXT: vmovaps %zmm0, (%r8)
; CHECK-NEXT: vzeroupper
; CHECK-NEXT: retq
entry:
>From d452906cbb0d0970aa2641b747290bf9867b9f0e Mon Sep 17 00:00:00 2001
From: Timur Golubovich <timur.golubovich at intel.com>
Date: Mon, 3 Aug 2026 15:48:25 +0200
Subject: [PATCH 2/3] added all_of algorithm
---
llvm/lib/Target/X86/X86ISelLowering.cpp | 17 ++++++++---------
.../test/CodeGen/X86/avx512-masked-op-fusion.ll | 4 ++--
2 files changed, 10 insertions(+), 11 deletions(-)
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index fc6dcce98d289..7922948bdbcaf 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -48576,15 +48576,14 @@ static SDValue commuteSelect(SDNode *N, SelectionDAG &DAG, const SDLoc &DL,
// For multi-use setcc, check that all users are vselects that benefit.
if (!Cond.hasOneUse()) {
- SDValue UserLHS, UserRHS;
- for (SDNode *User : Cond->users()) {
- if (!sd_match(User, m_VSelect(m_Specific(Cond), m_Value(UserLHS),
- m_Value(UserRHS))))
- return SDValue();
- if (canCombineAsMaskOperation(UserLHS, Subtarget) ||
- !canCombineAsMaskOperation(UserRHS, Subtarget))
- return SDValue();
- }
+ if (!llvm::all_of(Cond->users(), [&](SDNode *User) {
+ SDValue UserLHS, UserRHS;
+ return sd_match(User, m_VSelect(m_Specific(Cond), m_Value(UserLHS),
+ m_Value(UserRHS))) &&
+ !canCombineAsMaskOperation(UserLHS, Subtarget) &&
+ canCombineAsMaskOperation(UserRHS, Subtarget);
+ }))
+ return SDValue();
}
// Commute LHS and RHS to create opportunity to select mask instruction.
diff --git a/llvm/test/CodeGen/X86/avx512-masked-op-fusion.ll b/llvm/test/CodeGen/X86/avx512-masked-op-fusion.ll
index a9af94c92a9f1..b11ad0df8bbe8 100644
--- a/llvm/test/CodeGen/X86/avx512-masked-op-fusion.ll
+++ b/llvm/test/CodeGen/X86/avx512-masked-op-fusion.ll
@@ -8,8 +8,8 @@
define void @masked_min_max(ptr %pSrc, ptr %pMsk, i64 %n, ptr %pMin, ptr %pMax) {
; CHECK-LABEL: masked_min_max:
; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: vbroadcastss {{.*#+}} zmm1 = [-Inf,-Inf,-Inf,-Inf,-Inf,-Inf,-Inf,-Inf,-Inf,-Inf,-Inf,-Inf,-Inf,-Inf,-Inf,-Inf]
-; CHECK-NEXT: vbroadcastss {{.*#+}} zmm0 = [+Inf,+Inf,+Inf,+Inf,+Inf,+Inf,+Inf,+Inf,+Inf,+Inf,+Inf,+Inf,+Inf,+Inf,+Inf,+Inf]
+; CHECK-NEXT: vbroadcastss {{.*#+}} zmm0 = [-Inf,-Inf,-Inf,-Inf,-Inf,-Inf,-Inf,-Inf,-Inf,-Inf,-Inf,-Inf,-Inf,-Inf,-Inf,-Inf]
+; CHECK-NEXT: vbroadcastss {{.*#+}} zmm1 = [+Inf,+Inf,+Inf,+Inf,+Inf,+Inf,+Inf,+Inf,+Inf,+Inf,+Inf,+Inf,+Inf,+Inf,+Inf,+Inf]
; CHECK-NEXT: xorl %eax, %eax
; CHECK-NEXT: .p2align 4
; CHECK-NEXT: .LBB0_1: # %loop
>From 2fd3d84865b4979c3add9a7d09e290d6157ecdab Mon Sep 17 00:00:00 2001
From: Timur Golubovich <timur.golubovich at intel.com>
Date: Tue, 4 Aug 2026 15:36:47 +0200
Subject: [PATCH 3/3] reused a variable
---
llvm/lib/Target/X86/X86ISelLowering.cpp | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 7922948bdbcaf..3dba45d0af32a 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -48575,7 +48575,8 @@ static SDValue commuteSelect(SDNode *N, SelectionDAG &DAG, const SDLoc &DL,
return SDValue();
// For multi-use setcc, check that all users are vselects that benefit.
- if (!Cond.hasOneUse()) {
+ bool CondHasOneUse = Cond.hasOneUse();
+ if (!CondHasOneUse) {
if (!llvm::all_of(Cond->users(), [&](SDNode *User) {
SDValue UserLHS, UserRHS;
return sd_match(User, m_VSelect(m_Specific(Cond), m_Value(UserLHS),
@@ -48590,7 +48591,7 @@ static SDValue commuteSelect(SDNode *N, SelectionDAG &DAG, const SDLoc &DL,
// (vselect M, L, R) -> (vselect ~M, R, L)
ISD::CondCode NewCC = ISD::getSetCCInverse(CC, X.getValueType());
SDValue NewCond = DAG.getSetCC(SDLoc(Cond), Cond.getValueType(), X, Y, NewCC);
- if (Cond.hasOneUse())
+ if (CondHasOneUse)
return DAG.getSelect(DL, LHS.getValueType(), NewCond, RHS, LHS);
// Invert the setcc for all users and commute all vselects.
More information about the llvm-commits
mailing list