[llvm] [X86] Handle multi-use setcc in commuteSelect (PR #213645)
via llvm-commits
llvm-commits at lists.llvm.org
Mon Aug 3 04:33:46 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-x86
Author: Timur Golubovich (timurgol007)
<details>
<summary>Changes</summary>
When a setcc condition is shared between multiple vselects, commuteSelect previously bailed out due to the m_OneUse constraint. Extend it to check all users of the setcc and, if they all benefit from commuting, invert the condition once with ReplaceAllUsesOfValueWith and swap all vselect operands.
This fixes a write-mask fusion regression where masked min/max reductions produced unfused vminps + vmovaps {%k} instead of a single write-masked vminps {%k}.
---
Full diff: https://github.com/llvm/llvm-project/pull/213645.diff
2 Files Affected:
- (modified) llvm/lib/Target/X86/X86ISelLowering.cpp (+29-6)
- (added) llvm/test/CodeGen/X86/avx512-masked-op-fusion.ll (+57)
``````````diff
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index dfda1157a720e..fc6dcce98d289 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -48564,21 +48564,44 @@ static SDValue commuteSelect(SDNode *N, SelectionDAG &DAG, const SDLoc &DL,
ISD::CondCode CC;
SDValue Cond, X, Y, LHS, RHS;
- if (!sd_match(N, m_VSelect(m_AllOf(m_Value(Cond),
- m_OneUse(m_SetCC(m_Value(X), m_Value(Y),
- m_CondCode(CC)))),
- m_Value(LHS), m_Value(RHS))))
+ if (!sd_match(
+ N, m_VSelect(m_AllOf(m_Value(Cond),
+ m_SetCC(m_Value(X), m_Value(Y), m_CondCode(CC))),
+ m_Value(LHS), m_Value(RHS))))
return SDValue();
if (canCombineAsMaskOperation(LHS, Subtarget) ||
!canCombineAsMaskOperation(RHS, Subtarget))
return SDValue();
+ // For multi-use setcc, check that all users are vselects that benefit.
+ if (!Cond.hasOneUse()) {
+ SDValue UserLHS, UserRHS;
+ for (SDNode *User : Cond->users()) {
+ if (!sd_match(User, m_VSelect(m_Specific(Cond), m_Value(UserLHS),
+ m_Value(UserRHS))))
+ return SDValue();
+ if (canCombineAsMaskOperation(UserLHS, Subtarget) ||
+ !canCombineAsMaskOperation(UserRHS, Subtarget))
+ return SDValue();
+ }
+ }
+
// Commute LHS and RHS to create opportunity to select mask instruction.
// (vselect M, L, R) -> (vselect ~M, R, L)
ISD::CondCode NewCC = ISD::getSetCCInverse(CC, X.getValueType());
- Cond = DAG.getSetCC(SDLoc(Cond), Cond.getValueType(), X, Y, NewCC);
- return DAG.getSelect(DL, LHS.getValueType(), Cond, RHS, LHS);
+ SDValue NewCond = DAG.getSetCC(SDLoc(Cond), Cond.getValueType(), X, Y, NewCC);
+ if (Cond.hasOneUse())
+ return DAG.getSelect(DL, LHS.getValueType(), NewCond, RHS, LHS);
+
+ // Invert the setcc for all users and commute all vselects.
+ DAG.ReplaceAllUsesOfValueWith(Cond, NewCond);
+ for (SDNode *User : NewCond->users()) {
+ SDValue UserLHS = User->getOperand(1);
+ SDValue UserRHS = User->getOperand(2);
+ DAG.UpdateNodeOperands(User, NewCond, UserRHS, UserLHS);
+ }
+ return SDValue(N, 0);
}
/// Do target-specific dag combines on SELECT and VSELECT nodes.
diff --git a/llvm/test/CodeGen/X86/avx512-masked-op-fusion.ll b/llvm/test/CodeGen/X86/avx512-masked-op-fusion.ll
new file mode 100644
index 0000000000000..8bf8a83b90ee0
--- /dev/null
+++ b/llvm/test/CodeGen/X86/avx512-masked-op-fusion.ll
@@ -0,0 +1,57 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mcpu=skylake-avx512 | FileCheck %s
+
+; Verify that commuteSelect handles multi-use setcc conditions shared between
+; min and max vselects. The setcc should be inverted once and both selects
+; commuted, enabling ISel to emit fused write-masked vminps/vmaxps {%k}.
+
+define void @masked_min_max(ptr %pSrc, ptr %pMsk, i64 %n, ptr %pMin, ptr %pMax) {
+; CHECK-LABEL: masked_min_max:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: vbroadcastss {{.*#+}} zmm0 = [-Inf,-Inf,-Inf,-Inf,-Inf,-Inf,-Inf,-Inf,-Inf,-Inf,-Inf,-Inf,-Inf,-Inf,-Inf,-Inf]
+; CHECK-NEXT: vbroadcastss {{.*#+}} zmm1 = [+Inf,+Inf,+Inf,+Inf,+Inf,+Inf,+Inf,+Inf,+Inf,+Inf,+Inf,+Inf,+Inf,+Inf,+Inf,+Inf]
+; CHECK-NEXT: xorl %eax, %eax
+; CHECK-NEXT: .p2align 4
+; CHECK-NEXT: .LBB0_1: # %loop
+; CHECK-NEXT: # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: vmovdqu (%rsi,%rax), %xmm2
+; CHECK-NEXT: vptestmb %xmm2, %xmm2, %k1
+; CHECK-NEXT: vmovups (%rdi,%rax,4), %zmm2
+; CHECK-NEXT: vminps %zmm2, %zmm1, %zmm1 {%k1}
+; CHECK-NEXT: vmaxps %zmm2, %zmm0, %zmm0 {%k1}
+; CHECK-NEXT: addq $16, %rax
+; CHECK-NEXT: cmpq %rdx, %rax
+; CHECK-NEXT: jb .LBB0_1
+; CHECK-NEXT: # %bb.2: # %exit
+; CHECK-NEXT: vmovaps %zmm1, (%rcx)
+; CHECK-NEXT: vmovaps %zmm0, (%r8)
+; CHECK-NEXT: vzeroupper
+; CHECK-NEXT: retq
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %acc_min = phi <16 x float> [ splat (float 0x7FF0000000000000), %entry ], [ %res_min, %loop ]
+ %acc_max = phi <16 x float> [ splat (float 0xFFF0000000000000), %entry ], [ %res_max, %loop ]
+ %msk_ptr = getelementptr inbounds i8, ptr %pMsk, i64 %iv
+ %msk_bytes = load <16 x i8>, ptr %msk_ptr, align 1
+ %cmp = icmp eq <16 x i8> %msk_bytes, zeroinitializer
+ %src_ptr = getelementptr inbounds float, ptr %pSrc, i64 %iv
+ %src = load <16 x float>, ptr %src_ptr, align 1
+ %min = tail call <16 x float> @llvm.x86.avx512.min.ps.512(<16 x float> %acc_min, <16 x float> %src, i32 4)
+ %res_min = select <16 x i1> %cmp, <16 x float> %acc_min, <16 x float> %min
+ %max = tail call <16 x float> @llvm.x86.avx512.max.ps.512(<16 x float> %acc_max, <16 x float> %src, i32 4)
+ %res_max = select <16 x i1> %cmp, <16 x float> %acc_max, <16 x float> %max
+ %iv.next = add nuw nsw i64 %iv, 16
+ %done = icmp uge i64 %iv.next, %n
+ br i1 %done, label %exit, label %loop
+
+exit:
+ store <16 x float> %res_min, ptr %pMin, align 64
+ store <16 x float> %res_max, ptr %pMax, align 64
+ ret void
+}
+
+declare <16 x float> @llvm.x86.avx512.min.ps.512(<16 x float>, <16 x float>, i32)
+declare <16 x float> @llvm.x86.avx512.max.ps.512(<16 x float>, <16 x float>, i32)
``````````
</details>
https://github.com/llvm/llvm-project/pull/213645
More information about the llvm-commits
mailing list