[llvm] [X86][APX] Combine CMOV chain to CCMP+CMOV (PR #207929)
Phoebe Wang via llvm-commits
llvm-commits at lists.llvm.org
Tue Jul 7 01:50:33 PDT 2026
https://github.com/phoebewang updated https://github.com/llvm/llvm-project/pull/207929
>From b8a73ca4f1a59be3c83c2b1e25b1e86ada4bd064 Mon Sep 17 00:00:00 2001
From: Phoebe Wang <phoebe.wang at intel.com>
Date: Tue, 7 Jul 2026 01:34:56 -0700
Subject: [PATCH 1/3] [X86][APX] Combine CMOV chain to CCMP+CMOV
Fixes: #207886
---
llvm/lib/Target/X86/X86ISelLowering.cpp | 147 ++++++++++++++++++++++++
llvm/test/CodeGen/X86/apx/ccmp.ll | 81 +++++++++++++
2 files changed, 228 insertions(+)
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 899b7645b4c14..011a8feea6096 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -50314,6 +50314,153 @@ static SDValue combineCMov(SDNode *N, SelectionDAG &DAG,
}
}
+ // With CCMP, fold chained CMOVs representing AND/OR conditions.
+ if (Subtarget.hasCCMP() && Cond.getOpcode() == X86ISD::SUB) {
+ // AND: CMOV(F, CMOV(F, T, cc1, f1), cc2, f2)
+ // -> CMOV(F, T, cc1, CCMP(f1_args, dcf_~cc1, cc2, f2))
+ // This is: (cc2 && cc1) ? T : F.
+ // CCMP executes f1 when cc2 is true; when cc2 is false, DCF forces cc1 false.
+ if (TrueOp.getOpcode() == X86ISD::CMOV && TrueOp.hasOneUse()) {
+ SDValue InnerFalse = TrueOp.getOperand(0);
+ SDValue InnerTrue = TrueOp.getOperand(1);
+ X86::CondCode InnerCC =
+ (X86::CondCode)TrueOp.getConstantOperandVal(2);
+ SDValue InnerCond = TrueOp.getOperand(3);
+
+ if (InnerFalse == FalseOp && InnerCond.getOpcode() == X86ISD::SUB &&
+ CC != X86::COND_P && CC != X86::COND_NP &&
+ InnerCC != X86::COND_P && InnerCC != X86::COND_NP) {
+ // Check if FalseOp is itself a CMOV with the same TrueOp: this is the
+ // (cc_or || (cc2 && cc1)) pattern. Fold both levels in one step to
+ // avoid a second combine pass.
+ if (FalseOp.getOpcode() == X86ISD::CMOV &&
+ FalseOp.getOperand(1) == InnerTrue) {
+ SDValue OrFalse = FalseOp.getOperand(0);
+ X86::CondCode OrCC = (X86::CondCode)FalseOp.getConstantOperandVal(2);
+ SDValue OrCond = FalseOp.getOperand(3);
+ if (OrCond.getOpcode() == X86ISD::SUB &&
+ OrCC != X86::COND_P && OrCC != X86::COND_NP) {
+ X86::CondCode NotInnerCC = X86::GetOppositeBranchCondition(InnerCC);
+ // ccmp1 (AND gate): fire InnerCond when CC true; DCF forces InnerCC false.
+ SDValue CFlags1 = DAG.getTargetConstant(
+ X86::getCCMPCondFlagsFromCondCode(NotInnerCC), DL, MVT::i8);
+ SDValue SrcCC1 = DAG.getTargetConstant(CC, DL, MVT::i8);
+ SDValue CCMP1 = DAG.getNode(X86ISD::CCMP, DL, MVT::i32,
+ {InnerCond.getOperand(0), InnerCond.getOperand(1),
+ CFlags1, SrcCC1, Cond.getValue(1)});
+ // ccmp2 (OR gate): fire OrCond when ~InnerCC; DCF forces OrCC true.
+ SDValue CFlags2 = DAG.getTargetConstant(
+ X86::getCCMPCondFlagsFromCondCode(OrCC), DL, MVT::i8);
+ SDValue SrcCC2 = DAG.getTargetConstant(NotInnerCC, DL, MVT::i8);
+ SDValue CCMP2 = DAG.getNode(X86ISD::CCMP, DL, MVT::i32,
+ {OrCond.getOperand(0), OrCond.getOperand(1),
+ CFlags2, SrcCC2, CCMP1});
+ SDValue OrCCVal = DAG.getTargetConstant(OrCC, DL, MVT::i8);
+ return DAG.getNode(X86ISD::CMOV, DL, VT,
+ {OrFalse, InnerTrue, OrCCVal, CCMP2});
+ }
+ }
+
+ // Before the 2-level AND fold, check if this node is the FalseOp of a
+ // parent CMOV whose condition is the swapped form of CC (e.g. L and G).
+ // If so, apply the 3-level fold instead to avoid producing a CCMP
+ // inner-cond that blocks the subsequent OR fold.
+ if (N->hasOneUse()) {
+ SDNode *ParentN = (*N->use_begin()).getUser();
+ if (ParentN->getOpcode() == X86ISD::CMOV &&
+ ParentN->getOperand(0) == SDValue(N, 0)) {
+ SDValue ParentTrue = ParentN->getOperand(1);
+ X86::CondCode ParentCC =
+ (X86::CondCode)ParentN->getConstantOperandVal(2);
+ SDValue ParentCond = ParentN->getOperand(3);
+ // Swapped-operands pairs: L↔G, LE↔GE, B↔A, BE↔AE.
+ auto GetSwappedCC = [](X86::CondCode C) -> X86::CondCode {
+ switch (C) {
+ case X86::COND_L: return X86::COND_G;
+ case X86::COND_G: return X86::COND_L;
+ case X86::COND_LE: return X86::COND_GE;
+ case X86::COND_GE: return X86::COND_LE;
+ case X86::COND_B: return X86::COND_A;
+ case X86::COND_A: return X86::COND_B;
+ case X86::COND_BE: return X86::COND_AE;
+ case X86::COND_AE: return X86::COND_BE;
+ default: return X86::COND_INVALID;
+ }
+ };
+ if (ParentTrue == InnerTrue &&
+ ParentCond.getOpcode() == X86ISD::SUB &&
+ ParentCC != X86::COND_P && ParentCC != X86::COND_NP &&
+ GetSwappedCC(CC) == ParentCC) {
+ X86::CondCode NotCC = X86::GetOppositeBranchCondition(CC);
+ // ccmp1 (AND gate): fire f2 when cc1 true; DCF forces ~cc2 false.
+ SDValue CFlags1 = DAG.getTargetConstant(
+ X86::getCCMPCondFlagsFromCondCode(NotCC), DL, MVT::i8);
+ SDValue SrcCC1 = DAG.getTargetConstant(InnerCC, DL, MVT::i8);
+ SDValue CCMP1 = DAG.getNode(X86ISD::CCMP, DL, MVT::i32,
+ {Cond.getOperand(0), Cond.getOperand(1),
+ CFlags1, SrcCC1, InnerCond.getValue(1)});
+ // ccmp2 (OR gate): fire f3 swapped when ~cc2; DCF forces cc2 true.
+ SDValue CFlags2 = DAG.getTargetConstant(
+ X86::getCCMPCondFlagsFromCondCode(CC), DL, MVT::i8);
+ SDValue SrcCC2 = DAG.getTargetConstant(NotCC, DL, MVT::i8);
+ SDValue CCMP2 = DAG.getNode(X86ISD::CCMP, DL, MVT::i32,
+ {ParentCond.getOperand(1), ParentCond.getOperand(0),
+ CFlags2, SrcCC2, CCMP1});
+ SDValue CCVal = DAG.getTargetConstant(CC, DL, MVT::i8);
+ SDValue Result = DAG.getNode(X86ISD::CMOV, DL,
+ ParentN->getValueType(0),
+ {FalseOp, InnerTrue, CCVal, CCMP2});
+ DCI.CombineTo(ParentN, Result);
+ return SDValue(N, 0);
+ }
+ }
+ }
+
+ X86::CondCode DCFCode = X86::GetOppositeBranchCondition(InnerCC);
+ SDValue CFlags = DAG.getTargetConstant(
+ X86::getCCMPCondFlagsFromCondCode(DCFCode), DL, MVT::i8);
+ SDValue SrcCC = DAG.getTargetConstant(CC, DL, MVT::i8);
+ SDValue CCMPNode = DAG.getNode(
+ X86ISD::CCMP, DL, MVT::i32,
+ {InnerCond.getOperand(0), InnerCond.getOperand(1), CFlags, SrcCC,
+ Cond.getValue(1)});
+ SDValue InnerCCVal = DAG.getTargetConstant(InnerCC, DL, MVT::i8);
+ return DAG.getNode(X86ISD::CMOV, DL, VT,
+ {FalseOp, InnerTrue, InnerCCVal, CCMPNode});
+ }
+ }
+
+ // OR: CMOV(CMOV(X, T, cc_in, sub_in), T, cc_out, f_out)
+ // -> CMOV(X, T, cc_in, CCMP(sub_in_args, dcf_cc_in, ~cc_out, f_out))
+ // This is: (cc_out || cc_in) ? T : X.
+ // CCMP fires when cc_out is false, re-evaluating sub_in; when cc_out is
+ // true, DCF forces cc_in true.
+ if (FalseOp.getOpcode() == X86ISD::CMOV && FalseOp.hasOneUse()) {
+ SDValue InnerFalse = FalseOp.getOperand(0);
+ SDValue InnerTrue = FalseOp.getOperand(1);
+ X86::CondCode InnerCC =
+ (X86::CondCode)FalseOp.getConstantOperandVal(2);
+ SDValue InnerCond = FalseOp.getOperand(3);
+
+ if (InnerTrue == TrueOp && InnerCond.getOpcode() == X86ISD::SUB &&
+ CC != X86::COND_P && CC != X86::COND_NP &&
+ InnerCC != X86::COND_P && InnerCC != X86::COND_NP) {
+ SDValue CFlags = DAG.getTargetConstant(
+ X86::getCCMPCondFlagsFromCondCode(InnerCC), DL, MVT::i8);
+ X86::CondCode NotCC = X86::GetOppositeBranchCondition(CC);
+ SDValue SrcCC = DAG.getTargetConstant(NotCC, DL, MVT::i8);
+ SDValue CCMPNode = DAG.getNode(
+ X86ISD::CCMP, DL, MVT::i32,
+ {InnerCond.getOperand(0), InnerCond.getOperand(1), CFlags, SrcCC,
+ Cond.getValue(1)});
+ SDValue InnerCCVal = DAG.getTargetConstant(InnerCC, DL, MVT::i8);
+ return DAG.getNode(X86ISD::CMOV, DL, VT,
+ {InnerFalse, TrueOp, InnerCCVal, CCMPNode});
+ }
+ }
+ }
+
+
// Fold (CMOV C1, (ADD (CTTZ X), C2), (X != 0)) ->
// (ADD (CMOV C1-C2, (CTTZ X), (X != 0)), C2)
// Or (CMOV (ADD (CTTZ X), C2), C1, (X == 0)) ->
diff --git a/llvm/test/CodeGen/X86/apx/ccmp.ll b/llvm/test/CodeGen/X86/apx/ccmp.ll
index 94cbb7786721c..ae407df8009e8 100644
--- a/llvm/test/CodeGen/X86/apx/ccmp.ll
+++ b/llvm/test/CodeGen/X86/apx/ccmp.ll
@@ -2098,5 +2098,86 @@ define i32 @test_or_fp_int(double %a, double %b, i32 %c, i32 %d) {
ret i32 %ext
}
+; (b != d) && (a < c): AND of two comparisons - fold chained CMOV into CCMP+CMOV
+define i32 @ccmp_cmov_and(i32 %a, i32 %b, i32 %c, i32 %d) {
+; CHECK-LABEL: ccmp_cmov_and:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: movl %edi, %eax # encoding: [0x89,0xf8]
+; CHECK-NEXT: cmpl %ecx, %esi # encoding: [0x39,0xce]
+; CHECK-NEXT: ccmpnel {dfv=} %edx, %edi # encoding: [0x62,0xf4,0x04,0x05,0x39,0xd7]
+; CHECK-NEXT: cmovll %esi, %eax # encoding: [0x0f,0x4c,0xc6]
+; CHECK-NEXT: retq # encoding: [0xc3]
+;
+; NDD-LABEL: ccmp_cmov_and:
+; NDD: # %bb.0: # %entry
+; NDD-NEXT: cmpl %ecx, %esi # encoding: [0x39,0xce]
+; NDD-NEXT: ccmpnel {dfv=} %edx, %edi # encoding: [0x62,0xf4,0x04,0x05,0x39,0xd7]
+; NDD-NEXT: cmovll %esi, %edi, %eax # encoding: [0x62,0xf4,0x7c,0x18,0x4c,0xfe]
+; NDD-NEXT: retq # encoding: [0xc3]
+entry:
+ %cmp1 = icmp ne i32 %b, %d
+ %cmp2 = icmp slt i32 %a, %c
+ %and = and i1 %cmp1, %cmp2
+ %sel = select i1 %and, i32 %b, i32 %a
+ ret i32 %sel
+}
+
+; (b != d && a < c) || (a > d): AND+OR - fold chained CMOVs into CCMP+CCMP+CMOV
+define i32 @ccmp_cmov_and_or(i32 %a, i32 %b, i32 %c, i32 %d) {
+; CHECK-LABEL: ccmp_cmov_and_or:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: movl %edi, %eax # encoding: [0x89,0xf8]
+; CHECK-NEXT: cmpl %ecx, %esi # encoding: [0x39,0xce]
+; CHECK-NEXT: ccmpnel {dfv=} %edx, %edi # encoding: [0x62,0xf4,0x04,0x05,0x39,0xd7]
+; CHECK-NEXT: ccmpgel {dfv=} %ecx, %edi # encoding: [0x62,0xf4,0x04,0x0d,0x39,0xcf]
+; CHECK-NEXT: cmovgl %esi, %eax # encoding: [0x0f,0x4f,0xc6]
+; CHECK-NEXT: retq # encoding: [0xc3]
+;
+; NDD-LABEL: ccmp_cmov_and_or:
+; NDD: # %bb.0: # %entry
+; NDD-NEXT: cmpl %ecx, %esi # encoding: [0x39,0xce]
+; NDD-NEXT: ccmpnel {dfv=} %edx, %edi # encoding: [0x62,0xf4,0x04,0x05,0x39,0xd7]
+; NDD-NEXT: ccmpgel {dfv=} %ecx, %edi # encoding: [0x62,0xf4,0x04,0x0d,0x39,0xcf]
+; NDD-NEXT: cmovgl %esi, %edi, %eax # encoding: [0x62,0xf4,0x7c,0x18,0x4f,0xfe]
+; NDD-NEXT: retq # encoding: [0xc3]
+entry:
+ %cmp1 = icmp ne i32 %b, %d
+ %cmp2 = icmp slt i32 %a, %c
+ %and = and i1 %cmp1, %cmp2
+ %cmp3 = icmp sgt i32 %a, %d
+ %or = or i1 %and, %cmp3
+ %sel = select i1 %or, i32 %b, i32 %a
+ ret i32 %sel
+}
+
+; (a > d) || (a < c && b != d): AND+OR with outer OR first — 3-level CMOV chain
+; Emits 2 CCMPs with swapped operands on the OR comparison (same as GCC).
+define i32 @ccmp_cmov_and_or_c(i32 %a, i32 %b, i32 %c, i32 %d) {
+; CHECK-LABEL: ccmp_cmov_and_or_c:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: movl %edi, %eax # encoding: [0x89,0xf8]
+; CHECK-NEXT: cmpl %ecx, %esi # encoding: [0x39,0xce]
+; CHECK-NEXT: ccmpnel {dfv=} %edx, %edi # encoding: [0x62,0xf4,0x04,0x05,0x39,0xd7]
+; CHECK-NEXT: ccmpgel {dfv=sf} %edi, %ecx # encoding: [0x62,0xf4,0x24,0x0d,0x39,0xf9]
+; CHECK-NEXT: cmovll %esi, %eax # encoding: [0x0f,0x4c,0xc6]
+; CHECK-NEXT: retq # encoding: [0xc3]
+;
+; NDD-LABEL: ccmp_cmov_and_or_c:
+; NDD: # %bb.0: # %entry
+; NDD-NEXT: cmpl %ecx, %esi # encoding: [0x39,0xce]
+; NDD-NEXT: ccmpnel {dfv=} %edx, %edi # encoding: [0x62,0xf4,0x04,0x05,0x39,0xd7]
+; NDD-NEXT: ccmpgel {dfv=sf} %edi, %ecx # encoding: [0x62,0xf4,0x24,0x0d,0x39,0xf9]
+; NDD-NEXT: cmovll %esi, %edi, %eax # encoding: [0x62,0xf4,0x7c,0x18,0x4c,0xfe]
+; NDD-NEXT: retq # encoding: [0xc3]
+entry:
+ %cmp1 = icmp ne i32 %b, %d
+ %cmp2 = icmp slt i32 %a, %c
+ %and = and i1 %cmp2, %cmp1
+ %cmp3 = icmp sgt i32 %a, %d
+ %or = or i1 %cmp3, %and
+ %sel = select i1 %or, i32 %b, i32 %a
+ ret i32 %sel
+}
+
declare dso_local void @foo(...)
declare {i64, i1} @llvm.ssub.with.overflow.i64(i64, i64) nounwind readnone
>From d9ebbd0b1dd070e4815d2810992d690d64d07c76 Mon Sep 17 00:00:00 2001
From: Phoebe Wang <phoebe.wang at intel.com>
Date: Tue, 7 Jul 2026 01:45:39 -0700
Subject: [PATCH 2/3] clang-format
---
llvm/lib/Target/X86/X86ISelLowering.cpp | 103 +++++++++++++-----------
1 file changed, 58 insertions(+), 45 deletions(-)
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 011a8feea6096..d0d42fb4d44a2 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -50319,17 +50319,17 @@ static SDValue combineCMov(SDNode *N, SelectionDAG &DAG,
// AND: CMOV(F, CMOV(F, T, cc1, f1), cc2, f2)
// -> CMOV(F, T, cc1, CCMP(f1_args, dcf_~cc1, cc2, f2))
// This is: (cc2 && cc1) ? T : F.
- // CCMP executes f1 when cc2 is true; when cc2 is false, DCF forces cc1 false.
+ // CCMP executes f1 when cc2 is true; when cc2 is false, DCF forces cc1
+ // false.
if (TrueOp.getOpcode() == X86ISD::CMOV && TrueOp.hasOneUse()) {
SDValue InnerFalse = TrueOp.getOperand(0);
SDValue InnerTrue = TrueOp.getOperand(1);
- X86::CondCode InnerCC =
- (X86::CondCode)TrueOp.getConstantOperandVal(2);
+ X86::CondCode InnerCC = (X86::CondCode)TrueOp.getConstantOperandVal(2);
SDValue InnerCond = TrueOp.getOperand(3);
if (InnerFalse == FalseOp && InnerCond.getOpcode() == X86ISD::SUB &&
- CC != X86::COND_P && CC != X86::COND_NP &&
- InnerCC != X86::COND_P && InnerCC != X86::COND_NP) {
+ CC != X86::COND_P && CC != X86::COND_NP && InnerCC != X86::COND_P &&
+ InnerCC != X86::COND_NP) {
// Check if FalseOp is itself a CMOV with the same TrueOp: this is the
// (cc_or || (cc2 && cc1)) pattern. Fold both levels in one step to
// avoid a second combine pass.
@@ -50338,23 +50338,26 @@ static SDValue combineCMov(SDNode *N, SelectionDAG &DAG,
SDValue OrFalse = FalseOp.getOperand(0);
X86::CondCode OrCC = (X86::CondCode)FalseOp.getConstantOperandVal(2);
SDValue OrCond = FalseOp.getOperand(3);
- if (OrCond.getOpcode() == X86ISD::SUB &&
- OrCC != X86::COND_P && OrCC != X86::COND_NP) {
+ if (OrCond.getOpcode() == X86ISD::SUB && OrCC != X86::COND_P &&
+ OrCC != X86::COND_NP) {
X86::CondCode NotInnerCC = X86::GetOppositeBranchCondition(InnerCC);
- // ccmp1 (AND gate): fire InnerCond when CC true; DCF forces InnerCC false.
+ // ccmp1 (AND gate): fire InnerCond when CC true; DCF forces InnerCC
+ // false.
SDValue CFlags1 = DAG.getTargetConstant(
X86::getCCMPCondFlagsFromCondCode(NotInnerCC), DL, MVT::i8);
SDValue SrcCC1 = DAG.getTargetConstant(CC, DL, MVT::i8);
- SDValue CCMP1 = DAG.getNode(X86ISD::CCMP, DL, MVT::i32,
- {InnerCond.getOperand(0), InnerCond.getOperand(1),
- CFlags1, SrcCC1, Cond.getValue(1)});
+ SDValue CCMP1 =
+ DAG.getNode(X86ISD::CCMP, DL, MVT::i32,
+ {InnerCond.getOperand(0), InnerCond.getOperand(1),
+ CFlags1, SrcCC1, Cond.getValue(1)});
// ccmp2 (OR gate): fire OrCond when ~InnerCC; DCF forces OrCC true.
SDValue CFlags2 = DAG.getTargetConstant(
X86::getCCMPCondFlagsFromCondCode(OrCC), DL, MVT::i8);
SDValue SrcCC2 = DAG.getTargetConstant(NotInnerCC, DL, MVT::i8);
- SDValue CCMP2 = DAG.getNode(X86ISD::CCMP, DL, MVT::i32,
- {OrCond.getOperand(0), OrCond.getOperand(1),
- CFlags2, SrcCC2, CCMP1});
+ SDValue CCMP2 =
+ DAG.getNode(X86ISD::CCMP, DL, MVT::i32,
+ {OrCond.getOperand(0), OrCond.getOperand(1),
+ CFlags2, SrcCC2, CCMP1});
SDValue OrCCVal = DAG.getTargetConstant(OrCC, DL, MVT::i8);
return DAG.getNode(X86ISD::CMOV, DL, VT,
{OrFalse, InnerTrue, OrCCVal, CCMP2});
@@ -50376,15 +50379,24 @@ static SDValue combineCMov(SDNode *N, SelectionDAG &DAG,
// Swapped-operands pairs: L↔G, LE↔GE, B↔A, BE↔AE.
auto GetSwappedCC = [](X86::CondCode C) -> X86::CondCode {
switch (C) {
- case X86::COND_L: return X86::COND_G;
- case X86::COND_G: return X86::COND_L;
- case X86::COND_LE: return X86::COND_GE;
- case X86::COND_GE: return X86::COND_LE;
- case X86::COND_B: return X86::COND_A;
- case X86::COND_A: return X86::COND_B;
- case X86::COND_BE: return X86::COND_AE;
- case X86::COND_AE: return X86::COND_BE;
- default: return X86::COND_INVALID;
+ case X86::COND_L:
+ return X86::COND_G;
+ case X86::COND_G:
+ return X86::COND_L;
+ case X86::COND_LE:
+ return X86::COND_GE;
+ case X86::COND_GE:
+ return X86::COND_LE;
+ case X86::COND_B:
+ return X86::COND_A;
+ case X86::COND_A:
+ return X86::COND_B;
+ case X86::COND_BE:
+ return X86::COND_AE;
+ case X86::COND_AE:
+ return X86::COND_BE;
+ default:
+ return X86::COND_INVALID;
}
};
if (ParentTrue == InnerTrue &&
@@ -50396,20 +50408,23 @@ static SDValue combineCMov(SDNode *N, SelectionDAG &DAG,
SDValue CFlags1 = DAG.getTargetConstant(
X86::getCCMPCondFlagsFromCondCode(NotCC), DL, MVT::i8);
SDValue SrcCC1 = DAG.getTargetConstant(InnerCC, DL, MVT::i8);
- SDValue CCMP1 = DAG.getNode(X86ISD::CCMP, DL, MVT::i32,
- {Cond.getOperand(0), Cond.getOperand(1),
- CFlags1, SrcCC1, InnerCond.getValue(1)});
- // ccmp2 (OR gate): fire f3 swapped when ~cc2; DCF forces cc2 true.
+ SDValue CCMP1 =
+ DAG.getNode(X86ISD::CCMP, DL, MVT::i32,
+ {Cond.getOperand(0), Cond.getOperand(1), CFlags1,
+ SrcCC1, InnerCond.getValue(1)});
+ // ccmp2 (OR gate): fire f3 swapped when ~cc2; DCF forces cc2
+ // true.
SDValue CFlags2 = DAG.getTargetConstant(
X86::getCCMPCondFlagsFromCondCode(CC), DL, MVT::i8);
SDValue SrcCC2 = DAG.getTargetConstant(NotCC, DL, MVT::i8);
SDValue CCMP2 = DAG.getNode(X86ISD::CCMP, DL, MVT::i32,
- {ParentCond.getOperand(1), ParentCond.getOperand(0),
- CFlags2, SrcCC2, CCMP1});
+ {ParentCond.getOperand(1),
+ ParentCond.getOperand(0), CFlags2,
+ SrcCC2, CCMP1});
SDValue CCVal = DAG.getTargetConstant(CC, DL, MVT::i8);
- SDValue Result = DAG.getNode(X86ISD::CMOV, DL,
- ParentN->getValueType(0),
- {FalseOp, InnerTrue, CCVal, CCMP2});
+ SDValue Result =
+ DAG.getNode(X86ISD::CMOV, DL, ParentN->getValueType(0),
+ {FalseOp, InnerTrue, CCVal, CCMP2});
DCI.CombineTo(ParentN, Result);
return SDValue(N, 0);
}
@@ -50420,10 +50435,10 @@ static SDValue combineCMov(SDNode *N, SelectionDAG &DAG,
SDValue CFlags = DAG.getTargetConstant(
X86::getCCMPCondFlagsFromCondCode(DCFCode), DL, MVT::i8);
SDValue SrcCC = DAG.getTargetConstant(CC, DL, MVT::i8);
- SDValue CCMPNode = DAG.getNode(
- X86ISD::CCMP, DL, MVT::i32,
- {InnerCond.getOperand(0), InnerCond.getOperand(1), CFlags, SrcCC,
- Cond.getValue(1)});
+ SDValue CCMPNode =
+ DAG.getNode(X86ISD::CCMP, DL, MVT::i32,
+ {InnerCond.getOperand(0), InnerCond.getOperand(1),
+ CFlags, SrcCC, Cond.getValue(1)});
SDValue InnerCCVal = DAG.getTargetConstant(InnerCC, DL, MVT::i8);
return DAG.getNode(X86ISD::CMOV, DL, VT,
{FalseOp, InnerTrue, InnerCCVal, CCMPNode});
@@ -50438,21 +50453,20 @@ static SDValue combineCMov(SDNode *N, SelectionDAG &DAG,
if (FalseOp.getOpcode() == X86ISD::CMOV && FalseOp.hasOneUse()) {
SDValue InnerFalse = FalseOp.getOperand(0);
SDValue InnerTrue = FalseOp.getOperand(1);
- X86::CondCode InnerCC =
- (X86::CondCode)FalseOp.getConstantOperandVal(2);
+ X86::CondCode InnerCC = (X86::CondCode)FalseOp.getConstantOperandVal(2);
SDValue InnerCond = FalseOp.getOperand(3);
if (InnerTrue == TrueOp && InnerCond.getOpcode() == X86ISD::SUB &&
- CC != X86::COND_P && CC != X86::COND_NP &&
- InnerCC != X86::COND_P && InnerCC != X86::COND_NP) {
+ CC != X86::COND_P && CC != X86::COND_NP && InnerCC != X86::COND_P &&
+ InnerCC != X86::COND_NP) {
SDValue CFlags = DAG.getTargetConstant(
X86::getCCMPCondFlagsFromCondCode(InnerCC), DL, MVT::i8);
X86::CondCode NotCC = X86::GetOppositeBranchCondition(CC);
SDValue SrcCC = DAG.getTargetConstant(NotCC, DL, MVT::i8);
- SDValue CCMPNode = DAG.getNode(
- X86ISD::CCMP, DL, MVT::i32,
- {InnerCond.getOperand(0), InnerCond.getOperand(1), CFlags, SrcCC,
- Cond.getValue(1)});
+ SDValue CCMPNode =
+ DAG.getNode(X86ISD::CCMP, DL, MVT::i32,
+ {InnerCond.getOperand(0), InnerCond.getOperand(1),
+ CFlags, SrcCC, Cond.getValue(1)});
SDValue InnerCCVal = DAG.getTargetConstant(InnerCC, DL, MVT::i8);
return DAG.getNode(X86ISD::CMOV, DL, VT,
{InnerFalse, TrueOp, InnerCCVal, CCMPNode});
@@ -50460,7 +50474,6 @@ static SDValue combineCMov(SDNode *N, SelectionDAG &DAG,
}
}
-
// Fold (CMOV C1, (ADD (CTTZ X), C2), (X != 0)) ->
// (ADD (CMOV C1-C2, (CTTZ X), (X != 0)), C2)
// Or (CMOV (ADD (CTTZ X), C2), C1, (X == 0)) ->
>From 6ae9d34e2da3af38399cbf603085bc23669c3472 Mon Sep 17 00:00:00 2001
From: Phoebe Wang <phoebe.wang at intel.com>
Date: Tue, 7 Jul 2026 01:50:18 -0700
Subject: [PATCH 3/3] Reduce comments
---
llvm/lib/Target/X86/X86ISelLowering.cpp | 22 ++--------------------
1 file changed, 2 insertions(+), 20 deletions(-)
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index d0d42fb4d44a2..c51c2f2385a7d 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -50318,9 +50318,6 @@ static SDValue combineCMov(SDNode *N, SelectionDAG &DAG,
if (Subtarget.hasCCMP() && Cond.getOpcode() == X86ISD::SUB) {
// AND: CMOV(F, CMOV(F, T, cc1, f1), cc2, f2)
// -> CMOV(F, T, cc1, CCMP(f1_args, dcf_~cc1, cc2, f2))
- // This is: (cc2 && cc1) ? T : F.
- // CCMP executes f1 when cc2 is true; when cc2 is false, DCF forces cc1
- // false.
if (TrueOp.getOpcode() == X86ISD::CMOV && TrueOp.hasOneUse()) {
SDValue InnerFalse = TrueOp.getOperand(0);
SDValue InnerTrue = TrueOp.getOperand(1);
@@ -50330,9 +50327,6 @@ static SDValue combineCMov(SDNode *N, SelectionDAG &DAG,
if (InnerFalse == FalseOp && InnerCond.getOpcode() == X86ISD::SUB &&
CC != X86::COND_P && CC != X86::COND_NP && InnerCC != X86::COND_P &&
InnerCC != X86::COND_NP) {
- // Check if FalseOp is itself a CMOV with the same TrueOp: this is the
- // (cc_or || (cc2 && cc1)) pattern. Fold both levels in one step to
- // avoid a second combine pass.
if (FalseOp.getOpcode() == X86ISD::CMOV &&
FalseOp.getOperand(1) == InnerTrue) {
SDValue OrFalse = FalseOp.getOperand(0);
@@ -50341,8 +50335,6 @@ static SDValue combineCMov(SDNode *N, SelectionDAG &DAG,
if (OrCond.getOpcode() == X86ISD::SUB && OrCC != X86::COND_P &&
OrCC != X86::COND_NP) {
X86::CondCode NotInnerCC = X86::GetOppositeBranchCondition(InnerCC);
- // ccmp1 (AND gate): fire InnerCond when CC true; DCF forces InnerCC
- // false.
SDValue CFlags1 = DAG.getTargetConstant(
X86::getCCMPCondFlagsFromCondCode(NotInnerCC), DL, MVT::i8);
SDValue SrcCC1 = DAG.getTargetConstant(CC, DL, MVT::i8);
@@ -50350,7 +50342,6 @@ static SDValue combineCMov(SDNode *N, SelectionDAG &DAG,
DAG.getNode(X86ISD::CCMP, DL, MVT::i32,
{InnerCond.getOperand(0), InnerCond.getOperand(1),
CFlags1, SrcCC1, Cond.getValue(1)});
- // ccmp2 (OR gate): fire OrCond when ~InnerCC; DCF forces OrCC true.
SDValue CFlags2 = DAG.getTargetConstant(
X86::getCCMPCondFlagsFromCondCode(OrCC), DL, MVT::i8);
SDValue SrcCC2 = DAG.getTargetConstant(NotInnerCC, DL, MVT::i8);
@@ -50364,10 +50355,8 @@ static SDValue combineCMov(SDNode *N, SelectionDAG &DAG,
}
}
- // Before the 2-level AND fold, check if this node is the FalseOp of a
- // parent CMOV whose condition is the swapped form of CC (e.g. L and G).
- // If so, apply the 3-level fold instead to avoid producing a CCMP
- // inner-cond that blocks the subsequent OR fold.
+ // cc3 = GetSwappedCC(cc2) means cc3(x,y) == cc2(y,x); handle
+ // or(cc3, and(cc2, cc1)) by inspecting the parent CMOV.
if (N->hasOneUse()) {
SDNode *ParentN = (*N->use_begin()).getUser();
if (ParentN->getOpcode() == X86ISD::CMOV &&
@@ -50376,7 +50365,6 @@ static SDValue combineCMov(SDNode *N, SelectionDAG &DAG,
X86::CondCode ParentCC =
(X86::CondCode)ParentN->getConstantOperandVal(2);
SDValue ParentCond = ParentN->getOperand(3);
- // Swapped-operands pairs: L↔G, LE↔GE, B↔A, BE↔AE.
auto GetSwappedCC = [](X86::CondCode C) -> X86::CondCode {
switch (C) {
case X86::COND_L:
@@ -50404,7 +50392,6 @@ static SDValue combineCMov(SDNode *N, SelectionDAG &DAG,
ParentCC != X86::COND_P && ParentCC != X86::COND_NP &&
GetSwappedCC(CC) == ParentCC) {
X86::CondCode NotCC = X86::GetOppositeBranchCondition(CC);
- // ccmp1 (AND gate): fire f2 when cc1 true; DCF forces ~cc2 false.
SDValue CFlags1 = DAG.getTargetConstant(
X86::getCCMPCondFlagsFromCondCode(NotCC), DL, MVT::i8);
SDValue SrcCC1 = DAG.getTargetConstant(InnerCC, DL, MVT::i8);
@@ -50412,8 +50399,6 @@ static SDValue combineCMov(SDNode *N, SelectionDAG &DAG,
DAG.getNode(X86ISD::CCMP, DL, MVT::i32,
{Cond.getOperand(0), Cond.getOperand(1), CFlags1,
SrcCC1, InnerCond.getValue(1)});
- // ccmp2 (OR gate): fire f3 swapped when ~cc2; DCF forces cc2
- // true.
SDValue CFlags2 = DAG.getTargetConstant(
X86::getCCMPCondFlagsFromCondCode(CC), DL, MVT::i8);
SDValue SrcCC2 = DAG.getTargetConstant(NotCC, DL, MVT::i8);
@@ -50447,9 +50432,6 @@ static SDValue combineCMov(SDNode *N, SelectionDAG &DAG,
// OR: CMOV(CMOV(X, T, cc_in, sub_in), T, cc_out, f_out)
// -> CMOV(X, T, cc_in, CCMP(sub_in_args, dcf_cc_in, ~cc_out, f_out))
- // This is: (cc_out || cc_in) ? T : X.
- // CCMP fires when cc_out is false, re-evaluating sub_in; when cc_out is
- // true, DCF forces cc_in true.
if (FalseOp.getOpcode() == X86ISD::CMOV && FalseOp.hasOneUse()) {
SDValue InnerFalse = FalseOp.getOperand(0);
SDValue InnerTrue = FalseOp.getOperand(1);
More information about the llvm-commits
mailing list