[llvm] [X86] Preserve node flags when combineCMov rebuilds a CMOV (PR #223469)
Yangyu Chen via llvm-commits
llvm-commits at lists.llvm.org
Mon Sep 14 23:00:34 PDT 2026
https://github.com/cyyself updated https://github.com/llvm/llvm-project/pull/223469
>From 51c8fc1038cbf1c829008fe150180276a7d12672 Mon Sep 17 00:00:00 2001
From: Yangyu Chen <cyy at cyyself.name>
Date: Tue, 15 Sep 2026 03:05:58 +0800
Subject: [PATCH] [X86] Preserve node flags when combineCMov rebuilds a CMOV
X86CmovConversion skips CMOVs that carry the unpredictable flag, which
InstrEmitter sets from the SDNode flags of a select marked !unpredictable.
Three rewrites in combineCMov (simplified EFLAGS, `select (x == c), c, e` to
`select (x == c), x, e`, and the and/or-of-setcc double CMOV) create the new
CMOV node without the original flags, so the hint is lost and a CMOV with a
folded load is still turned into a branch. For example, with -O3:
struct S { int a, b, c, d; };
void f(S& s) { s.b = __builtin_unpredictable(s.a) ? s.c : 0; } // branch
Pass N->getFlags() through, so the CMOV keeps the hint and the load stays
folded into it.
Assisted-by: Claude Code:claude-fable-5-1
Signed-off-by: Yangyu Chen <cyy at cyyself.name>
---
llvm/lib/Target/X86/X86ISelLowering.cpp | 14 +++--
.../CodeGen/X86/cmov-unpredictable-flag.ll | 55 +++++++++++++++++++
.../CodeGen/X86/cmov-unpredictable-mem.ll | 41 ++++++++++++++
3 files changed, 104 insertions(+), 6 deletions(-)
create mode 100644 llvm/test/CodeGen/X86/cmov-unpredictable-flag.ll
create mode 100644 llvm/test/CodeGen/X86/cmov-unpredictable-mem.ll
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 775fd7642f040..dc95338dffc07 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -50295,7 +50295,7 @@ static SDValue combineCMov(SDNode *N, SelectionDAG &DAG,
!Subtarget.canUseCMOV() || hasFPCMov(CC)) {
SDValue Ops[] = {FalseOp, TrueOp, DAG.getTargetConstant(CC, DL, MVT::i8),
Flags};
- return DAG.getNode(X86ISD::CMOV, DL, VT, Ops);
+ return DAG.getNode(X86ISD::CMOV, DL, VT, Ops, N->getFlags());
}
}
@@ -50415,7 +50415,7 @@ static SDValue combineCMov(SDNode *N, SelectionDAG &DAG,
if (CC == X86::COND_E && CmpAgainst == dyn_cast<ConstantSDNode>(TrueOp)) {
SDValue Ops[] = {FalseOp, Cond.getOperand(0),
DAG.getTargetConstant(CC, DL, MVT::i8), Cond};
- return DAG.getNode(X86ISD::CMOV, DL, VT, Ops);
+ return DAG.getNode(X86ISD::CMOV, DL, VT, Ops, N->getFlags());
}
}
}
@@ -50475,10 +50475,10 @@ static SDValue combineCMov(SDNode *N, SelectionDAG &DAG,
SDValue LOps[] = {FalseOp, TrueOp,
DAG.getTargetConstant(CC0, DL, MVT::i8), Flags};
- SDValue LCMOV = DAG.getNode(X86ISD::CMOV, DL, VT, LOps);
+ SDValue LCMOV = DAG.getNode(X86ISD::CMOV, DL, VT, LOps, N->getFlags());
SDValue Ops[] = {LCMOV, TrueOp, DAG.getTargetConstant(CC1, DL, MVT::i8),
Flags};
- SDValue CMOV = DAG.getNode(X86ISD::CMOV, DL, VT, Ops);
+ SDValue CMOV = DAG.getNode(X86ISD::CMOV, DL, VT, Ops, N->getFlags());
return CMOV;
}
}
@@ -50519,8 +50519,10 @@ static SDValue combineCMov(SDNode *N, SelectionDAG &DAG,
// This should constant fold.
SDValue Diff = DAG.getNode(ISD::SUB, DL, VT, Const, Add.getOperand(1));
SDValue CMov =
- DAG.getNode(X86ISD::CMOV, DL, VT, Diff, Add.getOperand(0),
- DAG.getTargetConstant(X86::COND_NE, DL, MVT::i8), Cond);
+ DAG.getNode(X86ISD::CMOV, DL, VT,
+ {Diff, Add.getOperand(0),
+ DAG.getTargetConstant(X86::COND_NE, DL, MVT::i8), Cond},
+ N->getFlags());
return DAG.getNode(ISD::ADD, DL, VT, CMov, Add.getOperand(1));
}
}
diff --git a/llvm/test/CodeGen/X86/cmov-unpredictable-flag.ll b/llvm/test/CodeGen/X86/cmov-unpredictable-flag.ll
new file mode 100644
index 0000000000000..52f2d73753915
--- /dev/null
+++ b/llvm/test/CodeGen/X86/cmov-unpredictable-flag.ll
@@ -0,0 +1,55 @@
+; RUN: llc < %s -mtriple=x86_64 -stop-before=x86-cmov-conversion | FileCheck %s
+
+; The unpredictable flag of a select must survive the CMOV rewrites in
+; combineCMov, so that X86CmovConversion can see it. Every function below
+; loses the flag without the fix.
+
+; Simplified EFLAGS: the compare of the atomic result is folded into the
+; flags of the lock add.
+; CHECK-LABEL: name: simplified_eflags
+; CHECK: unpredictable CMOV32rr
+define i32 @simplified_eflags(ptr %p, i32 %x, i32 %y) {
+entry:
+ %old = atomicrmw add ptr %p, i32 1 seq_cst
+ %c = icmp slt i32 %old, 0
+ %r = select i1 %c, i32 %x, i32 %y, !unpredictable !0
+ ret i32 %r
+}
+
+; and/or of two setcc sharing EFLAGS (oeq is ZF and not PF), folded into two
+; CMOVs; both must keep the flag.
+; CHECK-LABEL: name: double_cmov
+; CHECK: unpredictable CMOV32rr
+; CHECK: unpredictable CMOV32rr
+define i32 @double_cmov(float %a, float %b, i32 %x, i32 %y) {
+entry:
+ %c = fcmp oeq float %a, %b
+ %r = select i1 %c, i32 %x, i32 %y, !unpredictable !0
+ ret i32 %r
+}
+
+; select (x == 0), 0, y -> select (x == 0), x, y.
+; CHECK-LABEL: name: constant_to_register
+; CHECK: unpredictable CMOV32rr
+define i32 @constant_to_register(i32 %a, i32 %y) {
+entry:
+ %c = icmp eq i32 %a, 0
+ %r = select i1 %c, i32 0, i32 %y, !unpredictable !0
+ ret i32 %r
+}
+
+; select (x == 0), C, (cttz x) + C2 -> (cmov (C - C2), (cttz x)) + C2.
+; CHECK-LABEL: name: cttz_add
+; CHECK: unpredictable CMOV32rr
+define i32 @cttz_add(i32 %a) {
+entry:
+ %t = call i32 @llvm.cttz.i32(i32 %a, i1 true)
+ %add = add i32 %t, 1
+ %c = icmp eq i32 %a, 0
+ %r = select i1 %c, i32 33, i32 %add, !unpredictable !0
+ ret i32 %r
+}
+
+declare i32 @llvm.cttz.i32(i32, i1)
+
+!0 = !{}
diff --git a/llvm/test/CodeGen/X86/cmov-unpredictable-mem.ll b/llvm/test/CodeGen/X86/cmov-unpredictable-mem.ll
new file mode 100644
index 0000000000000..a978b1591383b
--- /dev/null
+++ b/llvm/test/CodeGen/X86/cmov-unpredictable-mem.ll
@@ -0,0 +1,41 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64 | FileCheck %s
+
+; A select marked !unpredictable must stay a CMOV even when the CMOV folds a
+; load: X86CmovConversion skips CMOVs carrying the unpredictable flag, so the
+; flag must survive the CMOV rewrites in combineCMov.
+
+; The zero is replaced by the compared value (select (x == 0), 0, y -> x).
+define i32 @load_or_zero(i32 %a, ptr %p) {
+; CHECK-LABEL: load_or_zero:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: movl %edi, %eax
+; CHECK-NEXT: testl %edi, %edi
+; CHECK-NEXT: cmovnel (%rsi), %eax
+; CHECK-NEXT: retq
+entry:
+ %v = load i32, ptr %p, align 4
+ %c = icmp eq i32 %a, 0
+ %r = select i1 %c, i32 0, i32 %v, !unpredictable !0
+ ret i32 %r
+}
+
+; Without the hint the memory-operand CMOV is still turned into a branch.
+define i32 @load_or_zero_nohint(i32 %a, ptr %p) {
+; CHECK-LABEL: load_or_zero_nohint:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: movl %edi, %eax
+; CHECK-NEXT: testl %edi, %edi
+; CHECK-NEXT: je .LBB1_2
+; CHECK-NEXT: # %bb.1: # %entry
+; CHECK-NEXT: movl (%rsi), %eax
+; CHECK-NEXT: .LBB1_2: # %entry
+; CHECK-NEXT: retq
+entry:
+ %v = load i32, ptr %p, align 4
+ %c = icmp eq i32 %a, 0
+ %r = select i1 %c, i32 0, i32 %v
+ ret i32 %r
+}
+
+!0 = !{}
More information about the llvm-commits
mailing list