[llvm] [X86] Fix CCMP miscompile for flag-source-folded SETCC leaves (PR #217682)
Evgenii Kudriashov via llvm-commits
llvm-commits at lists.llvm.org
Wed Sep 23 06:33:06 PDT 2026
https://github.com/e-kud updated https://github.com/llvm/llvm-project/pull/217682
>From 4659b5bc3daeeba8ada0be2d88ea9c1afae30a92 Mon Sep 17 00:00:00 2001
From: Evgenii Kudriashov <evgenii.kudriashov at intel.com>
Date: Thu, 17 Sep 2026 09:46:09 -0700
Subject: [PATCH 1/2] [X86] Add tests for CCMP chain leaves with a folded flag
source
EmitCmp does not always emit a comparison: where it can it reuses the EFLAGS of
an instruction that is computed anyway. (X | Y) == 0 uses the OR's own flags,
(X & Y) == 0 an AND's, (0 - X) == Y is rewritten into (X + Y) == 0 and uses an
ADD's, and an equality can reuse an already existing XOR.
Co-Authored-By: Claude Opus 5 (1M context) <noreply at anthropic.com>
---
llvm/test/CodeGen/X86/apx/ccmp-flag-source.ll | 177 ++++++++++++++++++
1 file changed, 177 insertions(+)
create mode 100644 llvm/test/CodeGen/X86/apx/ccmp-flag-source.ll
diff --git a/llvm/test/CodeGen/X86/apx/ccmp-flag-source.ll b/llvm/test/CodeGen/X86/apx/ccmp-flag-source.ll
new file mode 100644
index 00000000000000..21e8d765ac7aa8
--- /dev/null
+++ b/llvm/test/CodeGen/X86/apx/ccmp-flag-source.ll
@@ -0,0 +1,177 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+ccmp | FileCheck %s
+
+; EmitCmp does not always emit a comparison: where it can it reuses the EFLAGS
+; of an instruction that is computed anyway. (X | Y) == 0 uses the OR's own
+; flags, (X & Y) == 0 an AND's, (0 - X) == Y is rewritten into (X + Y) == 0 and
+; uses an ADD's, and an equality can reuse an already existing XOR.
+;
+; A non-root leaf of a CCMP chain rebuilds a conditional comparison out of that
+; flag source's operands. Only a subtract (CCMP) and an AND (CTEST) can be made
+; conditional, so a flag source that computes something else must not have its
+; operands reused.
+
+; A comparison of an OR with zero, in a non-root slot of the chain.
+;
+; FIXME: Miscompiled. The OR's result is never tested, the CCMP compares x with
+; y instead.
+define i32 @or_vs_zero_interior(i32 %x, i32 %y, i32 %p, i32 %q, i32 %v) {
+; CHECK-LABEL: or_vs_zero_interior:
+; CHECK: # %bb.0:
+; CHECK-NEXT: cmpl %ecx, %edx
+; CHECK-NEXT: ccmpll {dfv=} %esi, %edi
+; CHECK-NEXT: movl $-1, %eax
+; CHECK-NEXT: cmovel %r8d, %eax
+; CHECK-NEXT: retq
+ %or = or i32 %x, %y
+ %c1 = icmp eq i32 %or, 0
+ %c2 = icmp slt i32 %p, %q
+ %cond = and i1 %c1, %c2
+ %sel = select i1 %cond, i32 %v, i32 -1
+ ret i32 %sel
+}
+
+; Same, with a flag source that has a second use and a condition that reads SF
+; rather than ZF.
+;
+; FIXME: Miscompiled. The ADD's result is never tested, the CCMP compares x with
+; y instead.
+define i32 @stored_add_vs_zero_interior(i32 %x, i32 %y, i32 %p, i32 %q, i32 %v, ptr %s) {
+; CHECK-LABEL: stored_add_vs_zero_interior:
+; CHECK: # %bb.0:
+; CHECK-NEXT: # kill: def $esi killed $esi def $rsi
+; CHECK-NEXT: # kill: def $edi killed $edi def $rdi
+; CHECK-NEXT: leal (%rdi,%rsi), %eax
+; CHECK-NEXT: movl %eax, (%r9)
+; CHECK-NEXT: cmpl %ecx, %edx
+; CHECK-NEXT: ccmpll {dfv=sf} %esi, %edi
+; CHECK-NEXT: movl $-1, %eax
+; CHECK-NEXT: cmovnsl %r8d, %eax
+; CHECK-NEXT: retq
+ %add = add i32 %x, %y
+ store i32 %add, ptr %s
+ %c1 = icmp sgt i32 %add, -1
+ %c2 = icmp slt i32 %p, %q
+ %cond = and i1 %c1, %c2
+ %sel = select i1 %cond, i32 %v, i32 -1
+ ret i32 %sel
+}
+
+; An AND compared with zero is a test of the AND's operands, so the AND itself
+; is not needed.
+define i32 @and_vs_zero_interior(i32 %x, i32 %y, i32 %p, i32 %q, i32 %v) {
+; CHECK-LABEL: and_vs_zero_interior:
+; CHECK: # %bb.0:
+; CHECK-NEXT: andl %esi, %edi
+; CHECK-NEXT: cmpl %ecx, %edx
+; CHECK-NEXT: ccmpll {dfv=} $0, %edi
+; CHECK-NEXT: movl $-1, %eax
+; CHECK-NEXT: cmovel %r8d, %eax
+; CHECK-NEXT: retq
+ %and = and i32 %x, %y
+ %c1 = icmp eq i32 %and, 0
+ %c2 = icmp slt i32 %p, %q
+ %cond = and i1 %c1, %c2
+ %sel = select i1 %cond, i32 %v, i32 -1
+ ret i32 %sel
+}
+
+; An AND whose result is used elsewhere, so EmitCmp reuses the AND's own EFLAGS
+; rather than emitting a test.
+;
+; FIXME: Miscompiled. The CCMP subtracts x and y instead of testing them.
+define i32 @stored_and_vs_zero_interior(i32 %x, i32 %y, i32 %p, i32 %q, i32 %v, ptr %s) {
+; CHECK-LABEL: stored_and_vs_zero_interior:
+; CHECK: # %bb.0:
+; CHECK-NEXT: movl %edi, %eax
+; CHECK-NEXT: andl %esi, %eax
+; CHECK-NEXT: movl %eax, (%r9)
+; CHECK-NEXT: cmpl %ecx, %edx
+; CHECK-NEXT: ccmpll {dfv=} %esi, %edi
+; CHECK-NEXT: movl $-1, %eax
+; CHECK-NEXT: cmovel %r8d, %eax
+; CHECK-NEXT: retq
+ %and = and i32 %x, %y
+ store i32 %and, ptr %s
+ %c1 = icmp eq i32 %and, 0
+ %c2 = icmp slt i32 %p, %q
+ %cond = and i1 %c1, %c2
+ %sel = select i1 %cond, i32 %v, i32 -1
+ ret i32 %sel
+}
+
+; An equality comparison is a test of the operands' XOR, so an already live XOR
+; can be reused instead of keeping both operands live.
+define i32 @eq_reuses_live_xor_interior(i32 %x, i32 %y, i32 %p, i32 %q, i32 %v, ptr %s) {
+; CHECK-LABEL: eq_reuses_live_xor_interior:
+; CHECK: # %bb.0:
+; CHECK-NEXT: movl %edi, %eax
+; CHECK-NEXT: xorl %esi, %eax
+; CHECK-NEXT: movl %eax, (%r9)
+; CHECK-NEXT: cmpl %ecx, %edx
+; CHECK-NEXT: ccmpll {dfv=} %esi, %edi
+; CHECK-NEXT: movl $-1, %eax
+; CHECK-NEXT: cmovel %r8d, %eax
+; CHECK-NEXT: retq
+ %xor = xor i32 %x, %y
+ store i32 %xor, ptr %s
+ %c1 = icmp eq i32 %x, %y
+ %c2 = icmp slt i32 %p, %q
+ %cond = and i1 %c1, %c2
+ %sel = select i1 %cond, i32 %v, i32 -1
+ ret i32 %sel
+}
+
+; Without a live XOR the comparison stays a comparison of the two operands.
+define i32 @eq_without_live_xor_interior(i32 %x, i32 %y, i32 %p, i32 %q, i32 %v) {
+; CHECK-LABEL: eq_without_live_xor_interior:
+; CHECK: # %bb.0:
+; CHECK-NEXT: cmpl %ecx, %edx
+; CHECK-NEXT: ccmpll {dfv=} %esi, %edi
+; CHECK-NEXT: movl $-1, %eax
+; CHECK-NEXT: cmovel %r8d, %eax
+; CHECK-NEXT: retq
+ %c1 = icmp eq i32 %x, %y
+ %c2 = icmp slt i32 %p, %q
+ %cond = and i1 %c1, %c2
+ %sel = select i1 %cond, i32 %v, i32 -1
+ ret i32 %sel
+}
+
+; (0 - b) == c is rewritten by EmitCmp into (b + c) == 0.
+;
+; FIXME: Miscompiled. Neither the negation nor an addition is tested, the CCMP
+; compares b with c instead.
+define i32 @neg_eq_interior(i32 %b, i32 %c, i32 %p, i32 %q, i32 %v) {
+; CHECK-LABEL: neg_eq_interior:
+; CHECK: # %bb.0:
+; CHECK-NEXT: cmpl %ecx, %edx
+; CHECK-NEXT: ccmpll {dfv=} %esi, %edi
+; CHECK-NEXT: movl $-1, %eax
+; CHECK-NEXT: cmovel %r8d, %eax
+; CHECK-NEXT: retq
+ %neg = sub i32 0, %b
+ %c1 = icmp eq i32 %neg, %c
+ %c2 = icmp slt i32 %p, %q
+ %cond = and i1 %c1, %c2
+ %sel = select i1 %cond, i32 %v, i32 -1
+ ret i32 %sel
+}
+
+; The root of the chain does not have this problem: unconditional flags are
+; exactly what it needs, so the OR provides the flags of the first comparison.
+define i32 @or_vs_zero_root(i32 %x, i32 %y, i32 %p, i32 %q, i32 %v) {
+; CHECK-LABEL: or_vs_zero_root:
+; CHECK: # %bb.0:
+; CHECK-NEXT: orl %esi, %edi
+; CHECK-NEXT: ccmpel {dfv=} %ecx, %edx
+; CHECK-NEXT: movl $-1, %eax
+; CHECK-NEXT: cmovll %r8d, %eax
+; CHECK-NEXT: retq
+ %or = or i32 %x, %y
+ %c1 = icmp eq i32 %or, 0
+ %c2 = icmp slt i32 %p, %q
+ %cond = and i1 %c2, %c1
+ %sel = select i1 %cond, i32 %v, i32 -1
+ ret i32 %sel
+}
>From 3e8dc482358c06e09f9118e342689357af6da9e2 Mon Sep 17 00:00:00 2001
From: Evgenii Kudriashov <evgenii.kudriashov at intel.com>
Date: Thu, 17 Sep 2026 10:01:31 -0700
Subject: [PATCH 2/2] [X86] Fix CCMP chain leaves whose comparison EmitCmp
folds away
emitConjunctionForCCMPRec built a non-root chain link out of the operands of
whatever EmitCmp returned. EmitCmp does not always return a comparison: where
it can it reuses the EFLAGS of an instruction that is computed anyway, so
(X | Y) == 0 comes back as the flags of an X86ISD::OR, (X & Y) == 0 as those of
an X86ISD::AND, and (0 - X) == Y is rewritten into (X + Y) == 0 and comes back
as the flags of an X86ISD::ADD. Those operands describe a different operation,
and a CCMP subtracts them, so the leaf's predicate silently changed and the
folded operation was dropped.
Only a subtract (CCMP) and an AND (CTEST) have conditional forms, so build the
link from the comparison's own operands rather than from a flag source:
X cmp Y -> CCMP X, Y
V cmp 0 -> CTEST V, V
(X & Y) cmp 0 -> CTEST X, Y
A comparison with zero can set neither OF nor CF, and TEST clears both, so the
last two are flag-equivalent for every condition code. An equality comparison
is a test of the operands' XOR, so an already live XOR is still reused instead
of keeping both operands live.
The leaf that starts the chain keeps using EmitCmp: unconditional flags are
exactly what it needs there.
Co-Authored-By: Claude Opus 5 (1M context) <noreply at anthropic.com>
---
llvm/lib/Target/X86/X86ISelLowering.cpp | 50 +++++++++++++++----
llvm/test/CodeGen/X86/apx/ccmp-flag-source.ll | 49 +++++++-----------
2 files changed, 60 insertions(+), 39 deletions(-)
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 7b3af6152fcfe1..d8d2586c388220 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -25718,7 +25718,10 @@ static bool canEmitConjunctionForCCMP(SelectionDAG &DAG, SDValue Val,
/// Negate: true if this sub-tree should be negated by inverting the
/// conditions on its SETCC leaves
/// OutCC: set to the condition code to test after the whole chain
-/// Returns the flags-producing node (CMP/SUB or CCMP).
+/// Returns the EFLAGS-producing node: for the leaf that starts the chain
+/// whatever EmitCmp picked as its flag source (a comparison, or an arithmetic
+/// or logic node it folded the comparison onto), and a CCMP/CTEST for every
+/// other leaf.
static SDValue emitConjunctionForCCMPRec(SDValue Val, X86::CondCode &OutCC,
bool Negate, SDValue CCOp,
X86::CondCode Predicate,
@@ -25736,23 +25739,52 @@ static SDValue emitConjunctionForCCMPRec(SDValue Val, X86::CondCode &OutCC,
assert(X86CC != X86::COND_INVALID);
OutCC = X86CC;
- SDValue Flags = EmitCmp(LHS, RHS, X86CC, DL, DAG, Subtarget);
- // Produce a normal comparison if we are first in the chain.
+ // Produce a normal comparison if we are first in the chain. Only there may
+ // EmitCmp fold the comparison into the EFLAGS of an existing arithmetic or
+ // logic node: those flags are unconditional, which is exactly what the root
+ // of the chain needs.
if (!CCOp)
- return Flags;
+ return EmitCmp(LHS, RHS, X86CC, DL, DAG, Subtarget);
- // Otherwise produce a CCMP. The CCMP executes (and updates EFLAGS) only
- // when Predicate holds; when it is skipped it forces the default condition
+ // Otherwise produce a CCMP or CTEST. It executes (and updates EFLAGS)
+ // only when Predicate holds; when skipped it forces the default condition
// flags, which must make OutCC evaluate to false. That default is encoded
// from the opposite of X86CC.
- SDNode *FlagsNode = Flags.getNode();
X86::CondCode DCFCode = X86::GetOppositeBranchCondition(X86CC);
SDValue CFlags = DAG.getTargetConstant(
X86::getCCMPCondFlagsFromCondCode(DCFCode), DL, MVT::i8);
SDValue SrcCC = DAG.getTargetConstant(Predicate, DL, MVT::i8);
+
+ // Compare the SETCC's own operands rather than asking EmitCmp for a flag
+ // source: CCMP is a conditional subtract, so it always reproduces the
+ // comparison, whereas a folded flag source computes something else and its
+ // operands cannot be reused. Comparing with zero is a TEST of the value,
+ // and an AND compared with zero is a TEST of the AND's operands, both of
+ // which CTEST performs conditionally.
+ if (isNullConstant(RHS)) {
+ SDValue Op0 = LHS, Op1 = LHS;
+ if (LHS.getOpcode() == ISD::AND) {
+ Op0 = LHS.getOperand(0);
+ Op1 = LHS.getOperand(1);
+ }
+ return DAG.getNode(X86ISD::CTEST, DL, MVT::i32,
+ {Op0, Op1, CFlags, SrcCC, CCOp});
+ }
+ // An equality comparison is a test of the operands' XOR, so reuse one that
+ // is already being computed instead of keeping both operands live.
+ if (X86CC == X86::COND_E || X86CC == X86::COND_NE) {
+ SDVTList XorVTs = DAG.getVTList(LHS.getValueType());
+ SDNode *Xor = DAG.getNodeIfExists(ISD::XOR, XorVTs, {LHS, RHS});
+ if (!Xor)
+ Xor = DAG.getNodeIfExists(ISD::XOR, XorVTs, {RHS, LHS});
+ if (Xor && Xor->hasAnyUseOfValue(0)) {
+ SDValue XorVal(Xor, 0);
+ return DAG.getNode(X86ISD::CTEST, DL, MVT::i32,
+ {XorVal, XorVal, CFlags, SrcCC, CCOp});
+ }
+ }
return DAG.getNode(X86ISD::CCMP, DL, MVT::i32,
- {FlagsNode->getOperand(0), FlagsNode->getOperand(1),
- CFlags, SrcCC, CCOp});
+ {LHS, RHS, CFlags, SrcCC, CCOp});
}
assert(Val.hasOneUse() && "Valid conjunction/disjunction tree");
diff --git a/llvm/test/CodeGen/X86/apx/ccmp-flag-source.ll b/llvm/test/CodeGen/X86/apx/ccmp-flag-source.ll
index 21e8d765ac7aa8..c98bdb9572940d 100644
--- a/llvm/test/CodeGen/X86/apx/ccmp-flag-source.ll
+++ b/llvm/test/CodeGen/X86/apx/ccmp-flag-source.ll
@@ -6,20 +6,18 @@
; flags, (X & Y) == 0 an AND's, (0 - X) == Y is rewritten into (X + Y) == 0 and
; uses an ADD's, and an equality can reuse an already existing XOR.
;
-; A non-root leaf of a CCMP chain rebuilds a conditional comparison out of that
-; flag source's operands. Only a subtract (CCMP) and an AND (CTEST) can be made
-; conditional, so a flag source that computes something else must not have its
-; operands reused.
+; Only a subtract (CCMP) and an AND (CTEST) can be made conditional, so a
+; non-root leaf of a CCMP chain is built from the comparison's own operands
+; rather than from a flag source's: a comparison with zero becomes a test of the
+; value, and an AND compared with zero a test of the AND's operands.
; A comparison of an OR with zero, in a non-root slot of the chain.
-;
-; FIXME: Miscompiled. The OR's result is never tested, the CCMP compares x with
-; y instead.
define i32 @or_vs_zero_interior(i32 %x, i32 %y, i32 %p, i32 %q, i32 %v) {
; CHECK-LABEL: or_vs_zero_interior:
; CHECK: # %bb.0:
+; CHECK-NEXT: orl %esi, %edi
; CHECK-NEXT: cmpl %ecx, %edx
-; CHECK-NEXT: ccmpll {dfv=} %esi, %edi
+; CHECK-NEXT: ctestll {dfv=} %edi, %edi
; CHECK-NEXT: movl $-1, %eax
; CHECK-NEXT: cmovel %r8d, %eax
; CHECK-NEXT: retq
@@ -33,18 +31,13 @@ define i32 @or_vs_zero_interior(i32 %x, i32 %y, i32 %p, i32 %q, i32 %v) {
; Same, with a flag source that has a second use and a condition that reads SF
; rather than ZF.
-;
-; FIXME: Miscompiled. The ADD's result is never tested, the CCMP compares x with
-; y instead.
define i32 @stored_add_vs_zero_interior(i32 %x, i32 %y, i32 %p, i32 %q, i32 %v, ptr %s) {
; CHECK-LABEL: stored_add_vs_zero_interior:
; CHECK: # %bb.0:
-; CHECK-NEXT: # kill: def $esi killed $esi def $rsi
-; CHECK-NEXT: # kill: def $edi killed $edi def $rdi
-; CHECK-NEXT: leal (%rdi,%rsi), %eax
-; CHECK-NEXT: movl %eax, (%r9)
+; CHECK-NEXT: addl %esi, %edi
+; CHECK-NEXT: movl %edi, (%r9)
; CHECK-NEXT: cmpl %ecx, %edx
-; CHECK-NEXT: ccmpll {dfv=sf} %esi, %edi
+; CHECK-NEXT: ctestll {dfv=sf} %edi, %edi
; CHECK-NEXT: movl $-1, %eax
; CHECK-NEXT: cmovnsl %r8d, %eax
; CHECK-NEXT: retq
@@ -62,9 +55,8 @@ define i32 @stored_add_vs_zero_interior(i32 %x, i32 %y, i32 %p, i32 %q, i32 %v,
define i32 @and_vs_zero_interior(i32 %x, i32 %y, i32 %p, i32 %q, i32 %v) {
; CHECK-LABEL: and_vs_zero_interior:
; CHECK: # %bb.0:
-; CHECK-NEXT: andl %esi, %edi
; CHECK-NEXT: cmpl %ecx, %edx
-; CHECK-NEXT: ccmpll {dfv=} $0, %edi
+; CHECK-NEXT: ctestll {dfv=} %esi, %edi
; CHECK-NEXT: movl $-1, %eax
; CHECK-NEXT: cmovel %r8d, %eax
; CHECK-NEXT: retq
@@ -77,9 +69,8 @@ define i32 @and_vs_zero_interior(i32 %x, i32 %y, i32 %p, i32 %q, i32 %v) {
}
; An AND whose result is used elsewhere, so EmitCmp reuses the AND's own EFLAGS
-; rather than emitting a test.
-;
-; FIXME: Miscompiled. The CCMP subtracts x and y instead of testing them.
+; rather than emitting a test. The AND is still tested conditionally, from its
+; operands.
define i32 @stored_and_vs_zero_interior(i32 %x, i32 %y, i32 %p, i32 %q, i32 %v, ptr %s) {
; CHECK-LABEL: stored_and_vs_zero_interior:
; CHECK: # %bb.0:
@@ -87,7 +78,7 @@ define i32 @stored_and_vs_zero_interior(i32 %x, i32 %y, i32 %p, i32 %q, i32 %v,
; CHECK-NEXT: andl %esi, %eax
; CHECK-NEXT: movl %eax, (%r9)
; CHECK-NEXT: cmpl %ecx, %edx
-; CHECK-NEXT: ccmpll {dfv=} %esi, %edi
+; CHECK-NEXT: ctestll {dfv=} %esi, %edi
; CHECK-NEXT: movl $-1, %eax
; CHECK-NEXT: cmovel %r8d, %eax
; CHECK-NEXT: retq
@@ -105,11 +96,10 @@ define i32 @stored_and_vs_zero_interior(i32 %x, i32 %y, i32 %p, i32 %q, i32 %v,
define i32 @eq_reuses_live_xor_interior(i32 %x, i32 %y, i32 %p, i32 %q, i32 %v, ptr %s) {
; CHECK-LABEL: eq_reuses_live_xor_interior:
; CHECK: # %bb.0:
-; CHECK-NEXT: movl %edi, %eax
-; CHECK-NEXT: xorl %esi, %eax
-; CHECK-NEXT: movl %eax, (%r9)
+; CHECK-NEXT: xorl %esi, %edi
+; CHECK-NEXT: movl %edi, (%r9)
; CHECK-NEXT: cmpl %ecx, %edx
-; CHECK-NEXT: ccmpll {dfv=} %esi, %edi
+; CHECK-NEXT: ctestll {dfv=} %edi, %edi
; CHECK-NEXT: movl $-1, %eax
; CHECK-NEXT: cmovel %r8d, %eax
; CHECK-NEXT: retq
@@ -138,13 +128,12 @@ define i32 @eq_without_live_xor_interior(i32 %x, i32 %y, i32 %p, i32 %q, i32 %v)
ret i32 %sel
}
-; (0 - b) == c is rewritten by EmitCmp into (b + c) == 0.
-;
-; FIXME: Miscompiled. Neither the negation nor an addition is tested, the CCMP
-; compares b with c instead.
+; (0 - b) == c is rewritten by EmitCmp into (b + c) == 0, whose EFLAGS have no
+; conditional form. The original operands are compared instead.
define i32 @neg_eq_interior(i32 %b, i32 %c, i32 %p, i32 %q, i32 %v) {
; CHECK-LABEL: neg_eq_interior:
; CHECK: # %bb.0:
+; CHECK-NEXT: negl %edi
; CHECK-NEXT: cmpl %ecx, %edx
; CHECK-NEXT: ccmpll {dfv=} %esi, %edi
; CHECK-NEXT: movl $-1, %eax
More information about the llvm-commits
mailing list