[llvm] [X86] Create special case for (a-b) - (a<b) -> sbb a, b (PR #161388)

[via llvm-commits]

https://github.com/AZero13 updated https://github.com/llvm/llvm-project/pull/161388

>From af8fe5c9fd3a1f2d6be92ddf6d55332ef8a47d65 Mon Sep 17 00:00:00 2001
From: AZero13 <gfunni234 at gmail.com>
Date: Tue, 30 Sep 2025 10:20:41 -0400
Subject: [PATCH 1/3] Pre-commit test (NFC)

---
 llvm/test/CodeGen/X86/sbb.ll | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/llvm/test/CodeGen/X86/sbb.ll b/llvm/test/CodeGen/X86/sbb.ll
index 78d609d3a17e6..496e5d01fc2db 100644
--- a/llvm/test/CodeGen/X86/sbb.ll
+++ b/llvm/test/CodeGen/X86/sbb.ll
@@ -365,3 +365,19 @@ define i32 @uge_sext_add(i32 %0, i32 %1, i32 %2) {
   %6 = add nsw i32 %5, %0
   ret i32 %6
 }
+
+define i32 @sub_sub_ugt(i32 %a, i32 %b) {
+; CHECK-LABEL: sub_sub_ugt:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %edi, %eax
+; CHECK-NEXT:    xorl %ecx, %ecx
+; CHECK-NEXT:    subl %esi, %eax
+; CHECK-NEXT:    seta %cl
+; CHECK-NEXT:    subl %ecx, %eax
+; CHECK-NEXT:    retq
+  %cmp = icmp ugt i32 %a, %b
+  %conv = zext i1 %cmp to i32
+  %sub = sub i32 %a, %b
+  %res = sub i32 %sub, %conv
+  ret i32 %res
+}

>From fdbf06612ccd069590c57fd25bcd133422008f2f Mon Sep 17 00:00:00 2001
From: AZero13 <gfunni234 at gmail.com>
Date: Tue, 30 Sep 2025 10:47:19 -0400
Subject: [PATCH 2/3] [X86] Create special case for (a-b) - (a< b) -> sbb a, b

---
 llvm/lib/Target/X86/X86ISelLowering.cpp | 12 ++++++++++++
 llvm/test/CodeGen/X86/sbb.ll            |  6 ++----
 2 files changed, 14 insertions(+), 4 deletions(-)

diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index cd04ff5bc7ef4..b16e77f1f6559 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -52380,6 +52380,18 @@ static SDValue combineAddOrSubToADCOrSBB(bool IsSub, const SDLoc &DL, EVT VT,
                          DAG.getVTList(VT, MVT::i32), X,
                          DAG.getConstant(0, DL, VT), NewEFLAGS);
     }
+
+    // if we have: tmp = SUB(a,b); e = SETC(flags from tmp); res = SUB(tmp, e)
+    // fold to: SBB a, b
+    if (IsSub && X.getOpcode() == X86ISD::SUB && EFLAGS == X.getValue(1) &&
+        X.getValueType().isInteger() && !isa<ConstantSDNode>(X.getOperand(1))) {
+      SDValue NewSub =
+          DAG.getNode(X86ISD::SUB, SDLoc(X), X.getNode()->getVTList(),
+                      X.getOperand(1), X.getOperand(0));
+      SDValue NewEFLAGS = NewSub.getValue(1);
+      return DAG.getNode(X86ISD::SBB, DL, DAG.getVTList(VT, MVT::i32),
+                         X.getOperand(0), X.getOperand(1), NewEFLAGS);
+    }
   }
 
   if (CC == X86::COND_AE) {
diff --git a/llvm/test/CodeGen/X86/sbb.ll b/llvm/test/CodeGen/X86/sbb.ll
index 496e5d01fc2db..579608b628582 100644
--- a/llvm/test/CodeGen/X86/sbb.ll
+++ b/llvm/test/CodeGen/X86/sbb.ll
@@ -370,10 +370,8 @@ define i32 @sub_sub_ugt(i32 %a, i32 %b) {
 ; CHECK-LABEL: sub_sub_ugt:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    movl %edi, %eax
-; CHECK-NEXT:    xorl %ecx, %ecx
-; CHECK-NEXT:    subl %esi, %eax
-; CHECK-NEXT:    seta %cl
-; CHECK-NEXT:    subl %ecx, %eax
+; CHECK-NEXT:    cmpl %edi, %esi
+; CHECK-NEXT:    sbbl %esi, %eax
 ; CHECK-NEXT:    retq
   %cmp = icmp ugt i32 %a, %b
   %conv = zext i1 %cmp to i32

>From 814ee173c811e69ce8566f321506458b278bc2f5 Mon Sep 17 00:00:00 2001
From: AZero13 <gfunni234 at gmail.com>
Date: Tue, 30 Sep 2025 11:24:03 -0400
Subject: [PATCH 3/3] Simplify

---
 llvm/lib/Target/X86/X86ISelLowering.cpp | 27 +++++++++++--------------
 1 file changed, 12 insertions(+), 15 deletions(-)

diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index b16e77f1f6559..5eb6ba6ee0952 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -52369,28 +52369,25 @@ static SDValue combineAddOrSubToADCOrSBB(bool IsSub, const SDLoc &DL, EVT VT,
     // Do not flip "e > c", where "c" is a constant, because Cmp instruction
     // cannot take an immediate as its first operand.
     //
-    if (EFLAGS.getOpcode() == X86ISD::SUB && EFLAGS.getNode()->hasOneUse() &&
+    if (EFLAGS.getOpcode() == X86ISD::SUB &&
         EFLAGS.getValueType().isInteger() &&
         !isa<ConstantSDNode>(EFLAGS.getOperand(1))) {
       SDValue NewSub =
           DAG.getNode(X86ISD::SUB, SDLoc(EFLAGS), EFLAGS.getNode()->getVTList(),
                       EFLAGS.getOperand(1), EFLAGS.getOperand(0));
       SDValue NewEFLAGS = NewSub.getValue(EFLAGS.getResNo());
-      return DAG.getNode(IsSub ? X86ISD::SBB : X86ISD::ADC, DL,
-                         DAG.getVTList(VT, MVT::i32), X,
-                         DAG.getConstant(0, DL, VT), NewEFLAGS);
-    }
 
-    // if we have: tmp = SUB(a,b); e = SETC(flags from tmp); res = SUB(tmp, e)
-    // fold to: SBB a, b
-    if (IsSub && X.getOpcode() == X86ISD::SUB && EFLAGS == X.getValue(1) &&
-        X.getValueType().isInteger() && !isa<ConstantSDNode>(X.getOperand(1))) {
-      SDValue NewSub =
-          DAG.getNode(X86ISD::SUB, SDLoc(X), X.getNode()->getVTList(),
-                      X.getOperand(1), X.getOperand(0));
-      SDValue NewEFLAGS = NewSub.getValue(1);
-      return DAG.getNode(X86ISD::SBB, DL, DAG.getVTList(VT, MVT::i32),
-                         X.getOperand(0), X.getOperand(1), NewEFLAGS);
+      if (EFLAGS.getNode()->hasOneUse()) {
+        return DAG.getNode(IsSub ? X86ISD::SBB : X86ISD::ADC, DL,
+                           DAG.getVTList(VT, MVT::i32), X,
+                           DAG.getConstant(0, DL, VT), NewEFLAGS);
+      }
+
+      if (IsSub && X == EFLAGS.getValue(0)) {
+        return DAG.getNode(X86ISD::SBB, DL, DAG.getVTList(VT, MVT::i32),
+                           EFLAGS.getOperand(0), EFLAGS.getOperand(1),
+                           NewEFLAGS);
+      }
     }
   }