[llvm] [DAGCombiner] Reconstruct borrow chain from icmp pattern for USUBO_CARRY (PR #193707)
via llvm-commits
llvm-commits at lists.llvm.org
Wed Apr 29 03:27:29 PDT 2026
=?utf-8?q?Paweł?= Bylica <pawel at hepcolgum.band>,
=?utf-8?q?Paweł?= Bylica <pawel at hepcolgum.band>
Message-ID:
In-Reply-To: <llvm.org/llvm/llvm-project/pull/193707 at github.com>
llvmbot wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-aarch64
Author: Paweł Bylica (chfast)
<details>
<summary>Changes</summary>
DAG-level alternative to #<!-- -->189018 (CGP): match the canonical icmp form
carry_out = or(icmp ult A, B, and(icmp eq A, B, carry_in))
in visitOR and rewrite to USUBO_CARRY so the backend can chain the
borrow through sbb/sbcs.
Gated on USUBO_CARRY being legal/custom at the type the integer
legalizes to, so targets without hardware carry-flag support are
unaffected. For oversize integers (e.g. i128 on x86_64/aarch64) type
legalization then expands one USUBO_CARRY into a chain of
register-width USUBO_CARRYs, which gives strictly better code than the
CGP-level reconstruction.
Fixes #<!-- -->106118.
---
Full diff: https://github.com/llvm/llvm-project/pull/193707.diff
3 Files Affected:
- (modified) llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp (+29)
- (modified) llvm/test/CodeGen/AArch64/cgp-usubo.ll (+112-5)
- (modified) llvm/test/CodeGen/X86/subcarry.ll (+115-27)
``````````diff
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index d1c7e668a2604..1d261b631cdeb 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -3907,6 +3907,32 @@ static SDValue combineCarryDiamond(SelectionDAG &DAG, const TargetLowering &TLI,
return Merged.getValue(1);
}
+// Reconstruct a subtract-with-borrow chain from its canonicalized icmp form:
+// carry_out = or(icmp ult A, B, and(icmp eq A, B, carry_in))
+// InstCombine collapses chained usub.with.overflow intrinsics into this
+// shape, losing the direct USUBO_CARRY that the backend can chain through
+// sbb/sbcs. Fixes llvm#106118.
+static SDValue combineOrOfSetCCToUSUBOCarry(SDNode *N, SelectionDAG &DAG,
+ const TargetLowering &TLI) {
+ SDValue A, B, CarryIn;
+ if (!sd_match(N, m_Or(m_SetCC(m_Value(A), m_Value(B),
+ m_SpecificCondCode(ISD::SETULT)),
+ m_And(m_c_SetCC(m_Deferred(A), m_Deferred(B),
+ m_SpecificCondCode(ISD::SETEQ)),
+ m_Value(CarryIn)))))
+ return SDValue();
+
+ EVT IntVT = A.getValueType();
+ if (!TLI.isOperationLegalOrCustom(
+ ISD::USUBO_CARRY,
+ TLI.getLegalTypeToTransformTo(*DAG.getContext(), IntVT)))
+ return SDValue();
+
+ SDLoc DL(N);
+ SDVTList VTs = DAG.getVTList(IntVT, N->getValueType(0));
+ return DAG.getNode(ISD::USUBO_CARRY, DL, VTs, A, B, CarryIn).getValue(1);
+}
+
SDValue DAGCombiner::visitUADDO_CARRYLike(SDValue N0, SDValue N1,
SDValue CarryIn, SDNode *N) {
// fold (uaddo_carry (xor a, -1), b, c) -> (usubo_carry b, a, !c) and flip
@@ -8744,6 +8770,9 @@ SDValue DAGCombiner::visitOR(SDNode *N) {
if (SDValue Combined = combineCarryDiamond(DAG, TLI, N0, N1, N))
return Combined;
+ if (SDValue Combined = combineOrOfSetCCToUSUBOCarry(N, DAG, TLI))
+ return Combined;
+
// Recognize halfword bswaps as (bswap + rotl 16) or (bswap + shl 16)
if (SDValue BSwap = MatchBSwapHWord(N, N0, N1))
return BSwap;
diff --git a/llvm/test/CodeGen/AArch64/cgp-usubo.ll b/llvm/test/CodeGen/AArch64/cgp-usubo.ll
index d307107fc07ee..be8aa3eb82075 100644
--- a/llvm/test/CodeGen/AArch64/cgp-usubo.ll
+++ b/llvm/test/CodeGen/AArch64/cgp-usubo.ll
@@ -120,6 +120,113 @@ define i1 @usubo_eq_constant1_op1_i32(i32 %x, ptr %p) nounwind {
ret i1 %ov
}
+; Borrow chain: or(icmp ult X, Y, and(icmp eq X, Y, carry_in)) -> cmp + sbcs.
+; See https://github.com/llvm/llvm-project/issues/106118.
+
+define i1 @subcarry_ult_2x64(i64 %x0, i64 %x1, i64 %y0, i64 %y1) nounwind {
+; CHECK-LABEL: subcarry_ult_2x64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: cmp x0, x2
+; CHECK-NEXT: sbcs xzr, x1, x3
+; CHECK-NEXT: cset w0, lo
+; CHECK-NEXT: ret
+ %b0 = icmp ult i64 %x0, %y0
+ %b1 = icmp ult i64 %x1, %y1
+ %e1 = icmp eq i64 %x1, %y1
+ %bp = and i1 %b0, %e1
+ %br = or i1 %b1, %bp
+ ret i1 %br
+}
+
+; Commuted eq operands: icmp eq Y, X instead of icmp eq X, Y.
+define i1 @subcarry_ult_2x64_commuted_eq(i64 %x0, i64 %x1, i64 %y0, i64 %y1) nounwind {
+; CHECK-LABEL: subcarry_ult_2x64_commuted_eq:
+; CHECK: // %bb.0:
+; CHECK-NEXT: cmp x0, x2
+; CHECK-NEXT: sbcs xzr, x1, x3
+; CHECK-NEXT: cset w0, lo
+; CHECK-NEXT: ret
+ %b0 = icmp ult i64 %x0, %y0
+ %b1 = icmp ult i64 %x1, %y1
+ %e1 = icmp eq i64 %y1, %x1
+ %bp = and i1 %b0, %e1
+ %br = or i1 %b1, %bp
+ ret i1 %br
+}
+
+define i1 @subcarry_ult_3x64(i64 %x0, i64 %x1, i64 %x2, i64 %y0, i64 %y1, i64 %y2) nounwind {
+; CHECK-LABEL: subcarry_ult_3x64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: cmp x0, x3
+; CHECK-NEXT: sbcs xzr, x1, x4
+; CHECK-NEXT: sbcs xzr, x2, x5
+; CHECK-NEXT: cset w0, lo
+; CHECK-NEXT: ret
+ %b0 = icmp ult i64 %x0, %y0
+ %b1 = icmp ult i64 %x1, %y1
+ %e1 = icmp eq i64 %x1, %y1
+ %bp1 = and i1 %b0, %e1
+ %br1 = or i1 %b1, %bp1
+ %b2 = icmp ult i64 %x2, %y2
+ %e2 = icmp eq i64 %x2, %y2
+ %bp2 = and i1 %br1, %e2
+ %br2 = or i1 %b2, %bp2
+ ret i1 %br2
+}
+
+; Unsigned less than of two 2x128 integers combined into a borrow chain.
+define i1 @subcarry_ult_2x128(i128 %x0, i128 %x1, i128 %y0, i128 %y1) nounwind {
+; CHECK-LABEL: subcarry_ult_2x128:
+; CHECK: // %bb.0:
+; CHECK-NEXT: cmp x0, x4
+; CHECK-NEXT: sbcs xzr, x1, x5
+; CHECK-NEXT: sbcs xzr, x2, x6
+; CHECK-NEXT: sbcs xzr, x3, x7
+; CHECK-NEXT: cset w0, lo
+; CHECK-NEXT: ret
+ %b0 = icmp ult i128 %x0, %y0
+ %b1 = icmp ult i128 %x1, %y1
+ %e1 = icmp eq i128 %x1, %y1
+ %bp = and i1 %b0, %e1
+ %br = or i1 %b1, %bp
+ ret i1 %br
+}
+
+; Negative test: icmp eq compares different operands than icmp ult.
+define i1 @no_subcarry_different_ops(i64 %x0, i64 %x1, i64 %y0, i64 %y1) nounwind {
+; CHECK-LABEL: no_subcarry_different_ops:
+; CHECK: // %bb.0:
+; CHECK-NEXT: cmp x0, x2
+; CHECK-NEXT: ccmp x1, x0, #0, lo
+; CHECK-NEXT: ccmp x1, x3, #0, ne
+; CHECK-NEXT: cset w0, lo
+; CHECK-NEXT: ret
+ %b0 = icmp ult i64 %x0, %y0
+ %b1 = icmp ult i64 %x1, %y1
+ %e1 = icmp eq i64 %x1, %x0
+ %bp = and i1 %b0, %e1
+ %br = or i1 %b1, %bp
+ ret i1 %br
+}
+
+; Negative test: icmp slt instead of icmp ult.
+define i1 @no_subcarry_signed(i64 %x0, i64 %x1, i64 %y0, i64 %y1) nounwind {
+; CHECK-LABEL: no_subcarry_signed:
+; CHECK: // %bb.0:
+; CHECK-NEXT: cmp x0, x2
+; CHECK-NEXT: cset w8, lo
+; CHECK-NEXT: cmp x1, x3
+; CHECK-NEXT: csel w8, wzr, w8, ne
+; CHECK-NEXT: csinc w0, w8, wzr, ge
+; CHECK-NEXT: ret
+ %b0 = icmp ult i64 %x0, %y0
+ %b1 = icmp slt i64 %x1, %y1
+ %e1 = icmp eq i64 %x1, %y1
+ %bp = and i1 %b0, %e1
+ %br = or i1 %b1, %bp
+ ret i1 %br
+}
+
; Verify insertion point for multi-BB.
declare void @call(i1)
@@ -127,12 +234,12 @@ declare void @call(i1)
define i1 @usubo_ult_sub_dominates_i64(i64 %x, i64 %y, ptr %p, i1 %cond) nounwind {
; CHECK-LABEL: usubo_ult_sub_dominates_i64:
; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: tbz w3, #0, .LBB7_2
+; CHECK-NEXT: tbz w3, #0, .LBB13_2
; CHECK-NEXT: // %bb.1: // %t
; CHECK-NEXT: subs x8, x0, x1
; CHECK-NEXT: cset w3, lo
; CHECK-NEXT: str x8, [x2]
-; CHECK-NEXT: .LBB7_2: // %common.ret
+; CHECK-NEXT: .LBB13_2: // %common.ret
; CHECK-NEXT: and w0, w3, #0x1
; CHECK-NEXT: ret
entry:
@@ -158,7 +265,7 @@ define i1 @usubo_ult_cmp_dominates_i64(i64 %x, i64 %y, ptr %p, i1 %cond) nounwin
; CHECK-NEXT: stp x20, x19, [sp, #32] // 16-byte Folded Spill
; CHECK-NEXT: mov w19, w3
; CHECK-NEXT: stp x22, x21, [sp, #16] // 16-byte Folded Spill
-; CHECK-NEXT: tbz w3, #0, .LBB8_3
+; CHECK-NEXT: tbz w3, #0, .LBB14_3
; CHECK-NEXT: // %bb.1: // %t
; CHECK-NEXT: cmp x0, x1
; CHECK-NEXT: mov x22, x0
@@ -168,11 +275,11 @@ define i1 @usubo_ult_cmp_dominates_i64(i64 %x, i64 %y, ptr %p, i1 %cond) nounwin
; CHECK-NEXT: mov w0, w21
; CHECK-NEXT: bl call
; CHECK-NEXT: subs x8, x22, x23
-; CHECK-NEXT: b.hs .LBB8_3
+; CHECK-NEXT: b.hs .LBB14_3
; CHECK-NEXT: // %bb.2: // %end
; CHECK-NEXT: mov w19, w21
; CHECK-NEXT: str x8, [x20]
-; CHECK-NEXT: .LBB8_3: // %common.ret
+; CHECK-NEXT: .LBB14_3: // %common.ret
; CHECK-NEXT: and w0, w19, #0x1
; CHECK-NEXT: ldp x20, x19, [sp, #32] // 16-byte Folded Reload
; CHECK-NEXT: ldp x22, x21, [sp, #16] // 16-byte Folded Reload
diff --git a/llvm/test/CodeGen/X86/subcarry.ll b/llvm/test/CodeGen/X86/subcarry.ll
index 60bedad53ac9a..c9b6a008eaf6b 100644
--- a/llvm/test/CodeGen/X86/subcarry.ll
+++ b/llvm/test/CodeGen/X86/subcarry.ll
@@ -1360,41 +1360,23 @@ define i1 @subcarry_ult_2x64_2(i64 %x0, i64 %x1, i64 %y0, i64 %y1) nounwind {
; X64-LABEL: subcarry_ult_2x64_2:
; X64: # %bb.0: # %entry
; X64-NEXT: cmpq %rdx, %rdi
-; X64-NEXT: setb %dl
-; X64-NEXT: cmpq %rcx, %rsi
-; X64-NEXT: setb %cl
-; X64-NEXT: sete %al
-; X64-NEXT: andb %dl, %al
-; X64-NEXT: orb %cl, %al
+; X64-NEXT: sbbq %rcx, %rsi
+; X64-NEXT: setb %al
; X64-NEXT: retq
;
; X86-LABEL: subcarry_ult_2x64_2:
; X86: # %bb.0: # %entry
-; X86-NEXT: pushl %ebx
-; X86-NEXT: pushl %edi
; X86-NEXT: pushl %esi
-; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: movl {{[0-9]+}}(%esp), %esi
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT: movl {{[0-9]+}}(%esp), %edi
-; X86-NEXT: movl {{[0-9]+}}(%esp), %ebx
-; X86-NEXT: cmpl {{[0-9]+}}(%esp), %edi
-; X86-NEXT: sbbl {{[0-9]+}}(%esp), %ebx
-; X86-NEXT: setb %bl
-; X86-NEXT: cmpl %ecx, %eax
-; X86-NEXT: movl %edx, %edi
-; X86-NEXT: sbbl %esi, %edi
-; X86-NEXT: setb %bh
-; X86-NEXT: xorl %esi, %edx
-; X86-NEXT: xorl %ecx, %eax
-; X86-NEXT: orl %edx, %eax
-; X86-NEXT: sete %al
-; X86-NEXT: andb %bl, %al
-; X86-NEXT: orb %bh, %al
+; X86-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT: cmpl {{[0-9]+}}(%esp), %edx
+; X86-NEXT: sbbl {{[0-9]+}}(%esp), %esi
+; X86-NEXT: sbbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: sbbl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: setb %al
; X86-NEXT: popl %esi
-; X86-NEXT: popl %edi
-; X86-NEXT: popl %ebx
; X86-NEXT: retl
entry:
%0 = icmp ult i64 %x0, %y0
@@ -1437,5 +1419,111 @@ entry:
%2 = and i1 %1, %0
ret i1 %2
}
+
+; Unsigned less than of two 2x128 integers combined into a borrow chain.
+define i1 @subcarry_ult_2x128(i128 %x0, i128 %x1, i128 %y0, i128 %y1) nounwind {
+; X64-LABEL: subcarry_ult_2x128:
+; X64: # %bb.0:
+; X64-NEXT: cmpq %r8, %rdi
+; X64-NEXT: sbbq %r9, %rsi
+; X64-NEXT: sbbq {{[0-9]+}}(%rsp), %rdx
+; X64-NEXT: sbbq {{[0-9]+}}(%rsp), %rcx
+; X64-NEXT: setb %al
+; X64-NEXT: retq
+;
+; X86-LABEL: subcarry_ult_2x128:
+; X86: # %bb.0:
+; X86-NEXT: pushl %ebp
+; X86-NEXT: movl %esp, %ebp
+; X86-NEXT: andl $-16, %esp
+; X86-NEXT: subl $16, %esp
+; X86-NEXT: movl 8(%ebp), %eax
+; X86-NEXT: cmpl 40(%ebp), %eax
+; X86-NEXT: movl 12(%ebp), %eax
+; X86-NEXT: sbbl 44(%ebp), %eax
+; X86-NEXT: movl 16(%ebp), %eax
+; X86-NEXT: sbbl 48(%ebp), %eax
+; X86-NEXT: movl 20(%ebp), %eax
+; X86-NEXT: sbbl 52(%ebp), %eax
+; X86-NEXT: movl 24(%ebp), %eax
+; X86-NEXT: sbbl 56(%ebp), %eax
+; X86-NEXT: movl 28(%ebp), %eax
+; X86-NEXT: sbbl 60(%ebp), %eax
+; X86-NEXT: movl 32(%ebp), %eax
+; X86-NEXT: sbbl 64(%ebp), %eax
+; X86-NEXT: movl 36(%ebp), %eax
+; X86-NEXT: sbbl 68(%ebp), %eax
+; X86-NEXT: setb %al
+; X86-NEXT: movl %ebp, %esp
+; X86-NEXT: popl %ebp
+; X86-NEXT: retl
+ %b0 = icmp ult i128 %x0, %y0
+ %b1 = icmp ult i128 %x1, %y1
+ %e1 = icmp eq i128 %x1, %y1
+ %bp = and i1 %b0, %e1
+ %br = or i1 %b1, %bp
+ ret i1 %br
+}
+
+; i256 borrow chain — exercises the non-simple EVT path (i256->i128->i64/i32).
+define i1 @subcarry_ult_2x256(i256 %x0, i256 %x1, i256 %y0, i256 %y1) nounwind {
+; X64-LABEL: subcarry_ult_2x256:
+; X64: # %bb.0:
+; X64-NEXT: movq 16(%rsp), %rax
+; X64-NEXT: movq 8(%rsp), %r10
+; X64-NEXT: cmpq 24(%rsp), %rdi
+; X64-NEXT: sbbq 32(%rsp), %rsi
+; X64-NEXT: sbbq 40(%rsp), %rdx
+; X64-NEXT: sbbq 48(%rsp), %rcx
+; X64-NEXT: sbbq 56(%rsp), %r8
+; X64-NEXT: sbbq 64(%rsp), %r9
+; X64-NEXT: sbbq 72(%rsp), %r10
+; X64-NEXT: sbbq 80(%rsp), %rax
+; X64-NEXT: setb %al
+; X64-NEXT: retq
+;
+; X86-LABEL: subcarry_ult_2x256:
+; X86: # %bb.0:
+; X86-NEXT: movl 4(%esp), %eax
+; X86-NEXT: cmpl 68(%esp), %eax
+; X86-NEXT: movl 8(%esp), %eax
+; X86-NEXT: sbbl 72(%esp), %eax
+; X86-NEXT: movl 12(%esp), %eax
+; X86-NEXT: sbbl 76(%esp), %eax
+; X86-NEXT: movl 16(%esp), %eax
+; X86-NEXT: sbbl 80(%esp), %eax
+; X86-NEXT: movl 20(%esp), %eax
+; X86-NEXT: sbbl 84(%esp), %eax
+; X86-NEXT: movl 24(%esp), %eax
+; X86-NEXT: sbbl 88(%esp), %eax
+; X86-NEXT: movl 28(%esp), %eax
+; X86-NEXT: sbbl 92(%esp), %eax
+; X86-NEXT: movl 32(%esp), %eax
+; X86-NEXT: sbbl 96(%esp), %eax
+; X86-NEXT: movl 36(%esp), %eax
+; X86-NEXT: sbbl 100(%esp), %eax
+; X86-NEXT: movl 40(%esp), %eax
+; X86-NEXT: sbbl 104(%esp), %eax
+; X86-NEXT: movl 44(%esp), %eax
+; X86-NEXT: sbbl 108(%esp), %eax
+; X86-NEXT: movl 48(%esp), %eax
+; X86-NEXT: sbbl 112(%esp), %eax
+; X86-NEXT: movl 52(%esp), %eax
+; X86-NEXT: sbbl 116(%esp), %eax
+; X86-NEXT: movl 56(%esp), %eax
+; X86-NEXT: sbbl 120(%esp), %eax
+; X86-NEXT: movl 60(%esp), %eax
+; X86-NEXT: sbbl 124(%esp), %eax
+; X86-NEXT: movl 64(%esp), %eax
+; X86-NEXT: sbbl 128(%esp), %eax
+; X86-NEXT: setb %al
+; X86-NEXT: retl
+ %b0 = icmp ult i256 %x0, %y0
+ %b1 = icmp ult i256 %x1, %y1
+ %e1 = icmp eq i256 %x1, %y1
+ %bp = and i1 %b0, %e1
+ %br = or i1 %b1, %bp
+ ret i1 %br
+}
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
; CHECK: {{.*}}
``````````
</details>
https://github.com/llvm/llvm-project/pull/193707
More information about the llvm-commits
mailing list