[llvm] [InstCombine] Fold (X ule ~Y) & (Y ugt ~X) to false (PR #178572)

[Mohammad Moeini via llvm-commits]

https://github.com/mbm6448 created https://github.com/llvm/llvm-project/pull/178572

These conditions are contradictory:
- (X ule ~Y) means X + Y does not overflow
- (Y ugt ~X) means X + Y overflows

Both cannot be true simultaneously, so fold to false.

Alive2 proof: https://alive2.llvm.org/ce/z/7BfGRs

Fixes #141479

>From c75be8964c3428e37740e3c576987898d58ee412 Mon Sep 17 00:00:00 2001
From: Mohammad Moeini <mmoeini3 at gatech.edu>
Date: Wed, 28 Jan 2026 21:49:28 -0500
Subject: [PATCH] [InstCombine] Fold (X ule ~Y) & (Y ugt ~X) to false

These conditions are contradictory:
- (X ule ~Y) means X + Y does not overflow
- (Y ugt ~X) means X + Y overflows

Both cannot be true simultaneously, so fold to false.

Fixes #141479
---
 .../InstCombine/InstCombineAndOrXor.cpp       | 28 +++++++
 .../contradictory-overflow-check.ll           | 82 +++++++++++++++++++
 2 files changed, 110 insertions(+)
 create mode 100644 llvm/test/Transforms/InstCombine/contradictory-overflow-check.ll

diff --git a/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp b/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp
index b23519fd9f77f..d727f88a71fda 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp
@@ -1150,6 +1150,29 @@ static Value *foldUnsignedUnderflowCheck(ICmpInst *ZeroICmp,
   return nullptr;
 }
 
+/// Fold (X ule ~Y) & (Y ugt ~X) -> false
+/// These conditions contradict: first says X+Y won't overflow,
+/// second says X+Y will overflow.
+static Value *foldContradictoryUnsignedOverflowCheck(ICmpInst *LHS, 
+                                                      ICmpInst *RHS,
+                                                      bool IsAnd,
+                                                      InstCombiner::BuilderTy &Builder) {
+  if (!IsAnd)
+    return nullptr;
+
+  CmpPredicate Pred1, Pred2;
+  Value *A, *B;
+                                                        
+  if (match(LHS, m_ICmp(Pred1, m_Value(B), m_Not(m_Value(A)))) &&
+      match(RHS, m_ICmp(Pred2, m_Specific(A), m_Not(m_Specific(B)))) &&
+      Pred1 == ICmpInst::ICMP_ULE && 
+      Pred2 == ICmpInst::ICMP_UGT) {
+    return ConstantInt::getFalse(LHS->getType());
+  }
+  
+  return nullptr;
+}
+
 struct IntPart {
   Value *From;
   unsigned StartBit;
@@ -3461,6 +3484,11 @@ Value *InstCombinerImpl::foldAndOrOfICmps(ICmpInst *LHS, ICmpInst *RHS,
       return X;
     if (Value *X = foldUnsignedUnderflowCheck(RHS, LHS, IsAnd, Q, Builder))
       return X;
+
+    if (Value *X = foldContradictoryUnsignedOverflowCheck(LHS, RHS, IsAnd, Builder))
+      return X;
+    if (Value *X = foldContradictoryUnsignedOverflowCheck(RHS, LHS, IsAnd, Builder))
+      return X;
   }
 
   // (icmp ne A, 0) | (icmp ne B, 0) --> (icmp ne (A|B), 0)
diff --git a/llvm/test/Transforms/InstCombine/contradictory-overflow-check.ll b/llvm/test/Transforms/InstCombine/contradictory-overflow-check.ll
new file mode 100644
index 0000000000000..4a925af3d6cef
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/contradictory-overflow-check.ll
@@ -0,0 +1,82 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -passes=instcombine -S | FileCheck %s
+
+; (B ule ~A) && (A ugt ~B) -> false
+; These conditions contradict each other.
+
+define i1 @test_basic(i32 %a, i32 %b) {
+; CHECK-LABEL: @test_basic(
+; CHECK-NEXT:    ret i1 false
+;
+  %nota = xor i32 %a, -1
+  %cmp1 = icmp ule i32 %b, %nota
+  %notb = xor i32 %b, -1
+  %cmp2 = icmp ugt i32 %a, %notb
+  %ret = and i1 %cmp1, %cmp2
+  ret i1 %ret
+}
+
+; Commuted: (A ugt ~B) && (B ule ~A) -> false
+define i1 @test_commuted(i32 %a, i32 %b) {
+; CHECK-LABEL: @test_commuted(
+; CHECK-NEXT:    ret i1 false
+;
+  %nota = xor i32 %a, -1
+  %notb = xor i32 %b, -1
+  %cmp1 = icmp ugt i32 %a, %notb
+  %cmp2 = icmp ule i32 %b, %nota
+  %ret = and i1 %cmp1, %cmp2
+  ret i1 %ret
+}
+
+; Different types: i64
+define i1 @test_i64(i64 %a, i64 %b) {
+; CHECK-LABEL: @test_i64(
+; CHECK-NEXT:    ret i1 false
+;
+  %nota = xor i64 %a, -1
+  %cmp1 = icmp ule i64 %b, %nota
+  %notb = xor i64 %b, -1
+  %cmp2 = icmp ugt i64 %a, %notb
+  %ret = and i1 %cmp1, %cmp2
+  ret i1 %ret
+}
+
+; Different types: i16
+define i1 @test_i16(i16 %a, i16 %b) {
+; CHECK-LABEL: @test_i16(
+; CHECK-NEXT:    ret i1 false
+;
+  %nota = xor i16 %a, -1
+  %cmp1 = icmp ule i16 %b, %nota
+  %notb = xor i16 %b, -1
+  %cmp2 = icmp ugt i16 %a, %notb
+  %ret = and i1 %cmp1, %cmp2
+  ret i1 %ret
+}
+
+; Negative test: OR instead of AND (should NOT fold)
+define i1 @test_negative_or(i32 %a, i32 %b) {
+; CHECK-LABEL: @test_negative_or(
+; CHECK-NOT:     ret i1 false
+;
+  %nota = xor i32 %a, -1
+  %cmp1 = icmp ule i32 %b, %nota
+  %notb = xor i32 %b, -1
+  %cmp2 = icmp ugt i32 %a, %notb
+  %ret = or i1 %cmp1, %cmp2
+  ret i1 %ret
+}
+
+; Negative test: wrong predicate (ult instead of ule)
+define i1 @test_negative_wrong_pred(i32 %a, i32 %b) {
+; CHECK-LABEL: @test_negative_wrong_pred(
+; CHECK-NOT:     ret i1 false
+;
+  %nota = xor i32 %a, -1
+  %cmp1 = icmp ult i32 %b, %nota
+  %notb = xor i32 %b, -1
+  %cmp2 = icmp ugt i32 %a, %notb
+  %ret = and i1 %cmp1, %cmp2
+  ret i1 %ret
+}