[llvm] [InstCombine] Missed fold: umax(x, C) > ~x -> x < 0 (PR #189396)

[Madhur Kumar via llvm-commits]

https://github.com/MadhurKumar004 created https://github.com/llvm/llvm-project/pull/189396

fix : https://github.com/llvm/llvm-project/issues/187648

Fix the missed optimization for 
`icmp ugt (umax(x, C)), ~x` and `icmp ult (umax(x, C)), ~x`

>From efc8bc64e9841a349e8ceb7c19442ef789b9ed29 Mon Sep 17 00:00:00 2001
From: Madhur Kumar <madhurkumar004 at gmail.com>
Date: Mon, 30 Mar 2026 20:07:16 +0530
Subject: [PATCH] [InstCombine] Missed fold: umax(x, C) > ~x -> x < 0

---
 .../InstCombine/InstCombineCompares.cpp       | 25 ++++++++++-
 .../Transforms/InstCombine/icmp-umax-notx.ll  | 45 +++++++++++++++++++
 2 files changed, 69 insertions(+), 1 deletion(-)
 create mode 100644 llvm/test/Transforms/InstCombine/icmp-umax-notx.ll

diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp
index ff1868f3a5840..e029b845d595e 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp
@@ -7785,13 +7785,36 @@ Instruction *InstCombinerImpl::visitICmpInst(ICmpInst &I) {
   if (Instruction *Res = foldICmpWithZero(I))
     return Res;
 
+  Value* X;
+  const APInt *C;
+  if (I.getPredicate() == ICmpInst::ICMP_UGT &&
+          match(Op0, m_UMax(m_Value(X), m_APInt(C))) &&
+          match(Op1, m_Not(m_Specific(X)))) {
+    if (C->isNonNegative()) {
+      return new ICmpInst(ICmpInst::ICMP_SLT, X,
+                        Constant::getNullValue(X->getType()));
+    }
+    return new ICmpInst(ICmpInst::ICMP_UGT, X,
+                    ConstantInt::get(X->getType(), ~*C));
+  }
+
+  if (I.getPredicate() == ICmpInst::ICMP_ULT &&
+          match(Op0, m_UMax(m_Value(X), m_APInt(C))) &&
+          match(Op1, m_Not(m_Specific(X)))) {
+    if (C->isNonNegative()) {
+      return new ICmpInst(ICmpInst::ICMP_SGT, X,
+                          Constant::getAllOnesValue(X->getType()));
+    }
+    return new ICmpInst(ICmpInst::ICMP_ULT, X,
+                        ConstantInt::get(X->getType(), ~*C));
+  }
+
   // FIXME: We only do this after checking for min/max to prevent infinite
   // looping caused by a reverse canonicalization of these patterns for min/max.
   // FIXME: The organization of folds is a mess. These would naturally go into
   // canonicalizeCmpWithConstant(), but we can't move all of the above folds
   // down here after the min/max restriction.
   ICmpInst::Predicate Pred = I.getPredicate();
-  const APInt *C;
   if (match(Op1, m_APInt(C))) {
     // For i32: x >u 2147483647 -> x <s 0  -> true if sign bit set
     if (Pred == ICmpInst::ICMP_UGT && C->isMaxSignedValue()) {
diff --git a/llvm/test/Transforms/InstCombine/icmp-umax-notx.ll b/llvm/test/Transforms/InstCombine/icmp-umax-notx.ll
new file mode 100644
index 0000000000000..5eae09375d27a
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/icmp-umax-notx.ll
@@ -0,0 +1,45 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -S -passes=instcombine < %s | FileCheck %s
+
+declare i64 @llvm.umax.i64(i64, i64)
+
+define i1 @test_ugt_small_c(i64 %x){
+; CHECK-LABEL: @test_ugt_small_c(
+; CHECK-NEXT:    %r = icmp slt i64 %x, 0
+; CHECK-NEXT:    ret i1 %r
+  %m = call i64 @llvm.umax.i64(i64 %x, i64 1)
+  %notx = xor i64 %x, -1
+  %r = icmp ugt i64 %m, %notx
+  ret i1 %r
+}
+
+define i1 @test_ugt_large_c(i64 %x){
+; CHECK-LABEL: @test_ugt_large_c(
+; CHECK-NEXT:    %r = icmp ugt i64 %x, 8446744073709551615
+; CHECK-NEXT:    ret i1 %r
+  %m = call i64 @llvm.umax.i64(i64 %x, i64 10000000000000000000)
+  %notx = xor i64 %x, -1
+  %r = icmp ugt i64 %m, %notx
+  ret i1 %r
+}
+
+define i1 @test_ult_small_c(i64 %x){
+; CHECK-LABEL: @test_ult_small_c(
+; CHECK-NEXT:    %r = icmp sgt i64 %x, -1
+; CHECK-NEXT:    ret i1 %r
+  %m = call i64 @llvm.umax.i64(i64 %x, i64 1)
+  %notx = xor i64 %x, -1
+  %r = icmp ult i64 %m, %notx
+  ret i1 %r
+}
+
+define i1 @test_ult_large_c(i64 %x){
+; CHECK-LABEL: @test_ult_large_c(
+; CHECK-NEXT:    %r = icmp ult i64 %x, 8446744073709551615
+; CHECK-NEXT:    ret i1 %r
+  %m = call i64 @llvm.umax.i64(i64 %x, i64 10000000000000000000)
+  %notx = xor i64 %x, -1
+  %r = icmp ult i64 %m, %notx
+  ret i1 %r
+}
+