[clang-tools-extra] [InstCombine] Simplify the pattern `a ne/eq (zext/sext (a ne/eq c))` (PR #65852)

[Nikita Popov via cfe-commits]

================
@@ -6309,7 +6309,69 @@ Instruction *InstCombinerImpl::foldICmpUsingBoolRange(ICmpInst &I) {
       Y->getType()->isIntOrIntVectorTy(1) && Pred == ICmpInst::ICMP_ULE)
     return BinaryOperator::CreateOr(Builder.CreateIsNull(X), Y);
 
+  ICmpInst::Predicate Pred1, Pred2;
   const APInt *C;
+  // icmp eq/ne X, (zext (icmp eq/ne X, C))
+  if (match(&I, m_c_ICmp(Pred1, m_Value(X),
+                         m_ZExt(m_ICmp(Pred2, m_Deferred(X), m_APInt(C))))) &&
+      ICmpInst::isEquality(Pred1) && ICmpInst::isEquality(Pred2)) {
+    if (C->isZero()) {
+      if (Pred2 == ICmpInst::ICMP_EQ) {
+        // icmp eq X, (zext (icmp eq X, 0)) --> false
+        // icmp ne X, (zext (icmp eq X, 0)) --> true
+        return replaceInstUsesWith(
+            I,
+            Constant::getIntegerValue(
+                I.getType(),
+                APInt(1U, static_cast<uint64_t>(Pred1 == ICmpInst::ICMP_NE))));
+      } else {
+        // icmp eq X, (zext (icmp ne X, 0)) --> icmp ult X, 2
+        // icmp ne X, (zext (icmp ne X, 0)) --> icmp ugt X, 1
+        return ICmpInst::Create(
+            Instruction::ICmp,
+            Pred1 == ICmpInst::ICMP_NE ? ICmpInst::ICMP_UGT
+                                       : ICmpInst::ICMP_ULT,
+            X,
+            Constant::getIntegerValue(
+                X->getType(), APInt(X->getType()->getScalarSizeInBits(),
+                                    Pred1 == ICmpInst::ICMP_NE ? 1 : 2)));
+      }
+    } else if (C->isOne()) {
+      if (Pred2 == ICmpInst::ICMP_NE) {
+        // icmp eq X, (zext (icmp ne X, 1)) --> false
+        // icmp ne X, (zext (icmp ne X, 1)) --> true
+        return replaceInstUsesWith(
+            I,
+            Constant::getIntegerValue(
+                I.getType(),
+                APInt(1U, static_cast<uint64_t>(Pred1 == ICmpInst::ICMP_NE))));
+      } else {
+        // icmp eq X, (zext (icmp eq X, 1)) --> icmp ult X, 2
+        // icmp ne X, (zext (icmp eq X, 1)) --> icmp ugt X, 1
+        return ICmpInst::Create(
+            Instruction::ICmp,
+            Pred1 == ICmpInst::ICMP_NE ? ICmpInst::ICMP_UGT
+                                       : ICmpInst::ICMP_ULT,
+            X,
+            Constant::getIntegerValue(
+                X->getType(), APInt(X->getType()->getScalarSizeInBits(),
+                                    Pred1 == ICmpInst::ICMP_NE ? 1 : 2)));
+      }
+    } else {
+      // C != 0 && C != 1
+      // icmp eq X, (zext (icmp eq X, C)) --> icmp eq X, 0
+      // icmp eq X, (zext (icmp ne X, C)) --> icmp eq X, 1
+      // icmp ne X, (zext (icmp eq X, C)) --> icmp ne X, 0
+      // icmp ne X, (zext (icmp ne X, C)) --> icmp ne X, 1
+      return ICmpInst::Create(
+          Instruction::ICmp, Pred1, X,
+          Constant::getIntegerValue(
+              X->getType(),
+              APInt(X->getType()->getScalarSizeInBits(),
+                    static_cast<uint64_t>(Pred2 == ICmpInst::ICMP_NE))));
+    }
+  }
+
----------------
nikic wrote:

My thought here was to do something along the lines of `a pred1 zext(a pred2 c)` to `a pred2 c ? (a pred1 1) : (a pred1 0)` and then check whether `a pred2 c` implies `a pred1 1`  is true/false or `a !pred2 c` implies `a pred1 0` is true/false. The remaining two comparisons will then get simplified based on and/or of icmps.

But I don't know whether this will handle all cases, and maybe the code will be more complex overall than listing all special cases.

https://github.com/llvm/llvm-project/pull/65852