[llvm] [InstCombine] disable select folding resulting in extra instructions (PR #97184)

[Alex MacLean via llvm-commits]

https://github.com/AlexMaclean created https://github.com/llvm/llvm-project/pull/97184

Disable conversion of a `(select (icmp))` when it would result in more instructions `(xor (lshr (and)))`. This transformation produces more instructions and can interfere with other more profitable folds for `select`.  For example before this change the following folding would occur:
```llvm
  %1 = icmp slt i32 %X, 0
  %2 = select i1 %1, i64 0, i64 8
```
to 
```llvm
  %1 = lshr i32 %X, 28
  %2 = and i32 %1, 8
  %3 = xor i32 %2, 8
  %4 = zext nneg i32 %3 to i64
```

>From 181b19186d9f4707f7ab6db94c55411e9c2062b4 Mon Sep 17 00:00:00 2001
From: Alex MacLean <amaclean at nvidia.com>
Date: Wed, 26 Jun 2024 18:27:16 +0000
Subject: [PATCH] [InstCombine] disable select folding resulting in extra
 instructions

---
 llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp | 10 ++++++++--
 llvm/test/Transforms/InstCombine/select-icmp-and.ll   | 11 +++++++++++
 2 files changed, 19 insertions(+), 2 deletions(-)

diff --git a/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp b/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp
index aff691b000413..db52684e174a4 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp
@@ -202,6 +202,14 @@ static Value *foldSelectICmpAnd(SelectInst &Sel, ICmpInst *Cmp,
   const APInt &ValC = !TC.isZero() ? TC : FC;
   unsigned ValZeros = ValC.logBase2();
   unsigned AndZeros = AndMask.logBase2();
+  bool ShouldNotVal = !TC.isZero();
+  ShouldNotVal ^= Pred == ICmpInst::ICMP_NE;
+
+  // If we would need to create an 'and' + 'shift' + 'xor' to replace a 'select'
+  // + 'icmp', then this transformation would result in more instructions and
+  // potentially interfere with other folding.
+  if (CreateAnd && ShouldNotVal && ValZeros != AndZeros)
+    return nullptr;
 
   // Insert the 'and' instruction on the input to the truncate.
   if (CreateAnd)
@@ -221,8 +229,6 @@ static Value *foldSelectICmpAnd(SelectInst &Sel, ICmpInst *Cmp,
 
   // Okay, now we know that everything is set up, we just don't know whether we
   // have a icmp_ne or icmp_eq and whether the true or false val is the zero.
-  bool ShouldNotVal = !TC.isZero();
-  ShouldNotVal ^= Pred == ICmpInst::ICMP_NE;
   if (ShouldNotVal)
     V = Builder.CreateXor(V, ValC);
 
diff --git a/llvm/test/Transforms/InstCombine/select-icmp-and.ll b/llvm/test/Transforms/InstCombine/select-icmp-and.ll
index e1799d1091dac..8bedf699dc922 100644
--- a/llvm/test/Transforms/InstCombine/select-icmp-and.ll
+++ b/llvm/test/Transforms/InstCombine/select-icmp-and.ll
@@ -299,6 +299,17 @@ define <2 x i32> @test74vec(<2 x i32> %x) {
   ret <2 x i32> %2
 }
 
+define i64 @test75(i32 %X) {
+; CHECK-LABEL: @test75(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i32 [[X:%.*]], 0
+; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], i64 0, i64 8
+; CHECK-NEXT:    ret i64 [[TMP2]]
+;
+  %1 = icmp slt i32 %X, 0
+  %2 = select i1 %1, i64 0, i64 8
+  ret i64 %2
+}
+
 ;; Code sequence for (X & 16) ? 16 : 0
 define i32 @test15a(i32 %X) {
 ; CHECK-LABEL: @test15a(