[llvm] [InstCombine] Prevent infinite loop with two shifts (PR #118806)

[Maurice Heumann via llvm-commits]

https://github.com/momo5502 created https://github.com/llvm/llvm-project/pull/118806

The following pattern: `(C2 << X) << C1` will usually be transformed into `(C2 << C1) << X`, essentially swapping `X` and `C1`.

However, this should not be done when `X` is also a constant, as this can lead to both constants being swapped forever.

This fixes #118798

>From 5977d79c6ce41cfc4cbf18764eec8cae1c40f12a Mon Sep 17 00:00:00 2001
From: Maurice Heumann <maurice.heumann at wibu.com>
Date: Thu, 5 Dec 2024 14:54:39 +0100
Subject: [PATCH] [InstCombine] Prevent infinite loop with two shifts

The following pattern: `(C2 << X) << C1` will usually be
transformed into `(C2 << C1) << X`, essentially swapping `X` and `C1`.

However, this should not be done when `X` is also a constant, as this
can lead to swapping both constants indefinitely.

This fixes #118798
---
 .../InstCombine/InstCombineShifts.cpp         |  4 +++-
 .../InstCombine/shl-twice-constant.ll         | 22 +++++++++++++++++++
 2 files changed, 25 insertions(+), 1 deletion(-)
 create mode 100644 llvm/test/Transforms/InstCombine/shl-twice-constant.ll

diff --git a/llvm/lib/Transforms/InstCombine/InstCombineShifts.cpp b/llvm/lib/Transforms/InstCombine/InstCombineShifts.cpp
index 10c3ccdb2243a1..3fb094579d585a 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineShifts.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineShifts.cpp
@@ -786,7 +786,9 @@ Instruction *InstCombinerImpl::FoldShiftByConstant(Value *Op0, Constant *C1,
   Constant *C2;
   Value *X;
   bool IsLeftShift = I.getOpcode() == Instruction::Shl;
-  if (match(Op0, m_BinOp(I.getOpcode(), m_ImmConstant(C2), m_Value(X)))) {
+  if (match(Op0,
+            m_BinOp(I.getOpcode(), m_ImmConstant(C2),
+                    m_CombineAnd(m_Value(X), m_Unless(m_Constant()))))) {
     Instruction *R = BinaryOperator::Create(
         I.getOpcode(), Builder.CreateBinOp(I.getOpcode(), C2, C1), X);
     BinaryOperator *BO0 = cast<BinaryOperator>(Op0);
diff --git a/llvm/test/Transforms/InstCombine/shl-twice-constant.ll b/llvm/test/Transforms/InstCombine/shl-twice-constant.ll
new file mode 100644
index 00000000000000..54c5611400f46c
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/shl-twice-constant.ll
@@ -0,0 +1,22 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -passes=instcombine -S | FileCheck %s
+
+ at c = external constant i8
+ at c2 = external constant i8
+
+define i16 @testfunc() {
+; CHECK-LABEL: @testfunc(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = shl nuw i64 1, ptrtoint (ptr @c2 to i64)
+; CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[TMP0]], ptrtoint (ptr @c to i64)
+; CHECK-NEXT:    [[TMP2:%.*]] = inttoptr i64 [[TMP1]] to ptr
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr [[TMP2]], align 1
+; CHECK-NEXT:    ret i16 [[TMP3]]
+;
+entry:
+  %0 = shl i64 1, ptrtoint (ptr @c2 to i64)
+  %1 = shl i64 %0, ptrtoint (ptr @c to i64)
+  %2 = inttoptr i64 %1 to ptr
+  %3 = load i16, ptr %2, align 1
+  ret i16 %3
+}