[llvm] [ValueTracking] Support GEPs in matchSimpleRecurrence. (PR #123518)

Florian Hahn via llvm-commits llvm-commits at lists.llvm.org
Fri Jun 27 04:13:58 PDT 2025


https://github.com/fhahn updated https://github.com/llvm/llvm-project/pull/123518

>From 01b104b0c65a9f42672ac5683d14a22960753813 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Fri, 27 Jun 2025 12:13:33 +0100
Subject: [PATCH] [ValueTracking] Suport GEPs in matchSimpleRecurrence.

Update matchSimpleRecurrence to also support GEPs. This allows inferring
larger alignments in a number of cases.

I noticed that we fail to infer alignments from calls when dropping
assumptions; inferring alignment from assumptions uses SCEV, if we drop
an assume for a aligned function return value, we fail to infer the
better alignment in InferAlignment without this patch.

It comes with a bit of a compile-time impact:

stage1-O3: +0.05%
stage1-ReleaseThinLTO: +0.04%
stage1-ReleaseLTO-g: +0.03%
stage1-O0-g: -0.04%
stage2-O3: +0.04%
stage2-O0-g: +0.02%
stage2-clang: +0.03%

https://llvm-compile-time-tracker.com/compare.php?from=a8c60790fd4f70a461113f0721bdb4a114ddf420&to=9a207c52e9c644691573a40ceb5b89a3c09ab609&stat=instructions:u
---
 llvm/include/llvm/Analysis/ValueTracking.h    |  6 +-
 llvm/lib/Analysis/ValueTracking.cpp           | 64 +++++++++++++------
 .../InferAlignment/gep-recurrence.ll          | 22 +++----
 3 files changed, 61 insertions(+), 31 deletions(-)

diff --git a/llvm/include/llvm/Analysis/ValueTracking.h b/llvm/include/llvm/Analysis/ValueTracking.h
index c804f551f5a75..240a72722b8d9 100644
--- a/llvm/include/llvm/Analysis/ValueTracking.h
+++ b/llvm/include/llvm/Analysis/ValueTracking.h
@@ -962,7 +962,11 @@ LLVM_ABI bool matchSimpleRecurrence(const PHINode *P, BinaryOperator *&BO,
                                     Value *&Start, Value *&Step);
 
 /// Analogous to the above, but starting from the binary operator
-LLVM_ABI bool matchSimpleRecurrence(const BinaryOperator *I, PHINode *&P,
+LLVM_ABI bool matchSimpleRecurrence(const Instruction *I, PHINode *&P,
+                                    Value *&Start, Value *&Step);
+
+/// Analogous to the above, but also supporting non-binary operators.
+LLVM_ABI bool matchSimpleRecurrence(const PHINode *P, Instruction *&BO,
                                     Value *&Start, Value *&Step);
 
 /// Return true if RHS is known to be implied true by LHS.  Return false if
diff --git a/llvm/lib/Analysis/ValueTracking.cpp b/llvm/lib/Analysis/ValueTracking.cpp
index 93c22212a27ce..bdfa3072743b1 100644
--- a/llvm/lib/Analysis/ValueTracking.cpp
+++ b/llvm/lib/Analysis/ValueTracking.cpp
@@ -1556,7 +1556,7 @@ static void computeKnownBitsFromOperator(const Operator *I,
   }
   case Instruction::PHI: {
     const PHINode *P = cast<PHINode>(I);
-    BinaryOperator *BO = nullptr;
+    Instruction *BO = nullptr;
     Value *R = nullptr, *L = nullptr;
     if (matchSimpleRecurrence(P, BO, R, L)) {
       // Handle the case of a simple two-predecessor recurrence PHI.
@@ -1620,6 +1620,7 @@ static void computeKnownBitsFromOperator(const Operator *I,
       case Instruction::Sub:
       case Instruction::And:
       case Instruction::Or:
+      case Instruction::GetElementPtr:
       case Instruction::Mul: {
         // Change the context instruction to the "edge" that flows into the
         // phi. This is important because that is where the value is actually
@@ -1638,6 +1639,11 @@ static void computeKnownBitsFromOperator(const Operator *I,
 
         // We need to take the minimum number of known bits
         KnownBits Known3(BitWidth);
+        if (BitWidth != getBitWidth(L->getType(), Q.DL)) {
+          assert(isa<GetElementPtrInst>(BO) &&
+                 "Bitwidth should only be different for GEPs.");
+          break;
+        }
         RecQ.CxtI = LInst;
         computeKnownBits(L, DemandedElts, Known3, RecQ, Depth + 1);
 
@@ -1800,6 +1806,7 @@ static void computeKnownBitsFromOperator(const Operator *I,
           Known.resetAll();
       }
     }
+
     if (const IntrinsicInst *II = dyn_cast<IntrinsicInst>(I)) {
       switch (II->getIntrinsicID()) {
       default:
@@ -2333,7 +2340,7 @@ void computeKnownBits(const Value *V, const APInt &DemandedElts,
 /// always a power of two (or zero).
 static bool isPowerOfTwoRecurrence(const PHINode *PN, bool OrZero,
                                    SimplifyQuery &Q, unsigned Depth) {
-  BinaryOperator *BO = nullptr;
+  Instruction *BO = nullptr;
   Value *Start = nullptr, *Step = nullptr;
   if (!matchSimpleRecurrence(PN, BO, Start, Step))
     return false;
@@ -2371,7 +2378,7 @@ static bool isPowerOfTwoRecurrence(const PHINode *PN, bool OrZero,
     // Divisor must be a power of two.
     // If OrZero is false, cannot guarantee induction variable is non-zero after
     // division, same for Shr, unless it is exact division.
-    return (OrZero || Q.IIQ.isExact(BO)) &&
+    return (OrZero || Q.IIQ.isExact(cast<BinaryOperator>(BO))) &&
            isKnownToBeAPowerOfTwo(Step, false, Q, Depth);
   case Instruction::Shl:
     return OrZero || Q.IIQ.hasNoUnsignedWrap(BO) || Q.IIQ.hasNoSignedWrap(BO);
@@ -2380,7 +2387,7 @@ static bool isPowerOfTwoRecurrence(const PHINode *PN, bool OrZero,
       return false;
     [[fallthrough]];
   case Instruction::LShr:
-    return OrZero || Q.IIQ.isExact(BO);
+    return OrZero || Q.IIQ.isExact(cast<BinaryOperator>(BO));
   default:
     return false;
   }
@@ -2792,7 +2799,7 @@ static bool rangeMetadataExcludesValue(const MDNode* Ranges, const APInt& Value)
 /// Try to detect a recurrence that monotonically increases/decreases from a
 /// non-zero starting value. These are common as induction variables.
 static bool isNonZeroRecurrence(const PHINode *PN) {
-  BinaryOperator *BO = nullptr;
+  Instruction *BO = nullptr;
   Value *Start = nullptr, *Step = nullptr;
   const APInt *StartC, *StepC;
   if (!matchSimpleRecurrence(PN, BO, Start, Step) ||
@@ -3632,9 +3639,9 @@ getInvertibleOperands(const Operator *Op1,
     // If PN1 and PN2 are both recurrences, can we prove the entire recurrences
     // are a single invertible function of the start values? Note that repeated
     // application of an invertible function is also invertible
-    BinaryOperator *BO1 = nullptr;
+    Instruction *BO1 = nullptr;
     Value *Start1 = nullptr, *Step1 = nullptr;
-    BinaryOperator *BO2 = nullptr;
+    Instruction *BO2 = nullptr;
     Value *Start2 = nullptr, *Step2 = nullptr;
     if (PN1->getParent() != PN2->getParent() ||
         !matchSimpleRecurrence(PN1, BO1, Start1, Step1) ||
@@ -9072,6 +9079,16 @@ llvm::canConvertToMinOrMaxIntrinsic(ArrayRef<Value *> VL) {
 
 bool llvm::matchSimpleRecurrence(const PHINode *P, BinaryOperator *&BO,
                                  Value *&Start, Value *&Step) {
+  Instruction *I;
+  if (matchSimpleRecurrence(P, I, Start, Step)) {
+    BO = dyn_cast<BinaryOperator>(I);
+    return BO;
+  }
+  return false;
+}
+
+bool llvm::matchSimpleRecurrence(const PHINode *P, Instruction *&BO,
+                                 Value *&Start, Value *&Step) {
   // Handle the case of a simple two-predecessor recurrence PHI.
   // There's a lot more that could theoretically be done here, but
   // this is sufficient to catch some interesting cases.
@@ -9082,20 +9099,29 @@ bool llvm::matchSimpleRecurrence(const PHINode *P, BinaryOperator *&BO,
   for (unsigned i = 0; i != 2; ++i) {
     Value *L = P->getIncomingValue(i);
     Value *R = P->getIncomingValue(!i);
-    auto *LU = dyn_cast<BinaryOperator>(L);
+    auto *LU = dyn_cast<Instruction>(L);
     if (!LU)
       continue;
-    Value *LL = LU->getOperand(0);
-    Value *LR = LU->getOperand(1);
 
-    // Find a recurrence.
-    if (LL == P)
+    Value *LR;
+    if (match(L, m_PtrAdd(m_Specific(P), m_Value(LR)))) {
+      // Found a match
       L = LR;
-    else if (LR == P)
-      L = LL;
-    else
-      continue; // Check for recurrence with L and R flipped.
-
+    } else if (isa<BinaryOperator>(LU)) {
+      Value *LL = LU->getOperand(0);
+      LR = LU->getOperand(1);
+
+      // Find a recurrence.
+      if (LL == P)
+        L = LR;
+      else if (LR == P)
+        L = LL;
+      else
+        continue; // Check for recurrence with L and R flipped.
+    } else {
+      // Skip unsupported op.
+      break;
+    }
     // We have matched a recurrence of the form:
     //   %iv = [R, %entry], [%iv.next, %backedge]
     //   %iv.next = binop %iv, L
@@ -9110,9 +9136,9 @@ bool llvm::matchSimpleRecurrence(const PHINode *P, BinaryOperator *&BO,
   return false;
 }
 
-bool llvm::matchSimpleRecurrence(const BinaryOperator *I, PHINode *&P,
+bool llvm::matchSimpleRecurrence(const Instruction *I, PHINode *&P,
                                  Value *&Start, Value *&Step) {
-  BinaryOperator *BO = nullptr;
+  Instruction *BO = nullptr;
   P = dyn_cast<PHINode>(I->getOperand(0));
   if (!P)
     P = dyn_cast<PHINode>(I->getOperand(1));
diff --git a/llvm/test/Transforms/InferAlignment/gep-recurrence.ll b/llvm/test/Transforms/InferAlignment/gep-recurrence.ll
index 50b968d206b83..2b241ca59ebce 100644
--- a/llvm/test/Transforms/InferAlignment/gep-recurrence.ll
+++ b/llvm/test/Transforms/InferAlignment/gep-recurrence.ll
@@ -12,7 +12,7 @@ define void @recur_i8_128(ptr align 128 %dst) {
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
 ; CHECK-NEXT:    [[IV:%.*]] = phi ptr [ [[DST]], %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT:    store i64 0, ptr [[IV]], align 1
+; CHECK-NEXT:    store i64 0, ptr [[IV]], align 128
 ; CHECK-NEXT:    [[IV_NEXT]] = getelementptr nusw i8, ptr [[IV]], i64 128
 ; CHECK-NEXT:    [[C:%.*]] = call i1 @cond()
 ; CHECK-NEXT:    br i1 [[C]], label %[[LOOP]], label %[[EXIT:.*]]
@@ -40,7 +40,7 @@ define void @recur_i8_128_no_nusw(ptr align 128 %dst) {
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
 ; CHECK-NEXT:    [[IV:%.*]] = phi ptr [ [[DST]], %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT:    store i64 0, ptr [[IV]], align 1
+; CHECK-NEXT:    store i64 0, ptr [[IV]], align 128
 ; CHECK-NEXT:    [[IV_NEXT]] = getelementptr i8, ptr [[IV]], i64 128
 ; CHECK-NEXT:    [[C:%.*]] = call i1 @cond()
 ; CHECK-NEXT:    br i1 [[C]], label %[[LOOP]], label %[[EXIT:.*]]
@@ -68,7 +68,7 @@ define void @recur_i8_64(ptr align 128 %dst) {
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
 ; CHECK-NEXT:    [[IV:%.*]] = phi ptr [ [[DST]], %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT:    store i64 0, ptr [[IV]], align 1
+; CHECK-NEXT:    store i64 0, ptr [[IV]], align 64
 ; CHECK-NEXT:    [[IV_NEXT]] = getelementptr nusw i8, ptr [[IV]], i64 64
 ; CHECK-NEXT:    [[C:%.*]] = call i1 @cond()
 ; CHECK-NEXT:    br i1 [[C]], label %[[LOOP]], label %[[EXIT:.*]]
@@ -124,7 +124,7 @@ define void @recur_i8_32(ptr align 128 %dst) {
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
 ; CHECK-NEXT:    [[IV:%.*]] = phi ptr [ [[DST]], %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT:    store i64 0, ptr [[IV]], align 1
+; CHECK-NEXT:    store i64 0, ptr [[IV]], align 32
 ; CHECK-NEXT:    [[IV_NEXT]] = getelementptr nusw i8, ptr [[IV]], i64 32
 ; CHECK-NEXT:    [[C:%.*]] = call i1 @cond()
 ; CHECK-NEXT:    br i1 [[C]], label %[[LOOP]], label %[[EXIT:.*]]
@@ -152,7 +152,7 @@ define void @recur_i8_16(ptr align 128 %dst) {
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
 ; CHECK-NEXT:    [[IV:%.*]] = phi ptr [ [[DST]], %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT:    store i64 0, ptr [[IV]], align 1
+; CHECK-NEXT:    store i64 0, ptr [[IV]], align 16
 ; CHECK-NEXT:    [[IV_NEXT]] = getelementptr nusw i8, ptr [[IV]], i64 16
 ; CHECK-NEXT:    [[C:%.*]] = call i1 @cond()
 ; CHECK-NEXT:    br i1 [[C]], label %[[LOOP]], label %[[EXIT:.*]]
@@ -180,7 +180,7 @@ define void @recur_i8_8(ptr align 128 %dst) {
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
 ; CHECK-NEXT:    [[IV:%.*]] = phi ptr [ [[DST]], %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT:    store i64 0, ptr [[IV]], align 1
+; CHECK-NEXT:    store i64 0, ptr [[IV]], align 8
 ; CHECK-NEXT:    [[IV_NEXT]] = getelementptr nusw i8, ptr [[IV]], i64 8
 ; CHECK-NEXT:    [[C:%.*]] = call i1 @cond()
 ; CHECK-NEXT:    br i1 [[C]], label %[[LOOP]], label %[[EXIT:.*]]
@@ -208,7 +208,7 @@ define void @recur_i8_4(ptr align 128 %dst) {
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
 ; CHECK-NEXT:    [[IV:%.*]] = phi ptr [ [[DST]], %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT:    store i64 0, ptr [[IV]], align 1
+; CHECK-NEXT:    store i64 0, ptr [[IV]], align 4
 ; CHECK-NEXT:    [[IV_NEXT]] = getelementptr nusw i8, ptr [[IV]], i64 4
 ; CHECK-NEXT:    [[C:%.*]] = call i1 @cond()
 ; CHECK-NEXT:    br i1 [[C]], label %[[LOOP]], label %[[EXIT:.*]]
@@ -236,7 +236,7 @@ define void @recur_i8_2(ptr align 128 %dst) {
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
 ; CHECK-NEXT:    [[IV:%.*]] = phi ptr [ [[DST]], %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT:    store i64 0, ptr [[IV]], align 1
+; CHECK-NEXT:    store i64 0, ptr [[IV]], align 2
 ; CHECK-NEXT:    [[IV_NEXT]] = getelementptr nusw i8, ptr [[IV]], i64 2
 ; CHECK-NEXT:    [[C:%.*]] = call i1 @cond()
 ; CHECK-NEXT:    br i1 [[C]], label %[[LOOP]], label %[[EXIT:.*]]
@@ -468,7 +468,7 @@ define void @recur_i8_neg_128(ptr align 128 %dst) {
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
 ; CHECK-NEXT:    [[IV:%.*]] = phi ptr [ [[DST]], %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT:    store i64 0, ptr [[IV]], align 1
+; CHECK-NEXT:    store i64 0, ptr [[IV]], align 128
 ; CHECK-NEXT:    [[IV_NEXT]] = getelementptr nusw i8, ptr [[IV]], i64 -128
 ; CHECK-NEXT:    [[C:%.*]] = call i1 @cond()
 ; CHECK-NEXT:    br i1 [[C]], label %[[LOOP]], label %[[EXIT:.*]]
@@ -496,7 +496,7 @@ define void @recur_i8_neg64(ptr align 128 %dst) {
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
 ; CHECK-NEXT:    [[IV:%.*]] = phi ptr [ [[DST]], %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT:    store i64 0, ptr [[IV]], align 1
+; CHECK-NEXT:    store i64 0, ptr [[IV]], align 64
 ; CHECK-NEXT:    [[IV_NEXT]] = getelementptr nusw i8, ptr [[IV]], i64 -64
 ; CHECK-NEXT:    [[C:%.*]] = call i1 @cond()
 ; CHECK-NEXT:    br i1 [[C]], label %[[LOOP]], label %[[EXIT:.*]]
@@ -552,7 +552,7 @@ define void @recur_i8_neg_32(ptr align 128 %dst) {
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
 ; CHECK-NEXT:    [[IV:%.*]] = phi ptr [ [[DST]], %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT:    store i64 0, ptr [[IV]], align 1
+; CHECK-NEXT:    store i64 0, ptr [[IV]], align 32
 ; CHECK-NEXT:    [[IV_NEXT]] = getelementptr nusw i8, ptr [[IV]], i64 -32
 ; CHECK-NEXT:    [[C:%.*]] = call i1 @cond()
 ; CHECK-NEXT:    br i1 [[C]], label %[[LOOP]], label %[[EXIT:.*]]



More information about the llvm-commits mailing list