[llvm] [GVN] Support rnflow pattern matching and transform (PR #162259)
Madhur Amilkanthwar via llvm-commits
llvm-commits at lists.llvm.org
Mon May 4 00:14:32 PDT 2026
https://github.com/madhur13490 updated https://github.com/llvm/llvm-project/pull/162259
>From d3758b5edd724bdc3c1952aa53a45c1ee1059584 Mon Sep 17 00:00:00 2001
From: Madhur Amilkanthwar <madhura at nvidia.com>
Date: Fri, 15 Aug 2025 00:34:49 -0700
Subject: [PATCH 01/13] [GVN] Support rnflow pattern matching and transform
---
llvm/include/llvm/Transforms/Scalar/GVN.h | 4 +
llvm/lib/Transforms/Scalar/GVN.cpp | 122 ++++++++++++++++++
.../test/Transforms/GVN/PRE/rnflow-gvn-pre.ll | 59 +++++++++
3 files changed, 185 insertions(+)
create mode 100644 llvm/test/Transforms/GVN/PRE/rnflow-gvn-pre.ll
diff --git a/llvm/include/llvm/Transforms/Scalar/GVN.h b/llvm/include/llvm/Transforms/Scalar/GVN.h
index bc0f108ac8260..b886140348e79 100644
--- a/llvm/include/llvm/Transforms/Scalar/GVN.h
+++ b/llvm/include/llvm/Transforms/Scalar/GVN.h
@@ -22,6 +22,7 @@
#include "llvm/IR/Dominators.h"
#include "llvm/IR/InstrTypes.h"
#include "llvm/IR/PassManager.h"
+#include "llvm/Analysis/LoopInfo.h"
#include "llvm/IR/ValueHandle.h"
#include "llvm/Support/Allocator.h"
#include "llvm/Support/Compiler.h"
@@ -45,6 +46,7 @@ class FunctionPass;
class GetElementPtrInst;
class ImplicitControlFlowTracking;
class LoadInst;
+class SelectInst;
class LoopInfo;
class MemDepResult;
class MemoryAccess;
@@ -405,6 +407,8 @@ class GVNPass : public PassInfoMixin<GVNPass> {
void addDeadBlock(BasicBlock *BB);
void assignValNumForDeadCode();
void assignBlockRPONumber(Function &F);
+
+ bool optimizeMinMaxFindingSelectPattern(SelectInst *Select);
};
/// Create a legacy GVN pass.
diff --git a/llvm/lib/Transforms/Scalar/GVN.cpp b/llvm/lib/Transforms/Scalar/GVN.cpp
index 3bc535595730e..cf00105d5711a 100644
--- a/llvm/lib/Transforms/Scalar/GVN.cpp
+++ b/llvm/lib/Transforms/Scalar/GVN.cpp
@@ -2746,6 +2746,10 @@ bool GVNPass::processInstruction(Instruction *I) {
}
return Changed;
}
+ if (SelectInst *Select = dyn_cast<SelectInst>(I)) {
+ if (optimizeMinMaxFindingSelectPattern(Select))
+ return true;
+ }
// Instructions with void type don't return a value, so there's
// no point in trying to find redundancies in them.
@@ -3333,6 +3337,124 @@ void GVNPass::assignValNumForDeadCode() {
}
}
+bool GVNPass::optimizeMinMaxFindingSelectPattern(SelectInst *Select) {
+ LLVM_DEBUG(
+ dbgs()
+ << "GVN: Analyzing select instruction for minimum finding pattern\n");
+ LLVM_DEBUG(dbgs() << "GVN: Select: " << *Select << "\n");
+ Value *Condition = Select->getCondition();
+ CmpInst *Comparison = dyn_cast<CmpInst>(Condition);
+ if (!Comparison) {
+ LLVM_DEBUG(dbgs() << "GVN: Condition is not a comparison\n");
+ return false;
+ }
+
+ // Check if this is ULT comparison.
+ CmpInst::Predicate Pred = Comparison->getPredicate();
+ if (Pred != CmpInst::ICMP_SLT && Pred != CmpInst::ICMP_ULT &&
+ Pred != CmpInst::FCMP_OLT && Pred != CmpInst::FCMP_ULT) {
+ LLVM_DEBUG(dbgs() << "GVN: Not a less-than comparison, predicate: " << Pred
+ << "\n");
+ return false;
+ }
+
+ // Check that both operands are loads.
+ Value *LHS = Comparison->getOperand(0);
+ Value *RHS = Comparison->getOperand(1);
+ if (!isa<LoadInst>(LHS) || !isa<LoadInst>(RHS)) {
+ LLVM_DEBUG(dbgs() << "GVN: Not both operands are loads\n");
+ return false;
+ }
+
+ LLVM_DEBUG(dbgs() << "GVN: Found minimum finding pattern in Block: "
+ << Select->getParent()->getName() << "\n");
+
+ // Transform the pattern.
+ // Hoist the chain of operations for the second load to preheader.
+ // Get predecessor of the block containing the select instruction.
+ BasicBlock *BB = Select->getParent();
+
+ // Get preheader of the loop.
+ Loop *L = LI->getLoopFor(BB);
+ if (!L) {
+ LLVM_DEBUG(dbgs() << "GVN: Could not find loop\n");
+ return false;
+ }
+ BasicBlock *Preheader = L->getLoopPreheader();
+ if (!Preheader) {
+ LLVM_DEBUG(dbgs() << "GVN: Could not find loop preheader\n");
+ return false;
+ }
+
+ // Hoist the chain of operations for the second load to preheader.
+ // %90 = sext i32 %.05.i to i64
+ // %91 = getelementptr float, ptr %0, i64 %90 ; %0 + (sext i32 %85 to i64)*4
+ // %92 = getelementptr i8, ptr %91, i64 -4 ; %0 + (sext i32 %85 to i64)*4 - 4
+ // %93 = load float, ptr %92, align 4
+
+ Value *BasePtr = nullptr, *IndexVal = nullptr, *OffsetVal = nullptr;
+ IRBuilder<> Builder(Preheader->getTerminator());
+ if (match(RHS,
+ m_Load(m_GEP(m_GEP(m_Value(BasePtr), m_SExt(m_Value(IndexVal))),
+ m_Value(OffsetVal))))) {
+ LLVM_DEBUG(dbgs() << "GVN: Found pattern: " << *RHS << "\n");
+ LLVM_DEBUG(dbgs() << "GVN: Found pattern: " << "\n");
+
+ PHINode *Phi = dyn_cast<PHINode>(IndexVal);
+ if (!Phi) {
+ LLVM_DEBUG(dbgs() << "GVN: IndexVal is not a PHI node\n");
+ return false;
+ }
+ Value *InitialMinIndex = Phi->getIncomingValueForBlock(Preheader);
+
+ // Insert PHI node at the top of this block.
+ PHINode *KnownMinPhi =
+ PHINode::Create(Builder.getFloatTy(), 2, "known_min", BB->begin());
+
+ // Build the GEP chain in the preheader.
+ // 1. hoist_0 = sext i32 to i64
+ Value *HoistedSExt =
+ Builder.CreateSExt(InitialMinIndex, Builder.getInt64Ty(), "hoist_sext");
+
+ // 2. hoist_gep1 = getelementptr float, ptr BasePtr, i64 HoistedSExt
+ Value *HoistedGEP1 = Builder.CreateGEP(Builder.getFloatTy(), BasePtr,
+ HoistedSExt, "hoist_gep1");
+
+ // 3. hoist_gep2 = getelementptr i8, ptr HoistedGEP1, i64 OffsetVal
+ Value *HoistedGEP2 = Builder.CreateGEP(Builder.getInt8Ty(), HoistedGEP1,
+ OffsetVal, "hoist_gep2");
+
+ // 4. hoisted_load = load float, ptr HoistedGEP2
+ LoadInst *NewLoad =
+ Builder.CreateLoad(Builder.getFloatTy(), HoistedGEP2, "hoisted_load");
+
+ // Replace all uses of load with new load.
+ RHS->replaceAllUsesWith(NewLoad);
+ dyn_cast<LoadInst>(RHS)->eraseFromParent();
+
+ // Replace second operand of comparison with KnownMinPhi.
+ Comparison->setOperand(1, KnownMinPhi);
+
+ // Create new select instruction for selecting the minimum value.
+ IRBuilder<> SelectBuilder(BB->getTerminator());
+ SelectInst *CurrentMinSelect =
+ dyn_cast<SelectInst>(SelectBuilder.CreateSelect(
+ Comparison, LHS, KnownMinPhi, "current_min"));
+
+ // Populate PHI node.
+ KnownMinPhi->addIncoming(NewLoad, Preheader);
+ KnownMinPhi->addIncoming(CurrentMinSelect, BB);
+ LLVM_DEBUG(dbgs() << "Transformed the code\n");
+ return true;
+ } else {
+ LLVM_DEBUG(dbgs() << "GVN: Could not find pattern: " << *RHS << "\n");
+ LLVM_DEBUG(dbgs() << "GVN: Could not find pattern: " << "\n");
+ return false;
+ }
+ return false;
+}
+
+
class llvm::gvn::GVNLegacyPass : public FunctionPass {
public:
static char ID; // Pass identification, replacement for typeid.
diff --git a/llvm/test/Transforms/GVN/PRE/rnflow-gvn-pre.ll b/llvm/test/Transforms/GVN/PRE/rnflow-gvn-pre.ll
new file mode 100644
index 0000000000000..6f17d4ab30240
--- /dev/null
+++ b/llvm/test/Transforms/GVN/PRE/rnflow-gvn-pre.ll
@@ -0,0 +1,59 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; Minimal test case containing only the .lr.ph.i basic block
+; RUN: opt -passes=gvn -S < %s | FileCheck %s
+
+define void @test_lr_ph_i(ptr %0) {
+; CHECK-LABEL: define void @test_lr_ph_i(
+; CHECK-SAME: ptr [[TMP0:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[HOIST_GEP1:%.*]] = getelementptr float, ptr [[TMP0]], i64 1
+; CHECK-NEXT: [[HOIST_GEP2:%.*]] = getelementptr i8, ptr [[HOIST_GEP1]], i64 -4
+; CHECK-NEXT: [[HOISTED_LOAD:%.*]] = load float, ptr [[HOIST_GEP2]], align 4
+; CHECK-NEXT: br label %[[DOTLR_PH_I:.*]]
+; CHECK: [[_LR_PH_I:.*:]]
+; CHECK-NEXT: [[KNOWN_MIN:%.*]] = phi float [ [[HOISTED_LOAD]], %[[ENTRY]] ], [ [[CURRENT_MIN:%.*]], %[[DOTLR_PH_I]] ]
+; CHECK-NEXT: [[INDVARS_IV_I:%.*]] = phi i64 [ 1, %[[ENTRY]] ], [ [[INDVARS_IV_NEXT_I:%.*]], %[[DOTLR_PH_I]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[TMP10:%.*]], %[[DOTLR_PH_I]] ]
+; CHECK-NEXT: [[DOT05_I:%.*]] = phi i32 [ 1, %[[ENTRY]] ], [ [[DOT1_I:%.*]], %[[DOTLR_PH_I]] ]
+; CHECK-NEXT: [[INDVARS_IV_NEXT_I]] = add nsw i64 [[INDVARS_IV_I]], -1
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr float, ptr [[TMP0]], i64 [[INDVARS_IV_I]]
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr [[TMP2]], i64 -8
+; CHECK-NEXT: [[TMP4:%.*]] = load float, ptr [[TMP3]], align 4
+; CHECK-NEXT: [[TMP5:%.*]] = sext i32 [[DOT05_I]] to i64
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr float, ptr [[TMP0]], i64 [[TMP5]]
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[TMP6]], i64 -4
+; CHECK-NEXT: [[TMP8:%.*]] = fcmp contract olt float [[TMP4]], [[KNOWN_MIN]]
+; CHECK-NEXT: [[TMP9:%.*]] = trunc nsw i64 [[INDVARS_IV_NEXT_I]] to i32
+; CHECK-NEXT: [[DOT1_I]] = select i1 [[TMP8]], i32 [[TMP9]], i32 [[DOT05_I]]
+; CHECK-NEXT: [[TMP10]] = add nsw i64 [[TMP1]], -1
+; CHECK-NEXT: [[TMP11:%.*]] = icmp samesign ugt i64 [[TMP1]], 1
+; CHECK-NEXT: [[CURRENT_MIN]] = select i1 [[TMP8]], float [[TMP4]], float [[KNOWN_MIN]]
+; CHECK-NEXT: br i1 [[TMP11]], label %[[DOTLR_PH_I]], label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %.lr.ph.i
+
+.lr.ph.i: ; preds = %.lr.ph.i, %entry
+ %indvars.iv.i = phi i64 [ 1, %entry ], [ %indvars.iv.next.i, %.lr.ph.i ]
+ %86 = phi i64 [ 0, %entry ], [ %96, %.lr.ph.i ]
+ %.05.i = phi i32 [ 1, %entry ], [ %.1.i, %.lr.ph.i ]
+ %indvars.iv.next.i = add nsw i64 %indvars.iv.i, -1
+ %87 = getelementptr float, ptr %0, i64 %indvars.iv.i
+ %88 = getelementptr i8, ptr %87, i64 -8 ; first load : %0 + 4 * 1 - 8
+ %89 = load float, ptr %88, align 4
+ %90 = sext i32 %.05.i to i64
+ %91 = getelementptr float, ptr %0, i64 %90 ; %0 + 4 * 1
+ %92 = getelementptr i8, ptr %91, i64 -4 ; second load : %0 + 4 * 1 - 4
+ %93 = load float, ptr %92, align 4
+ %94 = fcmp contract olt float %89, %93
+ %95 = trunc nsw i64 %indvars.iv.next.i to i32
+ %.1.i = select i1 %94, i32 %95, i32 %.05.i
+ %96 = add nsw i64 %86, -1
+ %97 = icmp samesign ugt i64 %86, 1
+ br i1 %97, label %.lr.ph.i, label %exit
+
+exit:
+ ret void
+}
>From 6a26d32a1e8efcf1d85d087e6173e13d5d7d1bde Mon Sep 17 00:00:00 2001
From: Madhur Amilkanthwar <madhura at nvidia.com>
Date: Sun, 9 Nov 2025 20:53:20 -0800
Subject: [PATCH 02/13] fixup! [GVN] Support rnflow pattern matching and
transform
---
llvm/include/llvm/Transforms/Scalar/GVN.h | 9 +-
llvm/lib/Transforms/Scalar/GVN.cpp | 185 +++++++++---------
llvm/test/Transforms/GVN/PRE/gvn-min-pre.ll | 58 ++++++
.../test/Transforms/GVN/PRE/rnflow-gvn-pre.ll | 59 ------
4 files changed, 160 insertions(+), 151 deletions(-)
create mode 100644 llvm/test/Transforms/GVN/PRE/gvn-min-pre.ll
delete mode 100644 llvm/test/Transforms/GVN/PRE/rnflow-gvn-pre.ll
diff --git a/llvm/include/llvm/Transforms/Scalar/GVN.h b/llvm/include/llvm/Transforms/Scalar/GVN.h
index b886140348e79..d598803be3585 100644
--- a/llvm/include/llvm/Transforms/Scalar/GVN.h
+++ b/llvm/include/llvm/Transforms/Scalar/GVN.h
@@ -407,8 +407,13 @@ class GVNPass : public PassInfoMixin<GVNPass> {
void addDeadBlock(BasicBlock *BB);
void assignValNumForDeadCode();
void assignBlockRPONumber(Function &F);
-
- bool optimizeMinMaxFindingSelectPattern(SelectInst *Select);
+
+ bool recognizeMinFindingSelectPattern(SelectInst *Select);
+ bool transformMinFindingSelectPattern(Loop *L, BasicBlock *Preheader,
+ BasicBlock *BB, Value *LHS, Value *RHS,
+ CmpInst *Comparison, SelectInst *Select,
+ Value *BasePtr, Value *IndexVal,
+ Value *OffsetVal);
};
/// Create a legacy GVN pass.
diff --git a/llvm/lib/Transforms/Scalar/GVN.cpp b/llvm/lib/Transforms/Scalar/GVN.cpp
index cf00105d5711a..cf14f8f0bf601 100644
--- a/llvm/lib/Transforms/Scalar/GVN.cpp
+++ b/llvm/lib/Transforms/Scalar/GVN.cpp
@@ -2747,7 +2747,7 @@ bool GVNPass::processInstruction(Instruction *I) {
return Changed;
}
if (SelectInst *Select = dyn_cast<SelectInst>(I)) {
- if (optimizeMinMaxFindingSelectPattern(Select))
+ if (recognizeMinFindingSelectPattern(Select))
return true;
}
@@ -3337,19 +3337,99 @@ void GVNPass::assignValNumForDeadCode() {
}
}
-bool GVNPass::optimizeMinMaxFindingSelectPattern(SelectInst *Select) {
+bool GVNPass::transformMinFindingSelectPattern(Loop *L, BasicBlock *Preheader,
+ BasicBlock *BB, Value *LHS,
+ Value *RHS, CmpInst *Comparison,
+ SelectInst *Select,
+ Value *BasePtr, Value *IndexVal,
+ Value *OffsetVal) {
+ // Hoist the chain of operations for the second load to preheader.
+ // %min.idx.ext = sext i32 %min.idx to i64
+ // %ptr.float.min = getelementptr float, ptr %0, i64 %min.idx.ext
+ // %ptr.second.load = getelementptr i8, ptr %ptr.float.min, i64 -4
+ // %val.current.min = load float, ptr %ptr.second.load, align 4
+ IRBuilder<> Builder(Preheader->getTerminator());
+
+ PHINode *Phi = dyn_cast<PHINode>(IndexVal);
+ if (!Phi) {
+ LLVM_DEBUG(dbgs() << "GVN: IndexVal is not a PHI node\n");
+ return false;
+ }
+
+ Value *InitialMinIndex = Phi->getIncomingValueForBlock(Preheader);
+
+ // Insert PHI node at the top of this block.
+ // This PHI node will be used to memoize the current minimum value so far.
+ PHINode *KnownMinPhi =
+ PHINode::Create(Builder.getFloatTy(), 2, "known_min", BB->begin());
+
+ // Hoist the load and build the necessary operations.
+ // 1. hoist_0 = sext i32 to i64
+ Value *HoistedSExt =
+ Builder.CreateSExt(InitialMinIndex, Builder.getInt64Ty(), "hoist_sext");
+
+ // 2. hoist_gep1 = getelementptr float, ptr BasePtr, i64 HoistedSExt
+ Value *HoistedGEP1 = Builder.CreateGEP(Builder.getFloatTy(), BasePtr,
+ HoistedSExt, "hoist_gep1");
+
+ // 3. hoist_gep2 = getelementptr i8, ptr HoistedGEP1, i64 OffsetVal
+ Value *HoistedGEP2 = Builder.CreateGEP(Builder.getInt8Ty(), HoistedGEP1,
+ OffsetVal, "hoist_gep2");
+
+ // 4. hoisted_load = load float, ptr HoistedGEP2
+ LoadInst *NewLoad =
+ Builder.CreateLoad(Builder.getFloatTy(), HoistedGEP2, "hoisted_load");
+
+ // Let the new load now take the place of the old load.
+ RHS->replaceAllUsesWith(NewLoad);
+ dyn_cast<LoadInst>(RHS)->eraseFromParent();
+
+ // Comparison should now compare the current value and the newly inserted
+ // PHI node.
+ Comparison->setOperand(1, KnownMinPhi);
+
+ // Create new select instruction for selecting the minimum value.
+ IRBuilder<> SelectBuilder(BB->getTerminator());
+ SelectInst *CurrentMinSelect = dyn_cast<SelectInst>(
+ SelectBuilder.CreateSelect(Comparison, LHS, KnownMinPhi, "current_min"));
+
+ // Populate the newly created PHI node
+ // with (hoisted) NewLoad from the preheader and CurrentMinSelect.
+ KnownMinPhi->addIncoming(NewLoad, Preheader);
+ KnownMinPhi->addIncoming(CurrentMinSelect, BB);
+ LLVM_DEBUG(dbgs() << "Transformed the code\n");
+ return true;
+}
+
+bool GVNPass::recognizeMinFindingSelectPattern(SelectInst *Select) {
+ Value *BasePtr = nullptr, *IndexVal = nullptr, *OffsetVal = nullptr;
LLVM_DEBUG(
dbgs()
- << "GVN: Analyzing select instruction for minimum finding pattern\n");
+ << "GVN: Analyzing select instruction for minimum finding pattern.\n");
LLVM_DEBUG(dbgs() << "GVN: Select: " << *Select << "\n");
+ BasicBlock *BB = Select->getParent();
+
+ // If the block is not in a loop, bail out.
+ Loop *L = LI->getLoopFor(BB);
+ if (!L) {
+ LLVM_DEBUG(dbgs() << "GVN: Could not find loop.\n");
+ return false;
+ }
+
+ // If preheader of the loop is not found, bail out.
+ BasicBlock *Preheader = L->getLoopPreheader();
+ if (!Preheader) {
+ LLVM_DEBUG(dbgs() << "GVN: Could not find loop preheader.\n");
+ return false;
+ }
Value *Condition = Select->getCondition();
CmpInst *Comparison = dyn_cast<CmpInst>(Condition);
if (!Comparison) {
- LLVM_DEBUG(dbgs() << "GVN: Condition is not a comparison\n");
+ LLVM_DEBUG(dbgs() << "GVN: Condition is not a comparison.\n");
return false;
}
- // Check if this is ULT comparison.
+ // Check if this is less-than comparison.
CmpInst::Predicate Pred = Comparison->getPredicate();
if (Pred != CmpInst::ICMP_SLT && Pred != CmpInst::ICMP_ULT &&
Pred != CmpInst::FCMP_OLT && Pred != CmpInst::FCMP_ULT) {
@@ -3362,99 +3442,24 @@ bool GVNPass::optimizeMinMaxFindingSelectPattern(SelectInst *Select) {
Value *LHS = Comparison->getOperand(0);
Value *RHS = Comparison->getOperand(1);
if (!isa<LoadInst>(LHS) || !isa<LoadInst>(RHS)) {
- LLVM_DEBUG(dbgs() << "GVN: Not both operands are loads\n");
+ LLVM_DEBUG(dbgs() << "GVN: Not both operands are loads.\n");
return false;
}
- LLVM_DEBUG(dbgs() << "GVN: Found minimum finding pattern in Block: "
- << Select->getParent()->getName() << "\n");
-
- // Transform the pattern.
- // Hoist the chain of operations for the second load to preheader.
- // Get predecessor of the block containing the select instruction.
- BasicBlock *BB = Select->getParent();
-
- // Get preheader of the loop.
- Loop *L = LI->getLoopFor(BB);
- if (!L) {
- LLVM_DEBUG(dbgs() << "GVN: Could not find loop\n");
- return false;
- }
- BasicBlock *Preheader = L->getLoopPreheader();
- if (!Preheader) {
- LLVM_DEBUG(dbgs() << "GVN: Could not find loop preheader\n");
+ if (!match(RHS,
+ m_Load(m_GEP(m_GEP(m_Value(BasePtr), m_SExt(m_Value(IndexVal))),
+ m_Value(OffsetVal))))) {
+ LLVM_DEBUG(dbgs() << "GVN: Not a required load pattern.\n");
return false;
}
+ LLVM_DEBUG(dbgs() << "GVN: Found minimum finding pattern in Block: "
+ << Select->getParent()->getName() << ".\n");
- // Hoist the chain of operations for the second load to preheader.
- // %90 = sext i32 %.05.i to i64
- // %91 = getelementptr float, ptr %0, i64 %90 ; %0 + (sext i32 %85 to i64)*4
- // %92 = getelementptr i8, ptr %91, i64 -4 ; %0 + (sext i32 %85 to i64)*4 - 4
- // %93 = load float, ptr %92, align 4
-
- Value *BasePtr = nullptr, *IndexVal = nullptr, *OffsetVal = nullptr;
- IRBuilder<> Builder(Preheader->getTerminator());
- if (match(RHS,
- m_Load(m_GEP(m_GEP(m_Value(BasePtr), m_SExt(m_Value(IndexVal))),
- m_Value(OffsetVal))))) {
- LLVM_DEBUG(dbgs() << "GVN: Found pattern: " << *RHS << "\n");
- LLVM_DEBUG(dbgs() << "GVN: Found pattern: " << "\n");
-
- PHINode *Phi = dyn_cast<PHINode>(IndexVal);
- if (!Phi) {
- LLVM_DEBUG(dbgs() << "GVN: IndexVal is not a PHI node\n");
- return false;
- }
- Value *InitialMinIndex = Phi->getIncomingValueForBlock(Preheader);
-
- // Insert PHI node at the top of this block.
- PHINode *KnownMinPhi =
- PHINode::Create(Builder.getFloatTy(), 2, "known_min", BB->begin());
-
- // Build the GEP chain in the preheader.
- // 1. hoist_0 = sext i32 to i64
- Value *HoistedSExt =
- Builder.CreateSExt(InitialMinIndex, Builder.getInt64Ty(), "hoist_sext");
-
- // 2. hoist_gep1 = getelementptr float, ptr BasePtr, i64 HoistedSExt
- Value *HoistedGEP1 = Builder.CreateGEP(Builder.getFloatTy(), BasePtr,
- HoistedSExt, "hoist_gep1");
-
- // 3. hoist_gep2 = getelementptr i8, ptr HoistedGEP1, i64 OffsetVal
- Value *HoistedGEP2 = Builder.CreateGEP(Builder.getInt8Ty(), HoistedGEP1,
- OffsetVal, "hoist_gep2");
-
- // 4. hoisted_load = load float, ptr HoistedGEP2
- LoadInst *NewLoad =
- Builder.CreateLoad(Builder.getFloatTy(), HoistedGEP2, "hoisted_load");
-
- // Replace all uses of load with new load.
- RHS->replaceAllUsesWith(NewLoad);
- dyn_cast<LoadInst>(RHS)->eraseFromParent();
-
- // Replace second operand of comparison with KnownMinPhi.
- Comparison->setOperand(1, KnownMinPhi);
-
- // Create new select instruction for selecting the minimum value.
- IRBuilder<> SelectBuilder(BB->getTerminator());
- SelectInst *CurrentMinSelect =
- dyn_cast<SelectInst>(SelectBuilder.CreateSelect(
- Comparison, LHS, KnownMinPhi, "current_min"));
-
- // Populate PHI node.
- KnownMinPhi->addIncoming(NewLoad, Preheader);
- KnownMinPhi->addIncoming(CurrentMinSelect, BB);
- LLVM_DEBUG(dbgs() << "Transformed the code\n");
- return true;
- } else {
- LLVM_DEBUG(dbgs() << "GVN: Could not find pattern: " << *RHS << "\n");
- LLVM_DEBUG(dbgs() << "GVN: Could not find pattern: " << "\n");
- return false;
- }
- return false;
+ return transformMinFindingSelectPattern(L, Preheader, BB, LHS, RHS,
+ Comparison, Select, BasePtr, IndexVal,
+ OffsetVal);
}
-
class llvm::gvn::GVNLegacyPass : public FunctionPass {
public:
static char ID; // Pass identification, replacement for typeid.
diff --git a/llvm/test/Transforms/GVN/PRE/gvn-min-pre.ll b/llvm/test/Transforms/GVN/PRE/gvn-min-pre.ll
new file mode 100644
index 0000000000000..33ba2e383bdf6
--- /dev/null
+++ b/llvm/test/Transforms/GVN/PRE/gvn-min-pre.ll
@@ -0,0 +1,58 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -passes=gvn -S < %s | FileCheck %s
+
+define void @test_gvn_min_pattern(ptr %0) {
+; CHECK-LABEL: define void @test_gvn_min_pattern(
+; CHECK-SAME: ptr [[TMP0:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[HOIST_GEP1:%.*]] = getelementptr float, ptr [[TMP0]], i64 1
+; CHECK-NEXT: [[HOIST_GEP2:%.*]] = getelementptr i8, ptr [[HOIST_GEP1]], i64 -4
+; CHECK-NEXT: [[HOISTED_LOAD:%.*]] = load float, ptr [[HOIST_GEP2]], align 4
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[KNOWN_MIN:%.*]] = phi float [ [[HOISTED_LOAD]], %[[ENTRY]] ], [ [[CURRENT_MIN:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[INDVARS_IV_I:%.*]] = phi i64 [ 1, %[[ENTRY]] ], [ [[INDVARS_IV_NEXT_I:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[LOOP_COUNTER:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[LOOP_COUNTER_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MIN_IDX:%.*]] = phi i32 [ 1, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[INDVARS_IV_NEXT_I]] = add nsw i64 [[INDVARS_IV_I]], -1
+; CHECK-NEXT: [[PTR_FLOAT_IV:%.*]] = getelementptr float, ptr [[TMP0]], i64 [[INDVARS_IV_I]]
+; CHECK-NEXT: [[PTR_FIRST_LOAD:%.*]] = getelementptr i8, ptr [[PTR_FLOAT_IV]], i64 -8
+; CHECK-NEXT: [[VAL_FIRST:%.*]] = load float, ptr [[PTR_FIRST_LOAD]], align 4
+; CHECK-NEXT: [[MIN_IDX_EXT:%.*]] = sext i32 [[MIN_IDX]] to i64
+; CHECK-NEXT: [[PTR_FLOAT_MIN:%.*]] = getelementptr float, ptr [[TMP0]], i64 [[MIN_IDX_EXT]]
+; CHECK-NEXT: [[PTR_SECOND_LOAD:%.*]] = getelementptr i8, ptr [[PTR_FLOAT_MIN]], i64 -4
+; CHECK-NEXT: [[CMP:%.*]] = fcmp contract olt float [[VAL_FIRST]], [[KNOWN_MIN]]
+; CHECK-NEXT: [[NEXT_IDX_TRUNC:%.*]] = trunc nsw i64 [[INDVARS_IV_NEXT_I]] to i32
+; CHECK-NEXT: [[MIN_IDX_NEXT]] = select i1 [[CMP]], i32 [[NEXT_IDX_TRUNC]], i32 [[MIN_IDX]]
+; CHECK-NEXT: [[LOOP_COUNTER_NEXT]] = add nsw i64 [[LOOP_COUNTER]], -1
+; CHECK-NEXT: [[LOOP_CONTINUE:%.*]] = icmp samesign ugt i64 [[LOOP_COUNTER]], 1
+; CHECK-NEXT: [[CURRENT_MIN]] = select i1 [[CMP]], float [[VAL_FIRST]], float [[KNOWN_MIN]]
+; CHECK-NEXT: br i1 [[LOOP_CONTINUE]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop: ; preds = %loop, %entry
+ %indvars.iv.i = phi i64 [ 1, %entry ], [ %indvars.iv.next.i, %loop ]
+ %loop.counter = phi i64 [ 0, %entry ], [ %loop.counter.next, %loop ]
+ %min.idx = phi i32 [ 1, %entry ], [ %min.idx.next, %loop ]
+ %indvars.iv.next.i = add nsw i64 %indvars.iv.i, -1
+ %ptr.float.iv = getelementptr float, ptr %0, i64 %indvars.iv.i
+ %ptr.first.load = getelementptr i8, ptr %ptr.float.iv, i64 -8
+ %val.first = load float, ptr %ptr.first.load, align 4
+ %min.idx.ext = sext i32 %min.idx to i64
+ %ptr.float.min = getelementptr float, ptr %0, i64 %min.idx.ext
+ %ptr.second.load = getelementptr i8, ptr %ptr.float.min, i64 -4
+ %val.current.min = load float, ptr %ptr.second.load, align 4
+ %cmp = fcmp contract olt float %val.first, %val.current.min
+ %next.idx.trunc = trunc nsw i64 %indvars.iv.next.i to i32
+ %min.idx.next = select i1 %cmp, i32 %next.idx.trunc, i32 %min.idx
+ %loop.counter.next = add nsw i64 %loop.counter, -1
+ %loop.continue = icmp samesign ugt i64 %loop.counter, 1
+ br i1 %loop.continue, label %loop, label %exit
+
+exit:
+ ret void
+}
diff --git a/llvm/test/Transforms/GVN/PRE/rnflow-gvn-pre.ll b/llvm/test/Transforms/GVN/PRE/rnflow-gvn-pre.ll
deleted file mode 100644
index 6f17d4ab30240..0000000000000
--- a/llvm/test/Transforms/GVN/PRE/rnflow-gvn-pre.ll
+++ /dev/null
@@ -1,59 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
-; Minimal test case containing only the .lr.ph.i basic block
-; RUN: opt -passes=gvn -S < %s | FileCheck %s
-
-define void @test_lr_ph_i(ptr %0) {
-; CHECK-LABEL: define void @test_lr_ph_i(
-; CHECK-SAME: ptr [[TMP0:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[HOIST_GEP1:%.*]] = getelementptr float, ptr [[TMP0]], i64 1
-; CHECK-NEXT: [[HOIST_GEP2:%.*]] = getelementptr i8, ptr [[HOIST_GEP1]], i64 -4
-; CHECK-NEXT: [[HOISTED_LOAD:%.*]] = load float, ptr [[HOIST_GEP2]], align 4
-; CHECK-NEXT: br label %[[DOTLR_PH_I:.*]]
-; CHECK: [[_LR_PH_I:.*:]]
-; CHECK-NEXT: [[KNOWN_MIN:%.*]] = phi float [ [[HOISTED_LOAD]], %[[ENTRY]] ], [ [[CURRENT_MIN:%.*]], %[[DOTLR_PH_I]] ]
-; CHECK-NEXT: [[INDVARS_IV_I:%.*]] = phi i64 [ 1, %[[ENTRY]] ], [ [[INDVARS_IV_NEXT_I:%.*]], %[[DOTLR_PH_I]] ]
-; CHECK-NEXT: [[TMP1:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[TMP10:%.*]], %[[DOTLR_PH_I]] ]
-; CHECK-NEXT: [[DOT05_I:%.*]] = phi i32 [ 1, %[[ENTRY]] ], [ [[DOT1_I:%.*]], %[[DOTLR_PH_I]] ]
-; CHECK-NEXT: [[INDVARS_IV_NEXT_I]] = add nsw i64 [[INDVARS_IV_I]], -1
-; CHECK-NEXT: [[TMP2:%.*]] = getelementptr float, ptr [[TMP0]], i64 [[INDVARS_IV_I]]
-; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr [[TMP2]], i64 -8
-; CHECK-NEXT: [[TMP4:%.*]] = load float, ptr [[TMP3]], align 4
-; CHECK-NEXT: [[TMP5:%.*]] = sext i32 [[DOT05_I]] to i64
-; CHECK-NEXT: [[TMP6:%.*]] = getelementptr float, ptr [[TMP0]], i64 [[TMP5]]
-; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[TMP6]], i64 -4
-; CHECK-NEXT: [[TMP8:%.*]] = fcmp contract olt float [[TMP4]], [[KNOWN_MIN]]
-; CHECK-NEXT: [[TMP9:%.*]] = trunc nsw i64 [[INDVARS_IV_NEXT_I]] to i32
-; CHECK-NEXT: [[DOT1_I]] = select i1 [[TMP8]], i32 [[TMP9]], i32 [[DOT05_I]]
-; CHECK-NEXT: [[TMP10]] = add nsw i64 [[TMP1]], -1
-; CHECK-NEXT: [[TMP11:%.*]] = icmp samesign ugt i64 [[TMP1]], 1
-; CHECK-NEXT: [[CURRENT_MIN]] = select i1 [[TMP8]], float [[TMP4]], float [[KNOWN_MIN]]
-; CHECK-NEXT: br i1 [[TMP11]], label %[[DOTLR_PH_I]], label %[[EXIT:.*]]
-; CHECK: [[EXIT]]:
-; CHECK-NEXT: ret void
-;
-entry:
- br label %.lr.ph.i
-
-.lr.ph.i: ; preds = %.lr.ph.i, %entry
- %indvars.iv.i = phi i64 [ 1, %entry ], [ %indvars.iv.next.i, %.lr.ph.i ]
- %86 = phi i64 [ 0, %entry ], [ %96, %.lr.ph.i ]
- %.05.i = phi i32 [ 1, %entry ], [ %.1.i, %.lr.ph.i ]
- %indvars.iv.next.i = add nsw i64 %indvars.iv.i, -1
- %87 = getelementptr float, ptr %0, i64 %indvars.iv.i
- %88 = getelementptr i8, ptr %87, i64 -8 ; first load : %0 + 4 * 1 - 8
- %89 = load float, ptr %88, align 4
- %90 = sext i32 %.05.i to i64
- %91 = getelementptr float, ptr %0, i64 %90 ; %0 + 4 * 1
- %92 = getelementptr i8, ptr %91, i64 -4 ; second load : %0 + 4 * 1 - 4
- %93 = load float, ptr %92, align 4
- %94 = fcmp contract olt float %89, %93
- %95 = trunc nsw i64 %indvars.iv.next.i to i32
- %.1.i = select i1 %94, i32 %95, i32 %.05.i
- %96 = add nsw i64 %86, -1
- %97 = icmp samesign ugt i64 %86, 1
- br i1 %97, label %.lr.ph.i, label %exit
-
-exit:
- ret void
-}
>From 740539218f78634aabe4b6563d399d2f3a5bb53e Mon Sep 17 00:00:00 2001
From: Madhur Amilkanthwar <madhura at nvidia.com>
Date: Thu, 13 Nov 2025 01:58:25 -0800
Subject: [PATCH 03/13] fixup! generalize load type.
---
llvm/include/llvm/Transforms/Scalar/GVN.h | 2 +-
llvm/lib/Transforms/Scalar/GVN.cpp | 13 ++++++++-----
llvm/test/Transforms/GVN/PRE/gvn-min-pre.ll | 4 ++--
3 files changed, 11 insertions(+), 8 deletions(-)
diff --git a/llvm/include/llvm/Transforms/Scalar/GVN.h b/llvm/include/llvm/Transforms/Scalar/GVN.h
index d598803be3585..3a56938f60d45 100644
--- a/llvm/include/llvm/Transforms/Scalar/GVN.h
+++ b/llvm/include/llvm/Transforms/Scalar/GVN.h
@@ -409,7 +409,7 @@ class GVNPass : public PassInfoMixin<GVNPass> {
void assignBlockRPONumber(Function &F);
bool recognizeMinFindingSelectPattern(SelectInst *Select);
- bool transformMinFindingSelectPattern(Loop *L, BasicBlock *Preheader,
+ bool transformMinFindingSelectPattern(Loop *L, Type *LoadType, BasicBlock *Preheader,
BasicBlock *BB, Value *LHS, Value *RHS,
CmpInst *Comparison, SelectInst *Select,
Value *BasePtr, Value *IndexVal,
diff --git a/llvm/lib/Transforms/Scalar/GVN.cpp b/llvm/lib/Transforms/Scalar/GVN.cpp
index cf14f8f0bf601..70a95b52edec1 100644
--- a/llvm/lib/Transforms/Scalar/GVN.cpp
+++ b/llvm/lib/Transforms/Scalar/GVN.cpp
@@ -3337,7 +3337,7 @@ void GVNPass::assignValNumForDeadCode() {
}
}
-bool GVNPass::transformMinFindingSelectPattern(Loop *L, BasicBlock *Preheader,
+bool GVNPass::transformMinFindingSelectPattern(Loop *L, Type *LoadType, BasicBlock *Preheader,
BasicBlock *BB, Value *LHS,
Value *RHS, CmpInst *Comparison,
SelectInst *Select,
@@ -3361,7 +3361,7 @@ bool GVNPass::transformMinFindingSelectPattern(Loop *L, BasicBlock *Preheader,
// Insert PHI node at the top of this block.
// This PHI node will be used to memoize the current minimum value so far.
PHINode *KnownMinPhi =
- PHINode::Create(Builder.getFloatTy(), 2, "known_min", BB->begin());
+ PHINode::Create(LoadType, 2, "known_min", BB->begin());
// Hoist the load and build the necessary operations.
// 1. hoist_0 = sext i32 to i64
@@ -3369,7 +3369,7 @@ bool GVNPass::transformMinFindingSelectPattern(Loop *L, BasicBlock *Preheader,
Builder.CreateSExt(InitialMinIndex, Builder.getInt64Ty(), "hoist_sext");
// 2. hoist_gep1 = getelementptr float, ptr BasePtr, i64 HoistedSExt
- Value *HoistedGEP1 = Builder.CreateGEP(Builder.getFloatTy(), BasePtr,
+ Value *HoistedGEP1 = Builder.CreateGEP(LoadType, BasePtr,
HoistedSExt, "hoist_gep1");
// 3. hoist_gep2 = getelementptr i8, ptr HoistedGEP1, i64 OffsetVal
@@ -3378,7 +3378,7 @@ bool GVNPass::transformMinFindingSelectPattern(Loop *L, BasicBlock *Preheader,
// 4. hoisted_load = load float, ptr HoistedGEP2
LoadInst *NewLoad =
- Builder.CreateLoad(Builder.getFloatTy(), HoistedGEP2, "hoisted_load");
+ Builder.CreateLoad(LoadType, HoistedGEP2, "hoisted_load");
// Let the new load now take the place of the old load.
RHS->replaceAllUsesWith(NewLoad);
@@ -3455,7 +3455,10 @@ bool GVNPass::recognizeMinFindingSelectPattern(SelectInst *Select) {
LLVM_DEBUG(dbgs() << "GVN: Found minimum finding pattern in Block: "
<< Select->getParent()->getName() << ".\n");
- return transformMinFindingSelectPattern(L, Preheader, BB, LHS, RHS,
+ // Get type of load.
+ Type *LoadType = dyn_cast<LoadInst>(LHS)->getType();
+ LLVM_DEBUG(dbgs() << "GVN: Transforming minimum finding pattern.\n");
+ return transformMinFindingSelectPattern(L, LoadType,Preheader, BB, LHS, RHS,
Comparison, Select, BasePtr, IndexVal,
OffsetVal);
}
diff --git a/llvm/test/Transforms/GVN/PRE/gvn-min-pre.ll b/llvm/test/Transforms/GVN/PRE/gvn-min-pre.ll
index 33ba2e383bdf6..19fec514b28fe 100644
--- a/llvm/test/Transforms/GVN/PRE/gvn-min-pre.ll
+++ b/llvm/test/Transforms/GVN/PRE/gvn-min-pre.ll
@@ -40,11 +40,11 @@ loop: ; preds = %loop, %entry
%min.idx = phi i32 [ 1, %entry ], [ %min.idx.next, %loop ]
%indvars.iv.next.i = add nsw i64 %indvars.iv.i, -1
%ptr.float.iv = getelementptr float, ptr %0, i64 %indvars.iv.i
- %ptr.first.load = getelementptr i8, ptr %ptr.float.iv, i64 -8
+ %ptr.first.load = getelementptr i8, ptr %ptr.float.iv, i64 -8
%val.first = load float, ptr %ptr.first.load, align 4
%min.idx.ext = sext i32 %min.idx to i64
%ptr.float.min = getelementptr float, ptr %0, i64 %min.idx.ext
- %ptr.second.load = getelementptr i8, ptr %ptr.float.min, i64 -4
+ %ptr.second.load = getelementptr i8, ptr %ptr.float.min, i64 -4
%val.current.min = load float, ptr %ptr.second.load, align 4
%cmp = fcmp contract olt float %val.first, %val.current.min
%next.idx.trunc = trunc nsw i64 %indvars.iv.next.i to i32
>From 1f08a06465427bfd56dea32bfb51e4ffcb77eaaa Mon Sep 17 00:00:00 2001
From: Madhur Amilkanthwar <madhura at nvidia.com>
Date: Thu, 13 Nov 2025 01:59:49 -0800
Subject: [PATCH 04/13] fixup! clang-format
---
llvm/include/llvm/Transforms/Scalar/GVN.h | 5 +++--
llvm/lib/Transforms/Scalar/GVN.cpp | 22 +++++++++-------------
2 files changed, 12 insertions(+), 15 deletions(-)
diff --git a/llvm/include/llvm/Transforms/Scalar/GVN.h b/llvm/include/llvm/Transforms/Scalar/GVN.h
index 3a56938f60d45..c35f9a03112cb 100644
--- a/llvm/include/llvm/Transforms/Scalar/GVN.h
+++ b/llvm/include/llvm/Transforms/Scalar/GVN.h
@@ -409,8 +409,9 @@ class GVNPass : public PassInfoMixin<GVNPass> {
void assignBlockRPONumber(Function &F);
bool recognizeMinFindingSelectPattern(SelectInst *Select);
- bool transformMinFindingSelectPattern(Loop *L, Type *LoadType, BasicBlock *Preheader,
- BasicBlock *BB, Value *LHS, Value *RHS,
+ bool transformMinFindingSelectPattern(Loop *L, Type *LoadType,
+ BasicBlock *Preheader, BasicBlock *BB,
+ Value *LHS, Value *RHS,
CmpInst *Comparison, SelectInst *Select,
Value *BasePtr, Value *IndexVal,
Value *OffsetVal);
diff --git a/llvm/lib/Transforms/Scalar/GVN.cpp b/llvm/lib/Transforms/Scalar/GVN.cpp
index 70a95b52edec1..3c6d6cdc91303 100644
--- a/llvm/lib/Transforms/Scalar/GVN.cpp
+++ b/llvm/lib/Transforms/Scalar/GVN.cpp
@@ -3337,12 +3337,10 @@ void GVNPass::assignValNumForDeadCode() {
}
}
-bool GVNPass::transformMinFindingSelectPattern(Loop *L, Type *LoadType, BasicBlock *Preheader,
- BasicBlock *BB, Value *LHS,
- Value *RHS, CmpInst *Comparison,
- SelectInst *Select,
- Value *BasePtr, Value *IndexVal,
- Value *OffsetVal) {
+bool GVNPass::transformMinFindingSelectPattern(
+ Loop *L, Type *LoadType, BasicBlock *Preheader, BasicBlock *BB, Value *LHS,
+ Value *RHS, CmpInst *Comparison, SelectInst *Select, Value *BasePtr,
+ Value *IndexVal, Value *OffsetVal) {
// Hoist the chain of operations for the second load to preheader.
// %min.idx.ext = sext i32 %min.idx to i64
// %ptr.float.min = getelementptr float, ptr %0, i64 %min.idx.ext
@@ -3360,8 +3358,7 @@ bool GVNPass::transformMinFindingSelectPattern(Loop *L, Type *LoadType, BasicBlo
// Insert PHI node at the top of this block.
// This PHI node will be used to memoize the current minimum value so far.
- PHINode *KnownMinPhi =
- PHINode::Create(LoadType, 2, "known_min", BB->begin());
+ PHINode *KnownMinPhi = PHINode::Create(LoadType, 2, "known_min", BB->begin());
// Hoist the load and build the necessary operations.
// 1. hoist_0 = sext i32 to i64
@@ -3369,16 +3366,15 @@ bool GVNPass::transformMinFindingSelectPattern(Loop *L, Type *LoadType, BasicBlo
Builder.CreateSExt(InitialMinIndex, Builder.getInt64Ty(), "hoist_sext");
// 2. hoist_gep1 = getelementptr float, ptr BasePtr, i64 HoistedSExt
- Value *HoistedGEP1 = Builder.CreateGEP(LoadType, BasePtr,
- HoistedSExt, "hoist_gep1");
+ Value *HoistedGEP1 =
+ Builder.CreateGEP(LoadType, BasePtr, HoistedSExt, "hoist_gep1");
// 3. hoist_gep2 = getelementptr i8, ptr HoistedGEP1, i64 OffsetVal
Value *HoistedGEP2 = Builder.CreateGEP(Builder.getInt8Ty(), HoistedGEP1,
OffsetVal, "hoist_gep2");
// 4. hoisted_load = load float, ptr HoistedGEP2
- LoadInst *NewLoad =
- Builder.CreateLoad(LoadType, HoistedGEP2, "hoisted_load");
+ LoadInst *NewLoad = Builder.CreateLoad(LoadType, HoistedGEP2, "hoisted_load");
// Let the new load now take the place of the old load.
RHS->replaceAllUsesWith(NewLoad);
@@ -3458,7 +3454,7 @@ bool GVNPass::recognizeMinFindingSelectPattern(SelectInst *Select) {
// Get type of load.
Type *LoadType = dyn_cast<LoadInst>(LHS)->getType();
LLVM_DEBUG(dbgs() << "GVN: Transforming minimum finding pattern.\n");
- return transformMinFindingSelectPattern(L, LoadType,Preheader, BB, LHS, RHS,
+ return transformMinFindingSelectPattern(L, LoadType, Preheader, BB, LHS, RHS,
Comparison, Select, BasePtr, IndexVal,
OffsetVal);
}
>From 069d2b819068864a5c4df7c53155c6826068c18c Mon Sep 17 00:00:00 2001
From: Madhur Amilkanthwar <madhura at nvidia.com>
Date: Wed, 10 Dec 2025 02:09:20 -0800
Subject: [PATCH 05/13] fixup! address review comments
1. Added 10+ negative tests.
2. Added positive tests for non-float types.
3. Strengthning checks in recognize function
4. clang-format changes
5. Addressed other review comments
---
llvm/include/llvm/Transforms/Scalar/GVN.h | 1 +
llvm/lib/Transforms/Scalar/GVN.cpp | 170 ++++-
llvm/test/Transforms/GVN/PRE/gvn-min-pre.ll | 676 +++++++++++++++++++-
3 files changed, 810 insertions(+), 37 deletions(-)
diff --git a/llvm/include/llvm/Transforms/Scalar/GVN.h b/llvm/include/llvm/Transforms/Scalar/GVN.h
index c35f9a03112cb..db74c2066013c 100644
--- a/llvm/include/llvm/Transforms/Scalar/GVN.h
+++ b/llvm/include/llvm/Transforms/Scalar/GVN.h
@@ -46,6 +46,7 @@ class FunctionPass;
class GetElementPtrInst;
class ImplicitControlFlowTracking;
class LoadInst;
+class Loop;
class SelectInst;
class LoopInfo;
class MemDepResult;
diff --git a/llvm/lib/Transforms/Scalar/GVN.cpp b/llvm/lib/Transforms/Scalar/GVN.cpp
index 3c6d6cdc91303..0a96b0fde49c9 100644
--- a/llvm/lib/Transforms/Scalar/GVN.cpp
+++ b/llvm/lib/Transforms/Scalar/GVN.cpp
@@ -34,7 +34,6 @@
#include "llvm/Analysis/InstructionPrecedenceTracking.h"
#include "llvm/Analysis/InstructionSimplify.h"
#include "llvm/Analysis/Loads.h"
-#include "llvm/Analysis/LoopInfo.h"
#include "llvm/Analysis/MemoryBuiltins.h"
#include "llvm/Analysis/MemoryDependenceAnalysis.h"
#include "llvm/Analysis/MemorySSA.h"
@@ -3337,31 +3336,64 @@ void GVNPass::assignValNumForDeadCode() {
}
}
+// Hoist the chain of operations for the second load to preheader.
+// In this transformation, we hoist the redundant load to the preheader,
+// caching the first value of the iteration. This value is used to compare with
+// the current value of the iteration and update the minimum value.
+// The comparison is done in the loop body using the new select instruction.
+//
+// *** Before transformation ***
+//
+// preheader:
+// ...
+// loop:
+// ...
+// ...
+// %val.first = load <TYPE>, ptr %ptr.first.load, align 4
+// %min.idx.ext = sext i32 %min.idx to i64
+// %ptr.<TYPE>.min = getelementptr <TYPE>, ptr %0, i64 %min.idx.ext
+// %ptr.second.load = getelementptr i8, ptr %ptr.<TYPE>.min, i64 -4
+// %val.current.min = load <TYPE>, ptr %ptr.second.load, align 4
+// ...
+// ...
+// br i1 %cond, label %loop, label %exit
+//
+// *** After transformation ***
+//
+// preheader:
+// %min.idx.ext = sext i32 %min.idx.ext to i64
+// %hoist_gep1 = getelementptr <TYPE>, ptr %0, i64 %min.idx.ext
+// %hoist_gep2 = getelementptr i8, ptr %hoist_gep1, i64 -4
+// %hoisted_load = load <TYPE>, ptr %hoist_gep2, align 4
+// br label %loop
+//
+// loop:
+// %val.first = load <TYPE>, ptr %ptr.first.load, align 4
+// ...
+// (new) %val.current.min = select i1 %cond, <TYPE> %hoisted_load, <TYPE>
+// %val.current.min
+// ...
+// ...
+// br i1 %cond, label %loop, label %exit
bool GVNPass::transformMinFindingSelectPattern(
Loop *L, Type *LoadType, BasicBlock *Preheader, BasicBlock *BB, Value *LHS,
- Value *RHS, CmpInst *Comparison, SelectInst *Select, Value *BasePtr,
+ Value *LoadVal, CmpInst *Comparison, SelectInst *Select, Value *BasePtr,
Value *IndexVal, Value *OffsetVal) {
- // Hoist the chain of operations for the second load to preheader.
- // %min.idx.ext = sext i32 %min.idx to i64
- // %ptr.float.min = getelementptr float, ptr %0, i64 %min.idx.ext
- // %ptr.second.load = getelementptr i8, ptr %ptr.float.min, i64 -4
- // %val.current.min = load float, ptr %ptr.second.load, align 4
- IRBuilder<> Builder(Preheader->getTerminator());
- PHINode *Phi = dyn_cast<PHINode>(IndexVal);
- if (!Phi) {
- LLVM_DEBUG(dbgs() << "GVN: IndexVal is not a PHI node\n");
- return false;
- }
+ assert(IndexVal && "IndexVal is null");
+ AAResults *AA = VN.getAliasAnalysis();
+ assert(AA && "AA is null");
- Value *InitialMinIndex = Phi->getIncomingValueForBlock(Preheader);
+ IRBuilder<> Builder(Preheader->getTerminator());
+ Value *InitialMinIndex =
+ dyn_cast<PHINode>(IndexVal)->getIncomingValueForBlock(Preheader);
// Insert PHI node at the top of this block.
// This PHI node will be used to memoize the current minimum value so far.
PHINode *KnownMinPhi = PHINode::Create(LoadType, 2, "known_min", BB->begin());
// Hoist the load and build the necessary operations.
- // 1. hoist_0 = sext i32 to i64
+ // 1. hoist_0 = sext i32 1 to i64
Value *HoistedSExt =
Builder.CreateSExt(InitialMinIndex, Builder.getInt64Ty(), "hoist_sext");
@@ -3373,12 +3405,40 @@ bool GVNPass::transformMinFindingSelectPattern(
Value *HoistedGEP2 = Builder.CreateGEP(Builder.getInt8Ty(), HoistedGEP1,
OffsetVal, "hoist_gep2");
+ MemoryLocation NewLoc = MemoryLocation(
+ HoistedGEP2,
+ LocationSize::precise(
+ L->getHeader()->getDataLayout().getTypeStoreSize(LoadType)));
+ // Check if any instruction in the loop clobbers this location.
+ bool CanHoist = true;
+ for (BasicBlock *BB : L->blocks()) {
+ for (Instruction &I : *BB) {
+ if (I.mayWriteToMemory()) {
+ // Check if this instruction may clobber our hoisted load.
+ ModRefInfo MRI = AA->getModRefInfo(&I, NewLoc);
+ if (isModOrRefSet(MRI)) {
+ LLVM_DEBUG(dbgs() << "GVN: Cannot hoist - may be clobbered by: " << I
+ << "\n");
+ CanHoist = false;
+ break;
+ }
+ }
+ }
+ if (!CanHoist)
+ break;
+ }
+ if (!CanHoist) {
+ LLVM_DEBUG(dbgs() << "GVN: Cannot hoist - may be clobbered by some "
+ "instruction in the loop.\n");
+ return false;
+ }
+
// 4. hoisted_load = load float, ptr HoistedGEP2
LoadInst *NewLoad = Builder.CreateLoad(LoadType, HoistedGEP2, "hoisted_load");
// Let the new load now take the place of the old load.
- RHS->replaceAllUsesWith(NewLoad);
- dyn_cast<LoadInst>(RHS)->eraseFromParent();
+ LoadVal->replaceAllUsesWith(NewLoad);
+ dyn_cast<LoadInst>(LoadVal)->eraseFromParent();
// Comparison should now compare the current value and the newly inserted
// PHI node.
@@ -3393,16 +3453,42 @@ bool GVNPass::transformMinFindingSelectPattern(
// with (hoisted) NewLoad from the preheader and CurrentMinSelect.
KnownMinPhi->addIncoming(NewLoad, Preheader);
KnownMinPhi->addIncoming(CurrentMinSelect, BB);
- LLVM_DEBUG(dbgs() << "Transformed the code\n");
+
+ if (MSSAU) {
+ auto *OrigUse =
+ MSSAU->getMemorySSA()->getMemoryAccess(dyn_cast<Instruction>(LoadVal));
+ if (OrigUse) {
+ MemoryAccess *DefiningAccess = OrigUse->getDefiningAccess();
+ MSSAU->createMemoryAccessInBB(NewLoad, DefiningAccess, Preheader,
+ MemorySSA::BeforeTerminator);
+ }
+ }
+ LLVM_DEBUG(
+ dbgs() << "GVN: Transformed the code for minimum finding pattern.\n");
return true;
}
+// We are looking for the following pattern:
+// loop:
+// ...
+// ...
+// %min.idx = phi i32 [ %initial_min_idx, %entry ], [ %min.idx.next, %loop ]
+// ...
+// %val.first = load <TYPE>, ptr %ptr.first.load, align 4
+// %min.idx.ext = sext i32 %min.idx to i64
+// %ptr.<TYPE>.min = getelementptr <TYPE>, ptr %0, i64 %min.idx.ext
+// %ptr.second.load = getelementptr i8, ptr %ptr.<TYPE>.min, i64 -4
+// %val.current.min = load <TYPE>, ptr %ptr.second.load, align 4
+// %cmp = <CMP_INST> <TYPE> %val.first, %val.current.min
+// ...
+// %min.idx.next = select i1 %cmp, ..., i32 %min.idx
+// ...
+// ...
+// br i1 ..., label %loop, ...
bool GVNPass::recognizeMinFindingSelectPattern(SelectInst *Select) {
- Value *BasePtr = nullptr, *IndexVal = nullptr, *OffsetVal = nullptr;
- LLVM_DEBUG(
- dbgs()
- << "GVN: Analyzing select instruction for minimum finding pattern.\n");
- LLVM_DEBUG(dbgs() << "GVN: Select: " << *Select << "\n");
+ IRBuilder<> Builder(Select);
+ Value *BasePtr = nullptr, *IndexVal = nullptr, *OffsetVal = nullptr,
+ *SExt = nullptr;
BasicBlock *BB = Select->getParent();
// If the block is not in a loop, bail out.
@@ -3442,21 +3528,41 @@ bool GVNPass::recognizeMinFindingSelectPattern(SelectInst *Select) {
return false;
}
- if (!match(RHS,
- m_Load(m_GEP(m_GEP(m_Value(BasePtr), m_SExt(m_Value(IndexVal))),
- m_Value(OffsetVal))))) {
+ if (!match(RHS, m_Load(m_GEP(m_GEP(m_Value(BasePtr), m_Value(SExt)),
+ m_Value(OffsetVal))))) {
LLVM_DEBUG(dbgs() << "GVN: Not a required load pattern.\n");
return false;
}
+ // Check if the SExt instruction is a sext instruction.
+ SExtInst *SEInst = dyn_cast<SExtInst>(SExt);
+ if (!SEInst) {
+ LLVM_DEBUG(dbgs() << "GVN: not a sext instruction.\n");
+ return false;
+ }
+ // Check if the "To" and "from" type of the sext instruction are i64 and i32
+ // respectively.
+ if (SEInst->getType() != Builder.getInt64Ty() ||
+ SEInst->getOperand(0)->getType() != Builder.getInt32Ty()) {
+ LLVM_DEBUG(
+ dbgs()
+ << "GVN: Not matching the required type for sext instruction.\n");
+ return false;
+ }
+
+ IndexVal = SEInst->getOperand(0);
+ // Check if the IndexVal is a PHI node.
+ PHINode *Phi = dyn_cast<PHINode>(IndexVal);
+ if (!Phi) {
+ LLVM_DEBUG(dbgs() << "GVN: IndexVal is not a PHI node\n");
+ return false;
+ }
+
LLVM_DEBUG(dbgs() << "GVN: Found minimum finding pattern in Block: "
<< Select->getParent()->getName() << ".\n");
- // Get type of load.
- Type *LoadType = dyn_cast<LoadInst>(LHS)->getType();
- LLVM_DEBUG(dbgs() << "GVN: Transforming minimum finding pattern.\n");
- return transformMinFindingSelectPattern(L, LoadType, Preheader, BB, LHS, RHS,
- Comparison, Select, BasePtr, IndexVal,
- OffsetVal);
+ return transformMinFindingSelectPattern(L, dyn_cast<LoadInst>(LHS)->getType(),
+ Preheader, BB, LHS, RHS, Comparison,
+ Select, BasePtr, IndexVal, OffsetVal);
}
class llvm::gvn::GVNLegacyPass : public FunctionPass {
diff --git a/llvm/test/Transforms/GVN/PRE/gvn-min-pre.ll b/llvm/test/Transforms/GVN/PRE/gvn-min-pre.ll
index 19fec514b28fe..1435417f80710 100644
--- a/llvm/test/Transforms/GVN/PRE/gvn-min-pre.ll
+++ b/llvm/test/Transforms/GVN/PRE/gvn-min-pre.ll
@@ -1,11 +1,14 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
; RUN: opt -passes=gvn -S < %s | FileCheck %s
-define void @test_gvn_min_pattern(ptr %0) {
+; Test the minimum finding pattern.
+; The following test case is extracted from rnflow app in Polyhedron benchmark suite.
+define void @test_gvn_min_pattern(ptr %0, i32 %initial_min_idx) {
; CHECK-LABEL: define void @test_gvn_min_pattern(
-; CHECK-SAME: ptr [[TMP0:%.*]]) {
+; CHECK-SAME: ptr [[TMP0:%.*]], i32 [[INITIAL_MIN_IDX:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[HOIST_GEP1:%.*]] = getelementptr float, ptr [[TMP0]], i64 1
+; CHECK-NEXT: [[HOIST_SEXT:%.*]] = sext i32 [[INITIAL_MIN_IDX]] to i64
+; CHECK-NEXT: [[HOIST_GEP1:%.*]] = getelementptr float, ptr [[TMP0]], i64 [[HOIST_SEXT]]
; CHECK-NEXT: [[HOIST_GEP2:%.*]] = getelementptr i8, ptr [[HOIST_GEP1]], i64 -4
; CHECK-NEXT: [[HOISTED_LOAD:%.*]] = load float, ptr [[HOIST_GEP2]], align 4
; CHECK-NEXT: br label %[[LOOP:.*]]
@@ -13,7 +16,7 @@ define void @test_gvn_min_pattern(ptr %0) {
; CHECK-NEXT: [[KNOWN_MIN:%.*]] = phi float [ [[HOISTED_LOAD]], %[[ENTRY]] ], [ [[CURRENT_MIN:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[INDVARS_IV_I:%.*]] = phi i64 [ 1, %[[ENTRY]] ], [ [[INDVARS_IV_NEXT_I:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[LOOP_COUNTER:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[LOOP_COUNTER_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[MIN_IDX:%.*]] = phi i32 [ 1, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MIN_IDX:%.*]] = phi i32 [ [[INITIAL_MIN_IDX]], %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[INDVARS_IV_NEXT_I]] = add nsw i64 [[INDVARS_IV_I]], -1
; CHECK-NEXT: [[PTR_FLOAT_IV:%.*]] = getelementptr float, ptr [[TMP0]], i64 [[INDVARS_IV_I]]
; CHECK-NEXT: [[PTR_FIRST_LOAD:%.*]] = getelementptr i8, ptr [[PTR_FLOAT_IV]], i64 -8
@@ -37,7 +40,7 @@ entry:
loop: ; preds = %loop, %entry
%indvars.iv.i = phi i64 [ 1, %entry ], [ %indvars.iv.next.i, %loop ]
%loop.counter = phi i64 [ 0, %entry ], [ %loop.counter.next, %loop ]
- %min.idx = phi i32 [ 1, %entry ], [ %min.idx.next, %loop ]
+ %min.idx = phi i32 [ %initial_min_idx, %entry ], [ %min.idx.next, %loop ]
%indvars.iv.next.i = add nsw i64 %indvars.iv.i, -1
%ptr.float.iv = getelementptr float, ptr %0, i64 %indvars.iv.i
%ptr.first.load = getelementptr i8, ptr %ptr.float.iv, i64 -8
@@ -56,3 +59,666 @@ loop: ; preds = %loop, %entry
exit:
ret void
}
+
+; Positive test: Minimum finding pattern with i32 loads.
+define void @test_gvn_min_pattern_i32(ptr %arr, i32 %initial_min_idx) {
+; CHECK-LABEL: define void @test_gvn_min_pattern_i32(
+; CHECK-SAME: ptr [[ARR:%.*]], i32 [[INITIAL_MIN_IDX:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[HOIST_SEXT:%.*]] = sext i32 [[INITIAL_MIN_IDX]] to i64
+; CHECK-NEXT: [[HOIST_GEP1:%.*]] = getelementptr i32, ptr [[ARR]], i64 [[HOIST_SEXT]]
+; CHECK-NEXT: [[HOIST_GEP2:%.*]] = getelementptr i8, ptr [[HOIST_GEP1]], i64 -4
+; CHECK-NEXT: [[HOISTED_LOAD:%.*]] = load i32, ptr [[HOIST_GEP2]], align 4
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[KNOWN_MIN:%.*]] = phi i32 [ [[HOISTED_LOAD]], %[[ENTRY]] ], [ [[CURRENT_MIN:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[INDVARS_IV_I:%.*]] = phi i64 [ 1, %[[ENTRY]] ], [ [[INDVARS_IV_NEXT_I:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[LOOP_COUNTER:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[LOOP_COUNTER_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MIN_IDX:%.*]] = phi i32 [ [[INITIAL_MIN_IDX]], %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[INDVARS_IV_NEXT_I]] = add nsw i64 [[INDVARS_IV_I]], -1
+; CHECK-NEXT: [[PTR_FLOAT_IV:%.*]] = getelementptr i32, ptr [[ARR]], i64 [[INDVARS_IV_I]]
+; CHECK-NEXT: [[PTR_FIRST_LOAD:%.*]] = getelementptr i8, ptr [[PTR_FLOAT_IV]], i64 -8
+; CHECK-NEXT: [[VAL_FIRST:%.*]] = load i32, ptr [[PTR_FIRST_LOAD]], align 4
+; CHECK-NEXT: [[MIN_IDX_EXT:%.*]] = sext i32 [[MIN_IDX]] to i64
+; CHECK-NEXT: [[PTR_FLOAT_MIN:%.*]] = getelementptr i32, ptr [[ARR]], i64 [[MIN_IDX_EXT]]
+; CHECK-NEXT: [[PTR_SECOND_LOAD:%.*]] = getelementptr i8, ptr [[PTR_FLOAT_MIN]], i64 -4
+; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[VAL_FIRST]], [[KNOWN_MIN]]
+; CHECK-NEXT: [[NEXT_IDX_TRUNC:%.*]] = trunc nsw i64 [[INDVARS_IV_NEXT_I]] to i32
+; CHECK-NEXT: [[MIN_IDX_NEXT]] = select i1 [[CMP]], i32 [[NEXT_IDX_TRUNC]], i32 [[MIN_IDX]]
+; CHECK-NEXT: [[LOOP_COUNTER_NEXT]] = add nsw i64 [[LOOP_COUNTER]], -1
+; CHECK-NEXT: [[LOOP_CONTINUE:%.*]] = icmp samesign ugt i64 [[LOOP_COUNTER]], 1
+; CHECK-NEXT: [[CURRENT_MIN]] = select i1 [[CMP]], i32 [[VAL_FIRST]], i32 [[KNOWN_MIN]]
+; CHECK-NEXT: br i1 [[LOOP_CONTINUE]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop: ; preds = %loop, %entry
+ %indvars.iv.i = phi i64 [ 1, %entry ], [ %indvars.iv.next.i, %loop ]
+ %loop.counter = phi i64 [ 0, %entry ], [ %loop.counter.next, %loop ]
+ %min.idx = phi i32 [ %initial_min_idx, %entry ], [ %min.idx.next, %loop ]
+ %indvars.iv.next.i = add nsw i64 %indvars.iv.i, -1
+ %ptr.i32.iv = getelementptr i32, ptr %arr, i64 %indvars.iv.i
+ %ptr.first.load = getelementptr i8, ptr %ptr.i32.iv, i64 -8
+ %val.first = load i32, ptr %ptr.first.load, align 4
+ %min.idx.ext = sext i32 %min.idx to i64
+ %ptr.i32.min = getelementptr i32, ptr %arr, i64 %min.idx.ext
+ %ptr.second.load = getelementptr i8, ptr %ptr.i32.min, i64 -4
+ %val.current.min = load i32, ptr %ptr.second.load, align 4
+ %cmp = icmp slt i32 %val.first, %val.current.min
+ %next.idx.trunc = trunc nsw i64 %indvars.iv.next.i to i32
+ %min.idx.next = select i1 %cmp, i32 %next.idx.trunc, i32 %min.idx
+ %loop.counter.next = add nsw i64 %loop.counter, -1
+ %loop.continue = icmp samesign ugt i64 %loop.counter, 1
+ br i1 %loop.continue, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+; Positive test: Minimum finding pattern with i64 loads.
+define void @test_gvn_min_pattern_i64(ptr %0, i32 %initial_min_idx) {
+; CHECK-LABEL: define void @test_gvn_min_pattern_i64(
+; CHECK-SAME: ptr [[TMP0:%.*]], i32 [[INITIAL_MIN_IDX:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[HOIST_SEXT:%.*]] = sext i32 [[INITIAL_MIN_IDX]] to i64
+; CHECK-NEXT: [[HOIST_GEP1:%.*]] = getelementptr i64, ptr [[TMP0]], i64 [[HOIST_SEXT]]
+; CHECK-NEXT: [[HOIST_GEP2:%.*]] = getelementptr i8, ptr [[HOIST_GEP1]], i64 -4
+; CHECK-NEXT: [[HOISTED_LOAD:%.*]] = load i64, ptr [[HOIST_GEP2]], align 4
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[KNOWN_MIN:%.*]] = phi i64 [ [[HOISTED_LOAD]], %[[ENTRY]] ], [ [[CURRENT_MIN:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[INDVARS_IV_I:%.*]] = phi i64 [ 1, %[[ENTRY]] ], [ [[INDVARS_IV_NEXT_I:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[LOOP_COUNTER:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[LOOP_COUNTER_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MIN_IDX:%.*]] = phi i32 [ [[INITIAL_MIN_IDX]], %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[INDVARS_IV_NEXT_I]] = add nsw i64 [[INDVARS_IV_I]], -1
+; CHECK-NEXT: [[PTR_I64_IV:%.*]] = getelementptr i64, ptr [[TMP0]], i64 [[INDVARS_IV_I]]
+; CHECK-NEXT: [[PTR_FIRST_LOAD:%.*]] = getelementptr i8, ptr [[PTR_I64_IV]], i64 -8
+; CHECK-NEXT: [[VAL_FIRST:%.*]] = load i64, ptr [[PTR_FIRST_LOAD]], align 4
+; CHECK-NEXT: [[MIN_IDX_EXT:%.*]] = sext i32 [[MIN_IDX]] to i64
+; CHECK-NEXT: [[PTR_I64_MIN:%.*]] = getelementptr i64, ptr [[TMP0]], i64 [[MIN_IDX_EXT]]
+; CHECK-NEXT: [[PTR_SECOND_LOAD:%.*]] = getelementptr i8, ptr [[PTR_I64_MIN]], i64 -4
+; CHECK-NEXT: [[CMP:%.*]] = icmp slt i64 [[VAL_FIRST]], [[KNOWN_MIN]]
+; CHECK-NEXT: [[NEXT_IDX_TRUNC:%.*]] = trunc nsw i64 [[INDVARS_IV_NEXT_I]] to i32
+; CHECK-NEXT: [[MIN_IDX_NEXT]] = select i1 [[CMP]], i32 [[NEXT_IDX_TRUNC]], i32 [[MIN_IDX]]
+; CHECK-NEXT: [[LOOP_COUNTER_NEXT]] = add nsw i64 [[LOOP_COUNTER]], -1
+; CHECK-NEXT: [[LOOP_CONTINUE:%.*]] = icmp samesign ugt i64 [[LOOP_COUNTER]], 1
+; CHECK-NEXT: [[CURRENT_MIN]] = select i1 [[CMP]], i64 [[VAL_FIRST]], i64 [[KNOWN_MIN]]
+; CHECK-NEXT: br i1 [[LOOP_CONTINUE]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop: ; preds = %loop, %entry
+ %indvars.iv.i = phi i64 [ 1, %entry ], [ %indvars.iv.next.i, %loop ]
+ %loop.counter = phi i64 [ 0, %entry ], [ %loop.counter.next, %loop ]
+ %min.idx = phi i32 [ %initial_min_idx, %entry ], [ %min.idx.next, %loop ]
+ %indvars.iv.next.i = add nsw i64 %indvars.iv.i, -1
+ %ptr.i64.iv = getelementptr i64, ptr %0, i64 %indvars.iv.i
+ %ptr.first.load = getelementptr i8, ptr %ptr.i64.iv, i64 -8
+ %val.first = load i64, ptr %ptr.first.load, align 4
+ %min.idx.ext = sext i32 %min.idx to i64
+ %ptr.i64.min = getelementptr i64, ptr %0, i64 %min.idx.ext
+ %ptr.second.load = getelementptr i8, ptr %ptr.i64.min, i64 -4
+ %val.current.min = load i64, ptr %ptr.second.load, align 4
+ %cmp = icmp slt i64 %val.first, %val.current.min
+ %next.idx.trunc = trunc nsw i64 %indvars.iv.next.i to i32
+ %min.idx.next = select i1 %cmp, i32 %next.idx.trunc, i32 %min.idx
+ %loop.counter.next = add nsw i64 %loop.counter, -1
+ %loop.continue = icmp samesign ugt i64 %loop.counter, 1
+ br i1 %loop.continue, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+; Negative test: Select not in a loop.
+define void @test_not_in_loop(ptr %arr) {
+; CHECK-LABEL: define void @test_not_in_loop(
+; CHECK-SAME: ptr [[ARR:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[LOAD1:%.*]] = load float, ptr [[ARR]], align 4
+; CHECK-NEXT: [[GEP2:%.*]] = getelementptr float, ptr [[ARR]], i64 1
+; CHECK-NEXT: [[LOAD2:%.*]] = load float, ptr [[GEP2]], align 4
+; CHECK-NEXT: [[CMP:%.*]] = fcmp olt float [[LOAD1]], [[LOAD2]]
+; CHECK-NEXT: [[SEL:%.*]] = select i1 [[CMP]], float [[LOAD1]], float [[LOAD2]]
+; CHECK-NEXT: ret void
+;
+entry:
+ %gep1 = getelementptr float, ptr %arr, i32 0
+ %load1 = load float, ptr %gep1
+ %sext = sext i32 1 to i64
+ %gep2 = getelementptr float, ptr %arr, i64 %sext
+ %gep3 = getelementptr i8, ptr %gep2, i64 0
+ %load2 = load float, ptr %gep3
+ %cmp = fcmp olt float %load1, %load2
+ %sel = select i1 %cmp, float %load1, float %load2
+ ret void
+}
+
+; Negative test: Loop without preheader (multiple entry points).
+define void @test_no_preheader(ptr %arr, i32 %n, i1 %cond) {
+; CHECK-LABEL: define void @test_no_preheader(
+; CHECK-SAME: ptr [[ARR:%.*]], i32 [[N:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br i1 [[COND]], label %[[LOOP:.*]], label %[[OTHER_ENTRY:.*]]
+; CHECK: [[OTHER_ENTRY]]:
+; CHECK-NEXT: br label %[[LOOP]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ 0, %[[OTHER_ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[GEP1:%.*]] = getelementptr float, ptr [[ARR]], i32 [[I]]
+; CHECK-NEXT: [[SEXT:%.*]] = sext i32 [[I]] to i64
+; CHECK-NEXT: [[GEP2:%.*]] = getelementptr float, ptr [[ARR]], i64 [[SEXT]]
+; CHECK-NEXT: [[I_NEXT]] = add i32 [[I]], 1
+; CHECK-NEXT: [[EXIT:%.*]] = icmp slt i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EXIT]], label %[[LOOP]], label %[[EXIT_BLOCK:.*]]
+; CHECK: [[EXIT_BLOCK]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br i1 %cond, label %loop, label %other_entry
+
+other_entry:
+ br label %loop
+
+loop:
+ %i = phi i32 [ 0, %entry ], [ 0, %other_entry ], [ %i.next, %loop ]
+ %gep1 = getelementptr float, ptr %arr, i32 %i
+ %load1 = load float, ptr %gep1
+ %sext = sext i32 %i to i64
+ %gep2 = getelementptr float, ptr %arr, i64 %sext
+ %gep3 = getelementptr i8, ptr %gep2, i64 0
+ %load2 = load float, ptr %gep3
+ %cmp = fcmp olt float %load1, %load2
+ %sel = select i1 %cmp, float %load1, float %load2
+ %i.next = add i32 %i, 1
+ %exit = icmp slt i32 %i.next, %n
+ br i1 %exit, label %loop, label %exit_block
+
+exit_block:
+ ret void
+}
+
+; Negative test: Condition is not a CmpInst.
+define void @test_condition_not_cmp(ptr %arr, i32 %n, i1 %bool) {
+; CHECK-LABEL: define void @test_condition_not_cmp(
+; CHECK-SAME: ptr [[ARR:%.*]], i32 [[N:%.*]], i1 [[BOOL:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[GEP1:%.*]] = getelementptr float, ptr [[ARR]], i32 [[I]]
+; CHECK-NEXT: [[SEXT:%.*]] = sext i32 [[I]] to i64
+; CHECK-NEXT: [[GEP2:%.*]] = getelementptr float, ptr [[ARR]], i64 [[SEXT]]
+; CHECK-NEXT: [[I_NEXT]] = add i32 [[I]], 1
+; CHECK-NEXT: [[EXIT:%.*]] = icmp slt i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EXIT]], label %[[LOOP]], label %[[EXIT_BLOCK:.*]]
+; CHECK: [[EXIT_BLOCK]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+ %gep1 = getelementptr float, ptr %arr, i32 %i
+ %load1 = load float, ptr %gep1
+ %sext = sext i32 %i to i64
+ %gep2 = getelementptr float, ptr %arr, i64 %sext
+ %gep3 = getelementptr i8, ptr %gep2, i64 0
+ %load2 = load float, ptr %gep3
+ %sel = select i1 %bool, float %load1, float %load2
+ %i.next = add i32 %i, 1
+ %exit = icmp slt i32 %i.next, %n
+ br i1 %exit, label %loop, label %exit_block
+
+exit_block:
+ ret void
+}
+
+; Negative test: Wrong comparison predicate (>= instead of <).
+define void @test_wrong_predicate(ptr %arr, i32 %n) {
+; CHECK-LABEL: define void @test_wrong_predicate(
+; CHECK-SAME: ptr [[ARR:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[GEP1:%.*]] = getelementptr float, ptr [[ARR]], i32 [[I]]
+; CHECK-NEXT: [[LOAD1:%.*]] = load float, ptr [[GEP1]], align 4
+; CHECK-NEXT: [[SEXT:%.*]] = sext i32 [[I]] to i64
+; CHECK-NEXT: [[GEP2:%.*]] = getelementptr float, ptr [[ARR]], i64 [[SEXT]]
+; CHECK-NEXT: [[CMP:%.*]] = fcmp oge float [[LOAD1]], [[LOAD1]]
+; CHECK-NEXT: [[I_NEXT]] = add i32 [[I]], 1
+; CHECK-NEXT: [[EXIT:%.*]] = icmp slt i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EXIT]], label %[[LOOP]], label %[[EXIT_BLOCK:.*]]
+; CHECK: [[EXIT_BLOCK]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+ %gep1 = getelementptr float, ptr %arr, i32 %i
+ %load1 = load float, ptr %gep1
+ %sext = sext i32 %i to i64
+ %gep2 = getelementptr float, ptr %arr, i64 %sext
+ %gep3 = getelementptr i8, ptr %gep2, i64 0
+ %load2 = load float, ptr %gep3
+ %cmp = fcmp oge float %load1, %load2
+ %sel = select i1 %cmp, float %load1, float %load2
+ %i.next = add i32 %i, 1
+ %exit = icmp slt i32 %i.next, %n
+ br i1 %exit, label %loop, label %exit_block
+
+exit_block:
+ ret void
+}
+
+; Negative test: Not both operands are loads (constant RHS).
+define void @test_not_both_loads(ptr %arr, i32 %n) {
+; CHECK-LABEL: define void @test_not_both_loads(
+; CHECK-SAME: ptr [[ARR:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[GEP1:%.*]] = getelementptr float, ptr [[ARR]], i32 [[I]]
+; CHECK-NEXT: [[LOAD1:%.*]] = load float, ptr [[GEP1]], align 4
+; CHECK-NEXT: [[CMP:%.*]] = fcmp olt float [[LOAD1]], 0.000000e+00
+; CHECK-NEXT: [[SEL:%.*]] = select i1 [[CMP]], float [[LOAD1]], float 0.000000e+00
+; CHECK-NEXT: [[I_NEXT]] = add i32 [[I]], 1
+; CHECK-NEXT: [[EXIT:%.*]] = icmp slt i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EXIT]], label %[[LOOP]], label %[[EXIT_BLOCK:.*]]
+; CHECK: [[EXIT_BLOCK]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+ %gep1 = getelementptr float, ptr %arr, i32 %i
+ %load1 = load float, ptr %gep1
+ %cmp = fcmp olt float %load1, 0.0
+ %sel = select i1 %cmp, float %load1, float 0.0
+ %i.next = add i32 %i, 1
+ %exit = icmp slt i32 %i.next, %n
+ br i1 %exit, label %loop, label %exit_block
+
+exit_block:
+ ret void
+}
+
+; Negative test: Load doesn't match GEP(GEP(...))nested pattern.
+define void @test_wrong_gep_pattern(ptr %arr, i32 %n) {
+; CHECK-LABEL: define void @test_wrong_gep_pattern(
+; CHECK-SAME: ptr [[ARR:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[GEP1:%.*]] = getelementptr float, ptr [[ARR]], i32 [[I]]
+; CHECK-NEXT: [[I_NEXT]] = add i32 [[I]], 1
+; CHECK-NEXT: [[EXIT:%.*]] = icmp slt i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EXIT]], label %[[LOOP]], label %[[EXIT_BLOCK:.*]]
+; CHECK: [[EXIT_BLOCK]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+ %gep1 = getelementptr float, ptr %arr, i32 %i
+ %load1 = load float, ptr %gep1
+ ; Simple GEP, not nested GEP(GEP(...))
+ %gep2 = getelementptr float, ptr %arr, i32 %i
+ %load2 = load float, ptr %gep2
+ %cmp = fcmp olt float %load1, %load2
+ %sel = select i1 %cmp, float %load1, float %load2
+ %i.next = add i32 %i, 1
+ %exit = icmp slt i32 %i.next, %n
+ br i1 %exit, label %loop, label %exit_block
+
+exit_block:
+ ret void
+}
+
+; Negative test: ZExt instead of SExt.
+define void @test_no_sext(ptr %arr, i32 %n) {
+; CHECK-LABEL: define void @test_no_sext(
+; CHECK-SAME: ptr [[ARR:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[GEP1:%.*]] = getelementptr float, ptr [[ARR]], i32 [[I]]
+; CHECK-NEXT: [[LOAD1:%.*]] = load float, ptr [[GEP1]], align 4
+; CHECK-NEXT: [[ZEXT:%.*]] = zext i32 [[I]] to i64
+; CHECK-NEXT: [[GEP2:%.*]] = getelementptr float, ptr [[ARR]], i64 [[ZEXT]]
+; CHECK-NEXT: [[LOAD2:%.*]] = load float, ptr [[GEP2]], align 4
+; CHECK-NEXT: [[CMP:%.*]] = fcmp olt float [[LOAD1]], [[LOAD2]]
+; CHECK-NEXT: [[SEL:%.*]] = select i1 [[CMP]], float [[LOAD1]], float [[LOAD2]]
+; CHECK-NEXT: [[I_NEXT]] = add i32 [[I]], 1
+; CHECK-NEXT: [[EXIT:%.*]] = icmp slt i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EXIT]], label %[[LOOP]], label %[[EXIT_BLOCK:.*]]
+; CHECK: [[EXIT_BLOCK]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+ %gep1 = getelementptr float, ptr %arr, i32 %i
+ %load1 = load float, ptr %gep1
+ %zext = zext i32 %i to i64
+ %gep2 = getelementptr float, ptr %arr, i64 %zext
+ %gep3 = getelementptr i8, ptr %gep2, i64 0
+ %load2 = load float, ptr %gep3
+ %cmp = fcmp olt float %load1, %load2
+ %sel = select i1 %cmp, float %load1, float %load2
+ %i.next = add i32 %i, 1
+ %exit = icmp slt i32 %i.next, %n
+ br i1 %exit, label %loop, label %exit_block
+
+exit_block:
+ ret void
+}
+
+; Negative test: SExt with wrong types (i16->i32 instead of i32->i64).
+define void @test_wrong_sext_types(ptr %arr, i32 %n) {
+; CHECK-LABEL: define void @test_wrong_sext_types(
+; CHECK-SAME: ptr [[ARR:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[GEP1:%.*]] = getelementptr float, ptr [[ARR]], i32 [[I]]
+; CHECK-NEXT: [[LOAD1:%.*]] = load float, ptr [[GEP1]], align 4
+; CHECK-NEXT: [[I16:%.*]] = trunc i32 [[I]] to i16
+; CHECK-NEXT: [[SEXT:%.*]] = sext i16 [[I16]] to i32
+; CHECK-NEXT: [[GEP2:%.*]] = getelementptr float, ptr [[ARR]], i32 [[SEXT]]
+; CHECK-NEXT: [[LOAD2:%.*]] = load float, ptr [[GEP2]], align 4
+; CHECK-NEXT: [[CMP:%.*]] = fcmp olt float [[LOAD1]], [[LOAD2]]
+; CHECK-NEXT: [[SEL:%.*]] = select i1 [[CMP]], float [[LOAD1]], float [[LOAD2]]
+; CHECK-NEXT: [[I_NEXT]] = add i32 [[I]], 1
+; CHECK-NEXT: [[EXIT:%.*]] = icmp slt i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EXIT]], label %[[LOOP]], label %[[EXIT_BLOCK:.*]]
+; CHECK: [[EXIT_BLOCK]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+ %gep1 = getelementptr float, ptr %arr, i32 %i
+ %load1 = load float, ptr %gep1
+ %i16 = trunc i32 %i to i16
+ %sext = sext i16 %i16 to i32
+ %gep2 = getelementptr float, ptr %arr, i32 %sext
+ %gep3 = getelementptr i8, ptr %gep2, i32 0
+ %load2 = load float, ptr %gep3
+ %cmp = fcmp olt float %load1, %load2
+ %sel = select i1 %cmp, float %load1, float %load2
+ %i.next = add i32 %i, 1
+ %exit = icmp slt i32 %i.next, %n
+ br i1 %exit, label %loop, label %exit_block
+
+exit_block:
+ ret void
+}
+
+; Negative test: IndexVal is a constant, not a PHI node.
+define void @test_index_is_constant(ptr %arr, i32 %n) {
+; CHECK-LABEL: define void @test_index_is_constant(
+; CHECK-SAME: ptr [[ARR:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[GEP2:%.*]] = getelementptr float, ptr [[ARR]], i64 5
+; CHECK-NEXT: [[LOAD2:%.*]] = load float, ptr [[GEP2]], align 4
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[GEP1:%.*]] = getelementptr float, ptr [[ARR]], i32 [[I]]
+; CHECK-NEXT: [[LOAD1:%.*]] = load float, ptr [[GEP1]], align 4
+; CHECK-NEXT: [[CMP:%.*]] = fcmp olt float [[LOAD1]], [[LOAD2]]
+; CHECK-NEXT: [[SEL:%.*]] = select i1 [[CMP]], float [[LOAD1]], float [[LOAD2]]
+; CHECK-NEXT: [[I_NEXT]] = add i32 [[I]], 1
+; CHECK-NEXT: [[EXIT:%.*]] = icmp slt i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EXIT]], label %[[LOOP]], label %[[EXIT_BLOCK:.*]]
+; CHECK: [[EXIT_BLOCK]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+
+ %gep1 = getelementptr float, ptr %arr, i32 %i
+ %load1 = load float, ptr %gep1
+ ; IndexVal is a constant (5), not a PHI node!
+ %sext = sext i32 5 to i64
+ %gep2 = getelementptr float, ptr %arr, i64 %sext
+ %gep3 = getelementptr i8, ptr %gep2, i64 0
+ %load2 = load float, ptr %gep3
+ %cmp = fcmp olt float %load1, %load2
+ %sel = select i1 %cmp, float %load1, float %load2
+ %i.next = add i32 %i, 1
+ %exit = icmp slt i32 %i.next, %n
+ br i1 %exit, label %loop, label %exit_block
+
+exit_block:
+ ret void
+}
+
+declare void @external_function(ptr)
+
+; Negative test: Initial min index value is not 1.
+define void @test_initial_index_not_one(ptr %arr, i32 %n) {
+; CHECK-LABEL: define void @test_initial_index_not_one(
+; CHECK-SAME: ptr [[ARR:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MIN_IDX:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[GEP1:%.*]] = getelementptr float, ptr [[ARR]], i32 [[I]]
+; CHECK-NEXT: [[LOAD1:%.*]] = load float, ptr [[GEP1]], align 4
+; CHECK-NEXT: [[SEXT:%.*]] = sext i32 [[MIN_IDX]] to i64
+; CHECK-NEXT: [[GEP2:%.*]] = getelementptr float, ptr [[ARR]], i64 [[SEXT]]
+; CHECK-NEXT: [[LOAD2:%.*]] = load float, ptr [[GEP2]], align 4
+; CHECK-NEXT: [[CMP:%.*]] = fcmp olt float [[LOAD1]], [[LOAD2]]
+; CHECK-NEXT: [[SEL:%.*]] = select i1 [[CMP]], float [[LOAD1]], float [[LOAD2]]
+; CHECK-NEXT: [[MIN_IDX_NEXT]] = select i1 [[CMP]], i32 [[I]], i32 [[MIN_IDX]]
+; CHECK-NEXT: [[I_NEXT]] = add i32 [[I]], 1
+; CHECK-NEXT: [[EXIT:%.*]] = icmp slt i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EXIT]], label %[[LOOP]], label %[[EXIT_BLOCK:.*]]
+; CHECK: [[EXIT_BLOCK]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+ ; Initial value is 0, not 1!
+ %min.idx = phi i32 [ 0, %entry ], [ %min.idx.next, %loop ]
+
+ %gep1 = getelementptr float, ptr %arr, i32 %i
+ %load1 = load float, ptr %gep1
+ %sext = sext i32 %min.idx to i64
+ %gep2 = getelementptr float, ptr %arr, i64 %sext
+ %gep3 = getelementptr i8, ptr %gep2, i64 0
+ %load2 = load float, ptr %gep3
+ %cmp = fcmp olt float %load1, %load2
+ %sel = select i1 %cmp, float %load1, float %load2
+ %min.idx.next = select i1 %cmp, i32 %i, i32 %min.idx
+ %i.next = add i32 %i, 1
+ %exit = icmp slt i32 %i.next, %n
+ br i1 %exit, label %loop, label %exit_block
+
+exit_block:
+ ret void
+}
+
+; Negative test: Pattern recognized but hoisting not safe due to aliasing store in loop.
+define void @test_aliasing_store_in_loop(ptr %arr, i32 %n) {
+; CHECK-LABEL: define void @test_aliasing_store_in_loop(
+; CHECK-SAME: ptr [[ARR:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MIN_IDX:%.*]] = phi i32 [ 1, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[STORE_GEP:%.*]] = getelementptr float, ptr [[ARR]], i32 [[I]]
+; CHECK-NEXT: store float 0.000000e+00, ptr [[STORE_GEP]], align 4
+; CHECK-NEXT: [[SEXT:%.*]] = sext i32 [[MIN_IDX]] to i64
+; CHECK-NEXT: [[GEP2:%.*]] = getelementptr float, ptr [[ARR]], i64 [[SEXT]]
+; CHECK-NEXT: [[LOAD2:%.*]] = load float, ptr [[GEP2]], align 4
+; CHECK-NEXT: [[CMP:%.*]] = fcmp olt float 0.000000e+00, [[LOAD2]]
+; CHECK-NEXT: [[SEL:%.*]] = select i1 [[CMP]], float 0.000000e+00, float [[LOAD2]]
+; CHECK-NEXT: [[MIN_IDX_NEXT]] = select i1 [[CMP]], i32 [[I]], i32 [[MIN_IDX]]
+; CHECK-NEXT: [[I_NEXT]] = add i32 [[I]], 1
+; CHECK-NEXT: [[EXIT:%.*]] = icmp slt i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EXIT]], label %[[LOOP]], label %[[EXIT_BLOCK:.*]]
+; CHECK: [[EXIT_BLOCK]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+ %min.idx = phi i32 [ 1, %entry ], [ %min.idx.next, %loop ]
+
+ ; This store aliases with the loads, preventing hoisting.
+ %store.gep = getelementptr float, ptr %arr, i32 %i
+ store float 0.0, ptr %store.gep
+
+ %gep1 = getelementptr float, ptr %arr, i32 %i
+ %load1 = load float, ptr %gep1
+ %sext = sext i32 %min.idx to i64
+ %gep2 = getelementptr float, ptr %arr, i64 %sext
+ %gep3 = getelementptr i8, ptr %gep2, i64 0
+ %load2 = load float, ptr %gep3
+ %cmp = fcmp olt float %load1, %load2
+ %sel = select i1 %cmp, float %load1, float %load2
+ %min.idx.next = select i1 %cmp, i32 %i, i32 %min.idx
+ %i.next = add i32 %i, 1
+ %exit = icmp slt i32 %i.next, %n
+ br i1 %exit, label %loop, label %exit_block
+
+exit_block:
+ ret void
+}
+
+; Negative test: Pattern recognized but BasePtr is not loop invariant.
+define void @test_non_invariant_base_ptr(ptr %arr, i32 %n) {
+; CHECK-LABEL: define void @test_non_invariant_base_ptr(
+; CHECK-SAME: ptr [[ARR:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MIN_IDX:%.*]] = phi i32 [ 1, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[BASE_PTR:%.*]] = phi ptr [ [[ARR]], %[[ENTRY]] ], [ [[BASE_PTR_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[GEP1:%.*]] = getelementptr float, ptr [[BASE_PTR]], i32 [[I]]
+; CHECK-NEXT: [[LOAD1:%.*]] = load float, ptr [[GEP1]], align 4
+; CHECK-NEXT: [[SEXT:%.*]] = sext i32 [[MIN_IDX]] to i64
+; CHECK-NEXT: [[GEP2:%.*]] = getelementptr float, ptr [[BASE_PTR]], i64 [[SEXT]]
+; CHECK-NEXT: [[LOAD2:%.*]] = load float, ptr [[GEP2]], align 4
+; CHECK-NEXT: [[CMP:%.*]] = fcmp olt float [[LOAD1]], [[LOAD2]]
+; CHECK-NEXT: [[SEL:%.*]] = select i1 [[CMP]], float [[LOAD1]], float [[LOAD2]]
+; CHECK-NEXT: [[MIN_IDX_NEXT]] = select i1 [[CMP]], i32 [[I]], i32 [[MIN_IDX]]
+; CHECK-NEXT: [[BASE_PTR_NEXT]] = getelementptr float, ptr [[BASE_PTR]], i32 1
+; CHECK-NEXT: [[I_NEXT]] = add i32 [[I]], 1
+; CHECK-NEXT: [[EXIT:%.*]] = icmp slt i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EXIT]], label %[[LOOP]], label %[[EXIT_BLOCK:.*]]
+; CHECK: [[EXIT_BLOCK]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+ %min.idx = phi i32 [ 1, %entry ], [ %min.idx.next, %loop ]
+ ; Base pointer changes each iteration - not loop invariant!
+ %base.ptr = phi ptr [ %arr, %entry ], [ %base.ptr.next, %loop ]
+
+ %gep1 = getelementptr float, ptr %base.ptr, i32 %i
+ %load1 = load float, ptr %gep1
+ %sext = sext i32 %min.idx to i64
+ %gep2 = getelementptr float, ptr %base.ptr, i64 %sext
+ %gep3 = getelementptr i8, ptr %gep2, i64 0
+ %load2 = load float, ptr %gep3
+ %cmp = fcmp olt float %load1, %load2
+ %sel = select i1 %cmp, float %load1, float %load2
+ %min.idx.next = select i1 %cmp, i32 %i, i32 %min.idx
+ %base.ptr.next = getelementptr float, ptr %base.ptr, i32 1
+ %i.next = add i32 %i, 1
+ %exit = icmp slt i32 %i.next, %n
+ br i1 %exit, label %loop, label %exit_block
+
+exit_block:
+ ret void
+}
+
+; Negative test: Pattern recognized but may-alias call in loop.
+define void @test_may_alias_call_in_loop(ptr %arr, i32 %n) {
+; CHECK-LABEL: define void @test_may_alias_call_in_loop(
+; CHECK-SAME: ptr [[ARR:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MIN_IDX:%.*]] = phi i32 [ 1, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: call void @external_function(ptr [[ARR]])
+; CHECK-NEXT: [[GEP1:%.*]] = getelementptr float, ptr [[ARR]], i32 [[I]]
+; CHECK-NEXT: [[LOAD1:%.*]] = load float, ptr [[GEP1]], align 4
+; CHECK-NEXT: [[SEXT:%.*]] = sext i32 [[MIN_IDX]] to i64
+; CHECK-NEXT: [[GEP2:%.*]] = getelementptr float, ptr [[ARR]], i64 [[SEXT]]
+; CHECK-NEXT: [[LOAD2:%.*]] = load float, ptr [[GEP2]], align 4
+; CHECK-NEXT: [[CMP:%.*]] = fcmp olt float [[LOAD1]], [[LOAD2]]
+; CHECK-NEXT: [[SEL:%.*]] = select i1 [[CMP]], float [[LOAD1]], float [[LOAD2]]
+; CHECK-NEXT: [[MIN_IDX_NEXT]] = select i1 [[CMP]], i32 [[I]], i32 [[MIN_IDX]]
+; CHECK-NEXT: [[I_NEXT]] = add i32 [[I]], 1
+; CHECK-NEXT: [[EXIT:%.*]] = icmp slt i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EXIT]], label %[[LOOP]], label %[[EXIT_BLOCK:.*]]
+; CHECK: [[EXIT_BLOCK]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+ %min.idx = phi i32 [ 1, %entry ], [ %min.idx.next, %loop ]
+
+ ; External function may modify memory.
+ call void @external_function(ptr %arr)
+
+ %gep1 = getelementptr float, ptr %arr, i32 %i
+ %load1 = load float, ptr %gep1
+ %sext = sext i32 %min.idx to i64
+ %gep2 = getelementptr float, ptr %arr, i64 %sext
+ %gep3 = getelementptr i8, ptr %gep2, i64 0
+ %load2 = load float, ptr %gep3
+ %cmp = fcmp olt float %load1, %load2
+ %sel = select i1 %cmp, float %load1, float %load2
+ %min.idx.next = select i1 %cmp, i32 %i, i32 %min.idx
+ %i.next = add i32 %i, 1
+ %exit = icmp slt i32 %i.next, %n
+ br i1 %exit, label %loop, label %exit_block
+
+exit_block:
+ ret void
+}
>From d33d6723853d8d4010974c1ab0f50252c25414de Mon Sep 17 00:00:00 2001
From: Madhur Amilkanthwar <madhura at nvidia.com>
Date: Wed, 28 Jan 2026 07:22:01 -0800
Subject: [PATCH 06/13] Address review comments + rebase
---
llvm/include/llvm/Transforms/Scalar/GVN.h | 2 +-
llvm/lib/Transforms/Scalar/GVN.cpp | 111 +++++++++++++---------
2 files changed, 67 insertions(+), 46 deletions(-)
diff --git a/llvm/include/llvm/Transforms/Scalar/GVN.h b/llvm/include/llvm/Transforms/Scalar/GVN.h
index db74c2066013c..179bee6164993 100644
--- a/llvm/include/llvm/Transforms/Scalar/GVN.h
+++ b/llvm/include/llvm/Transforms/Scalar/GVN.h
@@ -414,7 +414,7 @@ class GVNPass : public PassInfoMixin<GVNPass> {
BasicBlock *Preheader, BasicBlock *BB,
Value *LHS, Value *RHS,
CmpInst *Comparison, SelectInst *Select,
- Value *BasePtr, Value *IndexVal,
+ Value *BasePtr, PHINode *IndexValPhi,
Value *OffsetVal);
};
diff --git a/llvm/lib/Transforms/Scalar/GVN.cpp b/llvm/lib/Transforms/Scalar/GVN.cpp
index 0a96b0fde49c9..5a8c557289baa 100644
--- a/llvm/lib/Transforms/Scalar/GVN.cpp
+++ b/llvm/lib/Transforms/Scalar/GVN.cpp
@@ -3358,6 +3358,8 @@ void GVNPass::assignValNumForDeadCode() {
// ...
// br i1 %cond, label %loop, label %exit
//
+// We capture <TYPE> as a part of pattern matching and then later
+// use it in the transformation.
// *** After transformation ***
//
// preheader:
@@ -3378,37 +3380,17 @@ void GVNPass::assignValNumForDeadCode() {
bool GVNPass::transformMinFindingSelectPattern(
Loop *L, Type *LoadType, BasicBlock *Preheader, BasicBlock *BB, Value *LHS,
Value *LoadVal, CmpInst *Comparison, SelectInst *Select, Value *BasePtr,
- Value *IndexVal, Value *OffsetVal) {
+ PHINode *IndexValPhi, Value *OffsetVal) {
- assert(IndexVal && "IndexVal is null");
+ assert(BasePtr && "BasePtr is null");
+ assert(OffsetVal && "OffsetVal is null");
+ assert(IndexValPhi && "IndexValPhi is null");
AAResults *AA = VN.getAliasAnalysis();
assert(AA && "AA is null");
- IRBuilder<> Builder(Preheader->getTerminator());
- Value *InitialMinIndex =
- dyn_cast<PHINode>(IndexVal)->getIncomingValueForBlock(Preheader);
-
- // Insert PHI node at the top of this block.
- // This PHI node will be used to memoize the current minimum value so far.
- PHINode *KnownMinPhi = PHINode::Create(LoadType, 2, "known_min", BB->begin());
-
- // Hoist the load and build the necessary operations.
- // 1. hoist_0 = sext i32 1 to i64
- Value *HoistedSExt =
- Builder.CreateSExt(InitialMinIndex, Builder.getInt64Ty(), "hoist_sext");
-
- // 2. hoist_gep1 = getelementptr float, ptr BasePtr, i64 HoistedSExt
- Value *HoistedGEP1 =
- Builder.CreateGEP(LoadType, BasePtr, HoistedSExt, "hoist_gep1");
-
- // 3. hoist_gep2 = getelementptr i8, ptr HoistedGEP1, i64 OffsetVal
- Value *HoistedGEP2 = Builder.CreateGEP(Builder.getInt8Ty(), HoistedGEP1,
- OffsetVal, "hoist_gep2");
-
MemoryLocation NewLoc = MemoryLocation(
- HoistedGEP2,
- LocationSize::precise(
- L->getHeader()->getDataLayout().getTypeStoreSize(LoadType)));
+ LoadVal, LocationSize::precise(
+ L->getHeader()->getDataLayout().getTypeStoreSize(LoadType)));
// Check if any instruction in the loop clobbers this location.
bool CanHoist = true;
for (BasicBlock *BB : L->blocks()) {
@@ -3433,6 +3415,26 @@ bool GVNPass::transformMinFindingSelectPattern(
return false;
}
+ IRBuilder<> Builder(Preheader->getTerminator());
+ Value *InitialMinIndex = IndexValPhi->getIncomingValueForBlock(Preheader);
+
+ // Insert PHI node at the top of this block.
+ // This PHI node will be used to memoize the current minimum value so far.
+ PHINode *KnownMinPhi = PHINode::Create(LoadType, 2, "known_min", BB->begin());
+
+ // Hoist the load and build the necessary operations.
+ // 1. hoist_0 = sext i32 1 to i64
+ Value *HoistedSExt =
+ Builder.CreateSExt(InitialMinIndex, Builder.getInt64Ty(), "hoist_sext");
+
+ // 2. hoist_gep1 = getelementptr float, ptr BasePtr, i64 HoistedSExt
+ Value *HoistedGEP1 =
+ Builder.CreateGEP(LoadType, BasePtr, HoistedSExt, "hoist_gep1");
+
+ // 3. hoist_gep2 = getelementptr i8, ptr HoistedGEP1, i64 OffsetVal
+ Value *HoistedGEP2 = Builder.CreateGEP(Builder.getInt8Ty(), HoistedGEP1,
+ OffsetVal, "hoist_gep2");
+
// 4. hoisted_load = load float, ptr HoistedGEP2
LoadInst *NewLoad = Builder.CreateLoad(LoadType, HoistedGEP2, "hoisted_load");
@@ -3486,17 +3488,13 @@ bool GVNPass::transformMinFindingSelectPattern(
// ...
// br i1 ..., label %loop, ...
bool GVNPass::recognizeMinFindingSelectPattern(SelectInst *Select) {
- IRBuilder<> Builder(Select);
- Value *BasePtr = nullptr, *IndexVal = nullptr, *OffsetVal = nullptr,
- *SExt = nullptr;
+ Value *OffsetVal = nullptr;
BasicBlock *BB = Select->getParent();
// If the block is not in a loop, bail out.
Loop *L = LI->getLoopFor(BB);
- if (!L) {
- LLVM_DEBUG(dbgs() << "GVN: Could not find loop.\n");
+ if (!L)
return false;
- }
// If preheader of the loop is not found, bail out.
BasicBlock *Preheader = L->getLoopPreheader();
@@ -3528,31 +3526,54 @@ bool GVNPass::recognizeMinFindingSelectPattern(SelectInst *Select) {
return false;
}
- if (!match(RHS, m_Load(m_GEP(m_GEP(m_Value(BasePtr), m_Value(SExt)),
- m_Value(OffsetVal))))) {
+ // Check if the type of both loads are the same.
+ if (LHS->getType() != RHS->getType()) {
+ LLVM_DEBUG(dbgs() << "GVN: Not both loads are of the same type.\n");
+ return false;
+ }
+ Type *LoadType = LHS->getType();
+ Value *InnerGEP;
+ const APInt *OffsetAPInt;
+ if (!match(RHS, m_Load(m_PtrAdd(m_Value(InnerGEP), m_APInt(OffsetAPInt))))) {
LLVM_DEBUG(dbgs() << "GVN: Not a required load pattern.\n");
return false;
}
- // Check if the SExt instruction is a sext instruction.
- SExtInst *SEInst = dyn_cast<SExtInst>(SExt);
+ auto *TypedGEP = dyn_cast<GetElementPtrInst>(InnerGEP);
+ if (!TypedGEP) {
+ LLVM_DEBUG(dbgs() << "GVN: Not a typed GEP.\n");
+ return false;
+ }
+ Type *ElemTy = TypedGEP->getSourceElementType();
+ // Check if ElemTy is same as LoadType.
+ if (ElemTy != LoadType) {
+ LLVM_DEBUG(dbgs() << "GVN: Not a required element type.\n");
+ return false;
+ }
+ OffsetVal =
+ ConstantInt::get(Type::getInt64Ty(RHS->getContext()), *OffsetAPInt);
+
+ // Check if the second operand of InnerGEP is a sext instruction.
+ auto *SEInst = dyn_cast<SExtInst>(TypedGEP->getOperand(1));
if (!SEInst) {
- LLVM_DEBUG(dbgs() << "GVN: not a sext instruction.\n");
+ LLVM_DEBUG(dbgs() << "GVN: Not a sext instruction.\n");
return false;
}
+
// Check if the "To" and "from" type of the sext instruction are i64 and i32
// respectively.
- if (SEInst->getType() != Builder.getInt64Ty() ||
- SEInst->getOperand(0)->getType() != Builder.getInt32Ty()) {
+ if (SEInst->getType() != Type::getInt64Ty(SEInst->getContext()) ||
+ SEInst->getOperand(0)->getType() !=
+ Type::getInt32Ty(SEInst->getContext())) {
LLVM_DEBUG(
dbgs()
<< "GVN: Not matching the required type for sext instruction.\n");
return false;
}
- IndexVal = SEInst->getOperand(0);
+ Value *IndexVal = SEInst->getOperand(0);
// Check if the IndexVal is a PHI node.
- PHINode *Phi = dyn_cast<PHINode>(IndexVal);
- if (!Phi) {
+ PHINode *IndexValPhi = dyn_cast<PHINode>(IndexVal);
+ if (!IndexValPhi) {
LLVM_DEBUG(dbgs() << "GVN: IndexVal is not a PHI node\n");
return false;
}
@@ -3560,9 +3581,9 @@ bool GVNPass::recognizeMinFindingSelectPattern(SelectInst *Select) {
LLVM_DEBUG(dbgs() << "GVN: Found minimum finding pattern in Block: "
<< Select->getParent()->getName() << ".\n");
- return transformMinFindingSelectPattern(L, dyn_cast<LoadInst>(LHS)->getType(),
- Preheader, BB, LHS, RHS, Comparison,
- Select, BasePtr, IndexVal, OffsetVal);
+ return transformMinFindingSelectPattern(
+ L, cast<LoadInst>(LHS)->getType(), Preheader, BB, LHS, RHS, Comparison,
+ Select, TypedGEP->getPointerOperand(), IndexValPhi, OffsetVal);
}
class llvm::gvn::GVNLegacyPass : public FunctionPass {
>From a5bafbc8e937afc2075aa0ff8141ec8ffbd809ff Mon Sep 17 00:00:00 2001
From: Madhur Amilkanthwar <madhura at nvidia.com>
Date: Wed, 25 Feb 2026 03:02:29 -0800
Subject: [PATCH 07/13] fixup! Address review comments from alinas + rebase
---
llvm/include/llvm/Transforms/Scalar/GVN.h | 2 +-
llvm/lib/Transforms/Scalar/GVN.cpp | 252 +++++++++++++---------
2 files changed, 150 insertions(+), 104 deletions(-)
diff --git a/llvm/include/llvm/Transforms/Scalar/GVN.h b/llvm/include/llvm/Transforms/Scalar/GVN.h
index 179bee6164993..601d590c17f60 100644
--- a/llvm/include/llvm/Transforms/Scalar/GVN.h
+++ b/llvm/include/llvm/Transforms/Scalar/GVN.h
@@ -19,10 +19,10 @@
#include "llvm/ADT/MapVector.h"
#include "llvm/ADT/SetVector.h"
#include "llvm/ADT/SmallVector.h"
+#include "llvm/Analysis/LoopInfo.h"
#include "llvm/IR/Dominators.h"
#include "llvm/IR/InstrTypes.h"
#include "llvm/IR/PassManager.h"
-#include "llvm/Analysis/LoopInfo.h"
#include "llvm/IR/ValueHandle.h"
#include "llvm/Support/Allocator.h"
#include "llvm/Support/Compiler.h"
diff --git a/llvm/lib/Transforms/Scalar/GVN.cpp b/llvm/lib/Transforms/Scalar/GVN.cpp
index 5a8c557289baa..275d81e4c6908 100644
--- a/llvm/lib/Transforms/Scalar/GVN.cpp
+++ b/llvm/lib/Transforms/Scalar/GVN.cpp
@@ -3336,47 +3336,95 @@ void GVNPass::assignValNumForDeadCode() {
}
}
-// Hoist the chain of operations for the second load to preheader.
-// In this transformation, we hoist the redundant load to the preheader,
-// caching the first value of the iteration. This value is used to compare with
-// the current value of the iteration and update the minimum value.
-// The comparison is done in the loop body using the new select instruction.
-//
-// *** Before transformation ***
-//
-// preheader:
-// ...
-// loop:
-// ...
-// ...
-// %val.first = load <TYPE>, ptr %ptr.first.load, align 4
-// %min.idx.ext = sext i32 %min.idx to i64
-// %ptr.<TYPE>.min = getelementptr <TYPE>, ptr %0, i64 %min.idx.ext
-// %ptr.second.load = getelementptr i8, ptr %ptr.<TYPE>.min, i64 -4
-// %val.current.min = load <TYPE>, ptr %ptr.second.load, align 4
-// ...
-// ...
-// br i1 %cond, label %loop, label %exit
-//
-// We capture <TYPE> as a part of pattern matching and then later
-// use it in the transformation.
-// *** After transformation ***
-//
-// preheader:
-// %min.idx.ext = sext i32 %min.idx.ext to i64
-// %hoist_gep1 = getelementptr <TYPE>, ptr %0, i64 %min.idx.ext
-// %hoist_gep2 = getelementptr i8, ptr %hoist_gep1, i64 -4
-// %hoisted_load = load <TYPE>, ptr %hoist_gep2, align 4
-// br label %loop
-//
-// loop:
-// %val.first = load <TYPE>, ptr %ptr.first.load, align 4
-// ...
-// (new) %val.current.min = select i1 %cond, <TYPE> %hoisted_load, <TYPE>
-// %val.current.min
-// ...
-// ...
-// br i1 %cond, label %loop, label %exit
+/// Return true if the load can be hoisted to the loop preheader (no clobber
+/// in the loop) using MemorySSA's clobbering access.
+static bool canHoistLoadWithMSSA(Loop *L, Instruction *LoadInst,
+ MemorySSAUpdater *MSSAU) {
+ MemoryAccess *MA = MSSAU->getMemorySSA()->getMemoryAccess(LoadInst);
+ assert(MA && "MemoryAccess expected when MemorySSA is available");
+ MemoryAccess *Clobber =
+ MSSAU->getMemorySSA()->getSkipSelfWalker()->getClobberingMemoryAccess(MA);
+ if (!Clobber || MSSAU->getMemorySSA()->isLiveOnEntryDef(Clobber))
+ return true;
+ if (!L->contains(Clobber->getBlock()))
+ return true;
+ LLVM_DEBUG(dbgs() << "GVN: Cannot hoist - clobbered in loop by " << *Clobber
+ << "\n");
+ return false;
+}
+
+/// Return true if the load can be hoisted to the loop preheader (no clobber
+/// in the loop) using MemoryDependenceResults.
+static bool canHoistLoadWithMD(Loop *L, LoadInst *Load,
+ MemoryDependenceResults *MD) {
+ MemDepResult Dep = MD->getDependency(Load);
+ if (Dep.isLocal() && (Dep.isDef() || Dep.isClobber())) {
+ Instruction *DepInst = Dep.getInst();
+ if (DepInst && L->contains(DepInst->getParent())) {
+ LLVM_DEBUG(dbgs() << "GVN: Cannot hoist - clobbered in loop by "
+ << *DepInst << "\n");
+ return false;
+ }
+ } else if (Dep.isNonLocal()) {
+ SmallVector<NonLocalDepResult, 64> Deps;
+ MD->getNonLocalPointerDependency(Load, Deps);
+ for (const auto &NLDep : Deps) {
+ if (L->contains(NLDep.getBB()) &&
+ (NLDep.getResult().isDef() || NLDep.getResult().isClobber())) {
+ LLVM_DEBUG(dbgs() << "GVN: Cannot hoist - clobbered in loop (block "
+ << NLDep.getBB()->getName() << ")\n");
+ return false;
+ }
+ }
+ } else if (Dep.isUnknown()) {
+ LLVM_DEBUG(dbgs() << "GVN: Cannot hoist - unknown memory dependence\n");
+ return false;
+ }
+ return true;
+}
+
+/// Hoist the chain of operations for the second load to preheader.
+/// In this transformation, we hoist the redundant load to the preheader,
+/// caching the first value of the iteration. This value is used to compare with
+/// the current value of the iteration and update the minimum value.
+/// The comparison is done in the loop body using the new select instruction.
+///
+/// *** Before transformation ***
+///
+/// preheader:
+/// ...
+/// loop:
+/// ...
+/// ...
+/// %val.first = load <TYPE>, ptr %ptr.first.load, align 4
+/// %min.idx.ext = sext i32 %min.idx to i64
+/// %ptr.<TYPE>.min = getelementptr <TYPE>, ptr %0, i64 %min.idx.ext
+/// %ptr.second.load = getelementptr i8, ptr %ptr.<TYPE>.min, i64 -4
+/// %val.current.min = load <TYPE>, ptr %ptr.second.load, align 4
+/// ...
+/// ...
+/// br i1 %cond, label %loop, label %exit
+///
+/// We capture <TYPE> as a part of pattern matching and then later
+/// use it in the transformation.
+///
+/// *** After transformation ***
+///
+/// preheader:
+/// %min.idx.ext = sext i32 %min.idx.ext to i64
+/// %hoist_gep1 = getelementptr <TYPE>, ptr %0, i64 %min.idx.ext
+/// %hoist_gep2 = getelementptr i8, ptr %hoist_gep1, i64 -4
+/// %hoisted_load = load <TYPE>, ptr %hoist_gep2, align 4
+/// br label %loop
+///
+/// loop:
+/// %val.first = load <TYPE>, ptr %ptr.first.load, align 4
+/// ...
+/// (new) %val.current.min = select i1 %cond, <TYPE> %hoisted_load, <TYPE>
+/// %val.current.min
+/// ...
+/// ...
+/// br i1 %cond, label %loop, label %exit
bool GVNPass::transformMinFindingSelectPattern(
Loop *L, Type *LoadType, BasicBlock *Preheader, BasicBlock *BB, Value *LHS,
Value *LoadVal, CmpInst *Comparison, SelectInst *Select, Value *BasePtr,
@@ -3388,27 +3436,14 @@ bool GVNPass::transformMinFindingSelectPattern(
AAResults *AA = VN.getAliasAnalysis();
assert(AA && "AA is null");
- MemoryLocation NewLoc = MemoryLocation(
- LoadVal, LocationSize::precise(
- L->getHeader()->getDataLayout().getTypeStoreSize(LoadType)));
- // Check if any instruction in the loop clobbers this location.
- bool CanHoist = true;
- for (BasicBlock *BB : L->blocks()) {
- for (Instruction &I : *BB) {
- if (I.mayWriteToMemory()) {
- // Check if this instruction may clobber our hoisted load.
- ModRefInfo MRI = AA->getModRefInfo(&I, NewLoc);
- if (isModOrRefSet(MRI)) {
- LLVM_DEBUG(dbgs() << "GVN: Cannot hoist - may be clobbered by: " << I
- << "\n");
- CanHoist = false;
- break;
- }
- }
- }
- if (!CanHoist)
- break;
- }
+ // Check if any instruction in the loop clobbers this location. Require MSSA
+ // or MD to perform the transformation.
+ bool CanHoist = false;
+ if (MSSAU)
+ CanHoist = canHoistLoadWithMSSA(L, dyn_cast<Instruction>(LoadVal), MSSAU);
+ else if (MD)
+ CanHoist = canHoistLoadWithMD(L, cast<LoadInst>(LoadVal), MD);
+
if (!CanHoist) {
LLVM_DEBUG(dbgs() << "GVN: Cannot hoist - may be clobbered by some "
"instruction in the loop.\n");
@@ -3438,9 +3473,29 @@ bool GVNPass::transformMinFindingSelectPattern(
// 4. hoisted_load = load float, ptr HoistedGEP2
LoadInst *NewLoad = Builder.CreateLoad(LoadType, HoistedGEP2, "hoisted_load");
+ // Update MemorySSA before erasing the original load.
+ if (MSSAU) {
+ auto *OrigUse =
+ MSSAU->getMemorySSA()->getMemoryAccess(dyn_cast<Instruction>(LoadVal));
+ if (OrigUse) {
+ MemoryAccess *DefiningAccess = OrigUse->getDefiningAccess();
+ MSSAU->createMemoryAccessInBB(NewLoad, DefiningAccess, Preheader,
+ MemorySSA::BeforeTerminator);
+ MSSAU->removeMemoryAccess(OrigUse);
+ }
+ }
+
+ // Invalidate MD cache for the loaded pointer; we added a new load and removed
+ // the old one (removeInstruction handles removing the old load from MD).
+ if (MD)
+ MD->invalidateCachedPointerInfo(NewLoad->getPointerOperand());
+
// Let the new load now take the place of the old load.
LoadVal->replaceAllUsesWith(NewLoad);
- dyn_cast<LoadInst>(LoadVal)->eraseFromParent();
+ Instruction *LoadInst = dyn_cast<Instruction>(LoadVal);
+ if (uint32_t ValNo = VN.lookup(LoadInst, false))
+ LeaderTable.erase(ValNo, LoadInst, LoadInst->getParent());
+ removeInstruction(LoadInst);
// Comparison should now compare the current value and the newly inserted
// PHI node.
@@ -3455,38 +3510,28 @@ bool GVNPass::transformMinFindingSelectPattern(
// with (hoisted) NewLoad from the preheader and CurrentMinSelect.
KnownMinPhi->addIncoming(NewLoad, Preheader);
KnownMinPhi->addIncoming(CurrentMinSelect, BB);
-
- if (MSSAU) {
- auto *OrigUse =
- MSSAU->getMemorySSA()->getMemoryAccess(dyn_cast<Instruction>(LoadVal));
- if (OrigUse) {
- MemoryAccess *DefiningAccess = OrigUse->getDefiningAccess();
- MSSAU->createMemoryAccessInBB(NewLoad, DefiningAccess, Preheader,
- MemorySSA::BeforeTerminator);
- }
- }
LLVM_DEBUG(
dbgs() << "GVN: Transformed the code for minimum finding pattern.\n");
return true;
}
-// We are looking for the following pattern:
-// loop:
-// ...
-// ...
-// %min.idx = phi i32 [ %initial_min_idx, %entry ], [ %min.idx.next, %loop ]
-// ...
-// %val.first = load <TYPE>, ptr %ptr.first.load, align 4
-// %min.idx.ext = sext i32 %min.idx to i64
-// %ptr.<TYPE>.min = getelementptr <TYPE>, ptr %0, i64 %min.idx.ext
-// %ptr.second.load = getelementptr i8, ptr %ptr.<TYPE>.min, i64 -4
-// %val.current.min = load <TYPE>, ptr %ptr.second.load, align 4
-// %cmp = <CMP_INST> <TYPE> %val.first, %val.current.min
-// ...
-// %min.idx.next = select i1 %cmp, ..., i32 %min.idx
-// ...
-// ...
-// br i1 ..., label %loop, ...
+/// We are looking for the following pattern:
+/// loop:
+/// ...
+/// ...
+/// %min.idx = phi i32 [ %initial_min_idx, %entry ], [ %min.idx.next, %loop ]
+/// ...
+/// %val.first = load <TYPE>, ptr %ptr.first.load, align 4
+/// %min.idx.ext = sext i32 %min.idx to i64
+/// %ptr.<TYPE>.min = getelementptr <TYPE>, ptr %0, i64 %min.idx.ext
+/// %ptr.second.load = getelementptr i8, ptr %ptr.<TYPE>.min, i64 -4
+/// %val.current.min = load <TYPE>, ptr %ptr.second.load, align 4
+/// %cmp = <CMP_INST> <TYPE> %val.first, %val.current.min
+/// ...
+/// %min.idx.next = select i1 %cmp, ..., i32 %min.idx
+/// ...
+/// ...
+/// br i1 ..., label %loop, ...
bool GVNPass::recognizeMinFindingSelectPattern(SelectInst *Select) {
Value *OffsetVal = nullptr;
BasicBlock *BB = Select->getParent();
@@ -3499,13 +3544,13 @@ bool GVNPass::recognizeMinFindingSelectPattern(SelectInst *Select) {
// If preheader of the loop is not found, bail out.
BasicBlock *Preheader = L->getLoopPreheader();
if (!Preheader) {
- LLVM_DEBUG(dbgs() << "GVN: Could not find loop preheader.\n");
+ LLVM_DEBUG(dbgs() << "GVN (minindx): Could not find loop preheader.\n");
return false;
}
Value *Condition = Select->getCondition();
CmpInst *Comparison = dyn_cast<CmpInst>(Condition);
if (!Comparison) {
- LLVM_DEBUG(dbgs() << "GVN: Condition is not a comparison.\n");
+ LLVM_DEBUG(dbgs() << "GVN (minindx): Condition is not a comparison.\n");
return false;
}
@@ -3513,8 +3558,9 @@ bool GVNPass::recognizeMinFindingSelectPattern(SelectInst *Select) {
CmpInst::Predicate Pred = Comparison->getPredicate();
if (Pred != CmpInst::ICMP_SLT && Pred != CmpInst::ICMP_ULT &&
Pred != CmpInst::FCMP_OLT && Pred != CmpInst::FCMP_ULT) {
- LLVM_DEBUG(dbgs() << "GVN: Not a less-than comparison, predicate: " << Pred
- << "\n");
+ LLVM_DEBUG(
+ dbgs() << "GVN (minindx): Not a less-than comparison, predicate: "
+ << Pred << "\n");
return false;
}
@@ -3522,31 +3568,32 @@ bool GVNPass::recognizeMinFindingSelectPattern(SelectInst *Select) {
Value *LHS = Comparison->getOperand(0);
Value *RHS = Comparison->getOperand(1);
if (!isa<LoadInst>(LHS) || !isa<LoadInst>(RHS)) {
- LLVM_DEBUG(dbgs() << "GVN: Not both operands are loads.\n");
+ LLVM_DEBUG(dbgs() << "GVN (minindx): Not both operands are loads.\n");
return false;
}
// Check if the type of both loads are the same.
if (LHS->getType() != RHS->getType()) {
- LLVM_DEBUG(dbgs() << "GVN: Not both loads are of the same type.\n");
+ LLVM_DEBUG(
+ dbgs() << "GVN (minindx): Not both loads are of the same type.\n");
return false;
}
Type *LoadType = LHS->getType();
Value *InnerGEP;
const APInt *OffsetAPInt;
if (!match(RHS, m_Load(m_PtrAdd(m_Value(InnerGEP), m_APInt(OffsetAPInt))))) {
- LLVM_DEBUG(dbgs() << "GVN: Not a required load pattern.\n");
+ LLVM_DEBUG(dbgs() << "GVN (minindx): Not a required load pattern.\n");
return false;
}
auto *TypedGEP = dyn_cast<GetElementPtrInst>(InnerGEP);
if (!TypedGEP) {
- LLVM_DEBUG(dbgs() << "GVN: Not a typed GEP.\n");
+ LLVM_DEBUG(dbgs() << "GVN (minindx): Not a typed GEP.\n");
return false;
}
Type *ElemTy = TypedGEP->getSourceElementType();
// Check if ElemTy is same as LoadType.
if (ElemTy != LoadType) {
- LLVM_DEBUG(dbgs() << "GVN: Not a required element type.\n");
+ LLVM_DEBUG(dbgs() << "GVN (minindx): Not a required element type.\n");
return false;
}
OffsetVal =
@@ -3555,7 +3602,7 @@ bool GVNPass::recognizeMinFindingSelectPattern(SelectInst *Select) {
// Check if the second operand of InnerGEP is a sext instruction.
auto *SEInst = dyn_cast<SExtInst>(TypedGEP->getOperand(1));
if (!SEInst) {
- LLVM_DEBUG(dbgs() << "GVN: Not a sext instruction.\n");
+ LLVM_DEBUG(dbgs() << "GVN (minindx): Not a sext instruction.\n");
return false;
}
@@ -3564,9 +3611,8 @@ bool GVNPass::recognizeMinFindingSelectPattern(SelectInst *Select) {
if (SEInst->getType() != Type::getInt64Ty(SEInst->getContext()) ||
SEInst->getOperand(0)->getType() !=
Type::getInt32Ty(SEInst->getContext())) {
- LLVM_DEBUG(
- dbgs()
- << "GVN: Not matching the required type for sext instruction.\n");
+ LLVM_DEBUG(dbgs() << "GVN (minindx): Not matching the required type for "
+ "sext instruction.\n");
return false;
}
>From 1449a5646ff4439e9b7e78f262da7bf363f47192 Mon Sep 17 00:00:00 2001
From: Madhur Amilkanthwar <madhura at nvidia.com>
Date: Sun, 8 Mar 2026 23:30:01 -0700
Subject: [PATCH 08/13] fixup! address review comments
---
llvm/lib/Transforms/Scalar/GVN.cpp | 16 ++++++++--------
1 file changed, 8 insertions(+), 8 deletions(-)
diff --git a/llvm/lib/Transforms/Scalar/GVN.cpp b/llvm/lib/Transforms/Scalar/GVN.cpp
index 275d81e4c6908..aa5a37f2db90f 100644
--- a/llvm/lib/Transforms/Scalar/GVN.cpp
+++ b/llvm/lib/Transforms/Scalar/GVN.cpp
@@ -3339,12 +3339,13 @@ void GVNPass::assignValNumForDeadCode() {
/// Return true if the load can be hoisted to the loop preheader (no clobber
/// in the loop) using MemorySSA's clobbering access.
static bool canHoistLoadWithMSSA(Loop *L, Instruction *LoadInst,
- MemorySSAUpdater *MSSAU) {
- MemoryAccess *MA = MSSAU->getMemorySSA()->getMemoryAccess(LoadInst);
+ MemorySSA *MSSA) {
+ MemoryAccess *MA = MSSA->getMemoryAccess(LoadInst);
assert(MA && "MemoryAccess expected when MemorySSA is available");
MemoryAccess *Clobber =
- MSSAU->getMemorySSA()->getSkipSelfWalker()->getClobberingMemoryAccess(MA);
- if (!Clobber || MSSAU->getMemorySSA()->isLiveOnEntryDef(Clobber))
+ MSSA->getSkipSelfWalker()->getClobberingMemoryAccess(MA);
+ assert(Clobber && "getClobberingMemoryAccess should never return null");
+ if (MSSA->isLiveOnEntryDef(Clobber))
return true;
if (!L->contains(Clobber->getBlock()))
return true;
@@ -3433,14 +3434,13 @@ bool GVNPass::transformMinFindingSelectPattern(
assert(BasePtr && "BasePtr is null");
assert(OffsetVal && "OffsetVal is null");
assert(IndexValPhi && "IndexValPhi is null");
- AAResults *AA = VN.getAliasAnalysis();
- assert(AA && "AA is null");
// Check if any instruction in the loop clobbers this location. Require MSSA
// or MD to perform the transformation.
bool CanHoist = false;
if (MSSAU)
- CanHoist = canHoistLoadWithMSSA(L, dyn_cast<Instruction>(LoadVal), MSSAU);
+ CanHoist = canHoistLoadWithMSSA(L, dyn_cast<Instruction>(LoadVal),
+ MSSAU->getMemorySSA());
else if (MD)
CanHoist = canHoistLoadWithMD(L, cast<LoadInst>(LoadVal), MD);
@@ -3606,7 +3606,7 @@ bool GVNPass::recognizeMinFindingSelectPattern(SelectInst *Select) {
return false;
}
- // Check if the "To" and "from" type of the sext instruction are i64 and i32
+ // Check if the "to" and "from" type of the sext instruction are i64 and i32
// respectively.
if (SEInst->getType() != Type::getInt64Ty(SEInst->getContext()) ||
SEInst->getOperand(0)->getType() !=
>From b09e2709e902b9d3f8207acaf7b56623b5648596 Mon Sep 17 00:00:00 2001
From: Madhur Amilkanthwar <madhura at nvidia.com>
Date: Tue, 10 Mar 2026 00:29:02 -0700
Subject: [PATCH 09/13] fixup! wrap the feature under option
---
llvm/lib/Transforms/Scalar/GVN.cpp | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
diff --git a/llvm/lib/Transforms/Scalar/GVN.cpp b/llvm/lib/Transforms/Scalar/GVN.cpp
index aa5a37f2db90f..f1ac755e32fef 100644
--- a/llvm/lib/Transforms/Scalar/GVN.cpp
+++ b/llvm/lib/Transforms/Scalar/GVN.cpp
@@ -136,6 +136,10 @@ static cl::opt<uint32_t> MaxNumInsnsPerBlock(
cl::desc("Max number of instructions to scan in each basic block in GVN "
"(default = 100)"));
+static cl::opt<bool> GVNEnableMinFindingSelectPattern(
+ "gvn-enable-min-finding-select-pattern", cl::init(false), cl::Hidden,
+ cl::desc("Enable GVN transformation for minimum finding select pattern"));
+
struct llvm::GVNPass::Expression {
uint32_t Opcode;
bool Commutative = false;
@@ -2746,7 +2750,8 @@ bool GVNPass::processInstruction(Instruction *I) {
return Changed;
}
if (SelectInst *Select = dyn_cast<SelectInst>(I)) {
- if (recognizeMinFindingSelectPattern(Select))
+ if (GVNEnableMinFindingSelectPattern &&
+ recognizeMinFindingSelectPattern(Select))
return true;
}
>From fa0e987283c630be597349a4153bd408b4caf430 Mon Sep 17 00:00:00 2001
From: Madhur Amilkanthwar <madhura at nvidia.com>
Date: Tue, 10 Mar 2026 02:11:48 -0700
Subject: [PATCH 10/13] fixup! add flag to test
---
llvm/test/Transforms/GVN/PRE/gvn-min-pre.ll | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/test/Transforms/GVN/PRE/gvn-min-pre.ll b/llvm/test/Transforms/GVN/PRE/gvn-min-pre.ll
index 1435417f80710..014de5a20328d 100644
--- a/llvm/test/Transforms/GVN/PRE/gvn-min-pre.ll
+++ b/llvm/test/Transforms/GVN/PRE/gvn-min-pre.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
-; RUN: opt -passes=gvn -S < %s | FileCheck %s
+; RUN: opt -gvn-enable-min-finding-select-pattern -passes=gvn -S < %s | FileCheck %s
; Test the minimum finding pattern.
; The following test case is extracted from rnflow app in Polyhedron benchmark suite.
>From 905b949022f8f7d2beebea2a0761bd3b6ac07f95 Mon Sep 17 00:00:00 2001
From: Madhur Amilkanthwar <madhura at nvidia.com>
Date: Sun, 3 May 2026 22:51:28 -0700
Subject: [PATCH 11/13] fixup! Handle pending review comments.
* Make the transform more generic.
* Removed stale header inclusion.
* Added negative tests
---
llvm/include/llvm/Transforms/Scalar/GVN.h | 1 -
llvm/lib/Transforms/Scalar/GVN.cpp | 120 ++++++++---------
llvm/test/Transforms/GVN/PRE/gvn-min-pre.ll | 140 ++++++++++++++++++--
3 files changed, 191 insertions(+), 70 deletions(-)
diff --git a/llvm/include/llvm/Transforms/Scalar/GVN.h b/llvm/include/llvm/Transforms/Scalar/GVN.h
index 601d590c17f60..7ccddc0ed7b30 100644
--- a/llvm/include/llvm/Transforms/Scalar/GVN.h
+++ b/llvm/include/llvm/Transforms/Scalar/GVN.h
@@ -19,7 +19,6 @@
#include "llvm/ADT/MapVector.h"
#include "llvm/ADT/SetVector.h"
#include "llvm/ADT/SmallVector.h"
-#include "llvm/Analysis/LoopInfo.h"
#include "llvm/IR/Dominators.h"
#include "llvm/IR/InstrTypes.h"
#include "llvm/IR/PassManager.h"
diff --git a/llvm/lib/Transforms/Scalar/GVN.cpp b/llvm/lib/Transforms/Scalar/GVN.cpp
index f1ac755e32fef..e7ac3c55385c6 100644
--- a/llvm/lib/Transforms/Scalar/GVN.cpp
+++ b/llvm/lib/Transforms/Scalar/GVN.cpp
@@ -34,6 +34,7 @@
#include "llvm/Analysis/InstructionPrecedenceTracking.h"
#include "llvm/Analysis/InstructionSimplify.h"
#include "llvm/Analysis/Loads.h"
+#include "llvm/Analysis/LoopInfo.h"
#include "llvm/Analysis/MemoryBuiltins.h"
#include "llvm/Analysis/MemoryDependenceAnalysis.h"
#include "llvm/Analysis/MemorySSA.h"
@@ -3537,8 +3538,12 @@ bool GVNPass::transformMinFindingSelectPattern(
/// ...
/// ...
/// br i1 ..., label %loop, ...
+///
+/// Any compare predicate is accepted, and %val.current.min may appear on
+/// either side of the compare. The transform memoizes one operand of the
+/// compare in a loop-carried PHI, which is sound regardless of whether the
+/// pattern computes a min, a max, or an equality check.
bool GVNPass::recognizeMinFindingSelectPattern(SelectInst *Select) {
- Value *OffsetVal = nullptr;
BasicBlock *BB = Select->getParent();
// If the block is not in a loop, bail out.
@@ -3559,82 +3564,77 @@ bool GVNPass::recognizeMinFindingSelectPattern(SelectInst *Select) {
return false;
}
- // Check if this is less-than comparison.
- CmpInst::Predicate Pred = Comparison->getPredicate();
- if (Pred != CmpInst::ICMP_SLT && Pred != CmpInst::ICMP_ULT &&
- Pred != CmpInst::FCMP_OLT && Pred != CmpInst::FCMP_ULT) {
- LLVM_DEBUG(
- dbgs() << "GVN (minindx): Not a less-than comparison, predicate: "
- << Pred << "\n");
- return false;
- }
+ // The predicate itself is irrelevant: the rewrite memoizes one operand of
+ // the compare across iterations, and that is sound for any predicate. A
+ // min/max/eq/ne comparison all benefit equally.
- // Check that both operands are loads.
- Value *LHS = Comparison->getOperand(0);
- Value *RHS = Comparison->getOperand(1);
- if (!isa<LoadInst>(LHS) || !isa<LoadInst>(RHS)) {
+ // Check that both operands are loads of the same type.
+ Value *Op0 = Comparison->getOperand(0);
+ Value *Op1 = Comparison->getOperand(1);
+ if (!isa<LoadInst>(Op0) || !isa<LoadInst>(Op1)) {
LLVM_DEBUG(dbgs() << "GVN (minindx): Not both operands are loads.\n");
return false;
}
-
- // Check if the type of both loads are the same.
- if (LHS->getType() != RHS->getType()) {
+ if (Op0->getType() != Op1->getType()) {
LLVM_DEBUG(
dbgs() << "GVN (minindx): Not both loads are of the same type.\n");
return false;
}
- Type *LoadType = LHS->getType();
- Value *InnerGEP;
- const APInt *OffsetAPInt;
- if (!match(RHS, m_Load(m_PtrAdd(m_Value(InnerGEP), m_APInt(OffsetAPInt))))) {
- LLVM_DEBUG(dbgs() << "GVN (minindx): Not a required load pattern.\n");
- return false;
- }
- auto *TypedGEP = dyn_cast<GetElementPtrInst>(InnerGEP);
- if (!TypedGEP) {
- LLVM_DEBUG(dbgs() << "GVN (minindx): Not a typed GEP.\n");
- return false;
- }
- Type *ElemTy = TypedGEP->getSourceElementType();
- // Check if ElemTy is same as LoadType.
- if (ElemTy != LoadType) {
- LLVM_DEBUG(dbgs() << "GVN (minindx): Not a required element type.\n");
- return false;
- }
- OffsetVal =
- ConstantInt::get(Type::getInt64Ty(RHS->getContext()), *OffsetAPInt);
+ Type *LoadType = Op0->getType();
- // Check if the second operand of InnerGEP is a sext instruction.
- auto *SEInst = dyn_cast<SExtInst>(TypedGEP->getOperand(1));
- if (!SEInst) {
- LLVM_DEBUG(dbgs() << "GVN (minindx): Not a sext instruction.\n");
- return false;
- }
+ // Try to match the hoistable load shape on a single operand. On success,
+ // populate the captured GEP / offset / index-phi so the caller can build
+ // the hoisted preheader sequence.
+ GetElementPtrInst *TypedGEP = nullptr;
+ Value *OffsetVal = nullptr;
+ PHINode *IndexValPhi = nullptr;
+ auto MatchHoistableLoad = [&](Value *LoadOp) -> bool {
+ Value *InnerGEP;
+ const APInt *OffsetAPInt;
+ if (!match(LoadOp,
+ m_Load(m_PtrAdd(m_Value(InnerGEP), m_APInt(OffsetAPInt)))))
+ return false;
+ auto *GEP = dyn_cast<GetElementPtrInst>(InnerGEP);
+ if (!GEP || GEP->getSourceElementType() != LoadType)
+ return false;
+ auto *SEInst = dyn_cast<SExtInst>(GEP->getOperand(1));
+ if (!SEInst)
+ return false;
+ if (!SEInst->getType()->isIntegerTy(64) ||
+ !SEInst->getOperand(0)->getType()->isIntegerTy(32))
+ return false;
+ auto *Phi = dyn_cast<PHINode>(SEInst->getOperand(0));
+ if (!Phi)
+ return false;
+ TypedGEP = GEP;
+ OffsetVal = ConstantInt::get(SEInst->getType(), *OffsetAPInt);
+ IndexValPhi = Phi;
+ return true;
+ };
- // Check if the "to" and "from" type of the sext instruction are i64 and i32
- // respectively.
- if (SEInst->getType() != Type::getInt64Ty(SEInst->getContext()) ||
- SEInst->getOperand(0)->getType() !=
- Type::getInt32Ty(SEInst->getContext())) {
- LLVM_DEBUG(dbgs() << "GVN (minindx): Not matching the required type for "
- "sext instruction.\n");
- return false;
+ // The transform helper assumes the matched (hoistable) load is the second
+ // operand of Comparison. Try Op1 first to preserve the original IR shape;
+ // otherwise, try Op0 and swap the compare so the matched load lands at
+ // operand(1). swapOperands also flips the predicate to its swapped form,
+ // which keeps semantics intact.
+ if (!MatchHoistableLoad(Op1)) {
+ if (!MatchHoistableLoad(Op0)) {
+ LLVM_DEBUG(dbgs() << "GVN (minindx): No operand matches hoistable load "
+ "pattern.\n");
+ return false;
+ }
+ Comparison->swapOperands();
}
- Value *IndexVal = SEInst->getOperand(0);
- // Check if the IndexVal is a PHI node.
- PHINode *IndexValPhi = dyn_cast<PHINode>(IndexVal);
- if (!IndexValPhi) {
- LLVM_DEBUG(dbgs() << "GVN: IndexVal is not a PHI node\n");
- return false;
- }
+ Value *LHS = Comparison->getOperand(0);
+ Value *LoadVal = Comparison->getOperand(1);
LLVM_DEBUG(dbgs() << "GVN: Found minimum finding pattern in Block: "
<< Select->getParent()->getName() << ".\n");
return transformMinFindingSelectPattern(
- L, cast<LoadInst>(LHS)->getType(), Preheader, BB, LHS, RHS, Comparison,
- Select, TypedGEP->getPointerOperand(), IndexValPhi, OffsetVal);
+ L, LoadType, Preheader, BB, LHS, LoadVal, Comparison, Select,
+ TypedGEP->getPointerOperand(), IndexValPhi, OffsetVal);
}
class llvm::gvn::GVNLegacyPass : public FunctionPass {
diff --git a/llvm/test/Transforms/GVN/PRE/gvn-min-pre.ll b/llvm/test/Transforms/GVN/PRE/gvn-min-pre.ll
index 014de5a20328d..ca491bfe81afa 100644
--- a/llvm/test/Transforms/GVN/PRE/gvn-min-pre.ll
+++ b/llvm/test/Transforms/GVN/PRE/gvn-min-pre.ll
@@ -76,12 +76,12 @@ define void @test_gvn_min_pattern_i32(ptr %arr, i32 %initial_min_idx) {
; CHECK-NEXT: [[LOOP_COUNTER:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[LOOP_COUNTER_NEXT:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[MIN_IDX:%.*]] = phi i32 [ [[INITIAL_MIN_IDX]], %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[INDVARS_IV_NEXT_I]] = add nsw i64 [[INDVARS_IV_I]], -1
-; CHECK-NEXT: [[PTR_FLOAT_IV:%.*]] = getelementptr i32, ptr [[ARR]], i64 [[INDVARS_IV_I]]
-; CHECK-NEXT: [[PTR_FIRST_LOAD:%.*]] = getelementptr i8, ptr [[PTR_FLOAT_IV]], i64 -8
+; CHECK-NEXT: [[PTR_I32_IV:%.*]] = getelementptr i32, ptr [[ARR]], i64 [[INDVARS_IV_I]]
+; CHECK-NEXT: [[PTR_FIRST_LOAD:%.*]] = getelementptr i8, ptr [[PTR_I32_IV]], i64 -8
; CHECK-NEXT: [[VAL_FIRST:%.*]] = load i32, ptr [[PTR_FIRST_LOAD]], align 4
; CHECK-NEXT: [[MIN_IDX_EXT:%.*]] = sext i32 [[MIN_IDX]] to i64
-; CHECK-NEXT: [[PTR_FLOAT_MIN:%.*]] = getelementptr i32, ptr [[ARR]], i64 [[MIN_IDX_EXT]]
-; CHECK-NEXT: [[PTR_SECOND_LOAD:%.*]] = getelementptr i8, ptr [[PTR_FLOAT_MIN]], i64 -4
+; CHECK-NEXT: [[PTR_I32_MIN:%.*]] = getelementptr i32, ptr [[ARR]], i64 [[MIN_IDX_EXT]]
+; CHECK-NEXT: [[PTR_SECOND_LOAD:%.*]] = getelementptr i8, ptr [[PTR_I32_MIN]], i64 -4
; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[VAL_FIRST]], [[KNOWN_MIN]]
; CHECK-NEXT: [[NEXT_IDX_TRUNC:%.*]] = trunc nsw i64 [[INDVARS_IV_NEXT_I]] to i32
; CHECK-NEXT: [[MIN_IDX_NEXT]] = select i1 [[CMP]], i32 [[NEXT_IDX_TRUNC]], i32 [[MIN_IDX]]
@@ -280,7 +280,9 @@ exit_block:
ret void
}
-; Negative test: Wrong comparison predicate (>= instead of <).
+; Negative test: GVN's redundancy elimination collapses %load2 to %load1
+; before the recognizer sees the select, so the compare ends up with a
+; single load on both sides and the recognizer rightly bails.
define void @test_wrong_predicate(ptr %arr, i32 %n) {
; CHECK-LABEL: define void @test_wrong_predicate(
; CHECK-SAME: ptr [[ARR:%.*]], i32 [[N:%.*]]) {
@@ -481,15 +483,15 @@ define void @test_index_is_constant(ptr %arr, i32 %n) {
; CHECK-LABEL: define void @test_index_is_constant(
; CHECK-SAME: ptr [[ARR:%.*]], i32 [[N:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[GEP2:%.*]] = getelementptr float, ptr [[ARR]], i64 5
-; CHECK-NEXT: [[LOAD2:%.*]] = load float, ptr [[GEP2]], align 4
+; CHECK-NEXT: [[GEP2_PHI_TRANS_INSERT:%.*]] = getelementptr float, ptr [[ARR]], i64 5
+; CHECK-NEXT: [[LOAD2_PRE:%.*]] = load float, ptr [[GEP2_PHI_TRANS_INSERT]], align 4
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[GEP1:%.*]] = getelementptr float, ptr [[ARR]], i32 [[I]]
; CHECK-NEXT: [[LOAD1:%.*]] = load float, ptr [[GEP1]], align 4
-; CHECK-NEXT: [[CMP:%.*]] = fcmp olt float [[LOAD1]], [[LOAD2]]
-; CHECK-NEXT: [[SEL:%.*]] = select i1 [[CMP]], float [[LOAD1]], float [[LOAD2]]
+; CHECK-NEXT: [[CMP:%.*]] = fcmp olt float [[LOAD1]], [[LOAD2_PRE]]
+; CHECK-NEXT: [[SEL:%.*]] = select i1 [[CMP]], float [[LOAD1]], float [[LOAD2_PRE]]
; CHECK-NEXT: [[I_NEXT]] = add i32 [[I]], 1
; CHECK-NEXT: [[EXIT:%.*]] = icmp slt i32 [[I_NEXT]], [[N]]
; CHECK-NEXT: br i1 [[EXIT]], label %[[LOOP]], label %[[EXIT_BLOCK:.*]]
@@ -722,3 +724,123 @@ loop:
exit_block:
ret void
}
+
+; Positive test: non-LT predicate on the compare. The matched (hoistable)
+; load is on the RHS, so no operand swap is needed. Exercises that the
+; recognizer no longer rejects on predicate alone.
+define void @test_gvn_pattern_gt_predicate(ptr %0, i32 %initial_min_idx) {
+; CHECK-LABEL: define void @test_gvn_pattern_gt_predicate(
+; CHECK-SAME: ptr [[TMP0:%.*]], i32 [[INITIAL_MIN_IDX:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[HOIST_SEXT:%.*]] = sext i32 [[INITIAL_MIN_IDX]] to i64
+; CHECK-NEXT: [[HOIST_GEP1:%.*]] = getelementptr float, ptr [[TMP0]], i64 [[HOIST_SEXT]]
+; CHECK-NEXT: [[HOIST_GEP2:%.*]] = getelementptr i8, ptr [[HOIST_GEP1]], i64 -4
+; CHECK-NEXT: [[HOISTED_LOAD:%.*]] = load float, ptr [[HOIST_GEP2]], align 4
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[KNOWN_MIN:%.*]] = phi float [ [[HOISTED_LOAD]], %[[ENTRY]] ], [ [[CURRENT_MIN:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[INDVARS_IV_I:%.*]] = phi i64 [ 1, %[[ENTRY]] ], [ [[INDVARS_IV_NEXT_I:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[LOOP_COUNTER:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[LOOP_COUNTER_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MIN_IDX:%.*]] = phi i32 [ [[INITIAL_MIN_IDX]], %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[INDVARS_IV_NEXT_I]] = add nsw i64 [[INDVARS_IV_I]], -1
+; CHECK-NEXT: [[PTR_FLOAT_IV:%.*]] = getelementptr float, ptr [[TMP0]], i64 [[INDVARS_IV_I]]
+; CHECK-NEXT: [[PTR_FIRST_LOAD:%.*]] = getelementptr i8, ptr [[PTR_FLOAT_IV]], i64 -8
+; CHECK-NEXT: [[VAL_FIRST:%.*]] = load float, ptr [[PTR_FIRST_LOAD]], align 4
+; CHECK-NEXT: [[MIN_IDX_EXT:%.*]] = sext i32 [[MIN_IDX]] to i64
+; CHECK-NEXT: [[PTR_FLOAT_MIN:%.*]] = getelementptr float, ptr [[TMP0]], i64 [[MIN_IDX_EXT]]
+; CHECK-NEXT: [[PTR_SECOND_LOAD:%.*]] = getelementptr i8, ptr [[PTR_FLOAT_MIN]], i64 -4
+; CHECK-NEXT: [[CMP:%.*]] = fcmp contract ogt float [[VAL_FIRST]], [[KNOWN_MIN]]
+; CHECK-NEXT: [[NEXT_IDX_TRUNC:%.*]] = trunc nsw i64 [[INDVARS_IV_NEXT_I]] to i32
+; CHECK-NEXT: [[MIN_IDX_NEXT]] = select i1 [[CMP]], i32 [[NEXT_IDX_TRUNC]], i32 [[MIN_IDX]]
+; CHECK-NEXT: [[LOOP_COUNTER_NEXT]] = add nsw i64 [[LOOP_COUNTER]], -1
+; CHECK-NEXT: [[LOOP_CONTINUE:%.*]] = icmp samesign ugt i64 [[LOOP_COUNTER]], 1
+; CHECK-NEXT: [[CURRENT_MIN]] = select i1 [[CMP]], float [[VAL_FIRST]], float [[KNOWN_MIN]]
+; CHECK-NEXT: br i1 [[LOOP_CONTINUE]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %indvars.iv.i = phi i64 [ 1, %entry ], [ %indvars.iv.next.i, %loop ]
+ %loop.counter = phi i64 [ 0, %entry ], [ %loop.counter.next, %loop ]
+ %min.idx = phi i32 [ %initial_min_idx, %entry ], [ %min.idx.next, %loop ]
+ %indvars.iv.next.i = add nsw i64 %indvars.iv.i, -1
+ %ptr.float.iv = getelementptr float, ptr %0, i64 %indvars.iv.i
+ %ptr.first.load = getelementptr i8, ptr %ptr.float.iv, i64 -8
+ %val.first = load float, ptr %ptr.first.load, align 4
+ %min.idx.ext = sext i32 %min.idx to i64
+ %ptr.float.min = getelementptr float, ptr %0, i64 %min.idx.ext
+ %ptr.second.load = getelementptr i8, ptr %ptr.float.min, i64 -4
+ %val.current.min = load float, ptr %ptr.second.load, align 4
+ %cmp = fcmp contract ogt float %val.first, %val.current.min
+ %next.idx.trunc = trunc nsw i64 %indvars.iv.next.i to i32
+ %min.idx.next = select i1 %cmp, i32 %next.idx.trunc, i32 %min.idx
+ %loop.counter.next = add nsw i64 %loop.counter, -1
+ %loop.continue = icmp samesign ugt i64 %loop.counter, 1
+ br i1 %loop.continue, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+; Positive test: matched (hoistable) load is the LHS of the compare. The
+; recognizer must swap the compare's operands so the existing transform
+; (which expects the matched load at operand(1)) keeps working.
+define void @test_gvn_pattern_matched_load_on_lhs(ptr %0, i32 %initial_min_idx) {
+; CHECK-LABEL: define void @test_gvn_pattern_matched_load_on_lhs(
+; CHECK-SAME: ptr [[TMP0:%.*]], i32 [[INITIAL_MIN_IDX:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[HOIST_SEXT:%.*]] = sext i32 [[INITIAL_MIN_IDX]] to i64
+; CHECK-NEXT: [[HOIST_GEP1:%.*]] = getelementptr float, ptr [[TMP0]], i64 [[HOIST_SEXT]]
+; CHECK-NEXT: [[HOIST_GEP2:%.*]] = getelementptr i8, ptr [[HOIST_GEP1]], i64 -4
+; CHECK-NEXT: [[HOISTED_LOAD:%.*]] = load float, ptr [[HOIST_GEP2]], align 4
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[KNOWN_MIN:%.*]] = phi float [ [[HOISTED_LOAD]], %[[ENTRY]] ], [ [[CURRENT_MIN:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[INDVARS_IV_I:%.*]] = phi i64 [ 1, %[[ENTRY]] ], [ [[INDVARS_IV_NEXT_I:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[LOOP_COUNTER:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[LOOP_COUNTER_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MIN_IDX:%.*]] = phi i32 [ [[INITIAL_MIN_IDX]], %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[INDVARS_IV_NEXT_I]] = add nsw i64 [[INDVARS_IV_I]], -1
+; CHECK-NEXT: [[PTR_FLOAT_IV:%.*]] = getelementptr float, ptr [[TMP0]], i64 [[INDVARS_IV_I]]
+; CHECK-NEXT: [[PTR_FIRST_LOAD:%.*]] = getelementptr i8, ptr [[PTR_FLOAT_IV]], i64 -8
+; CHECK-NEXT: [[VAL_FIRST:%.*]] = load float, ptr [[PTR_FIRST_LOAD]], align 4
+; CHECK-NEXT: [[MIN_IDX_EXT:%.*]] = sext i32 [[MIN_IDX]] to i64
+; CHECK-NEXT: [[PTR_FLOAT_MIN:%.*]] = getelementptr float, ptr [[TMP0]], i64 [[MIN_IDX_EXT]]
+; CHECK-NEXT: [[PTR_SECOND_LOAD:%.*]] = getelementptr i8, ptr [[PTR_FLOAT_MIN]], i64 -4
+; CHECK-NEXT: [[CMP:%.*]] = fcmp contract olt float [[VAL_FIRST]], [[KNOWN_MIN]]
+; CHECK-NEXT: [[NEXT_IDX_TRUNC:%.*]] = trunc nsw i64 [[INDVARS_IV_NEXT_I]] to i32
+; CHECK-NEXT: [[MIN_IDX_NEXT]] = select i1 [[CMP]], i32 [[MIN_IDX]], i32 [[NEXT_IDX_TRUNC]]
+; CHECK-NEXT: [[LOOP_COUNTER_NEXT]] = add nsw i64 [[LOOP_COUNTER]], -1
+; CHECK-NEXT: [[LOOP_CONTINUE:%.*]] = icmp samesign ugt i64 [[LOOP_COUNTER]], 1
+; CHECK-NEXT: [[CURRENT_MIN]] = select i1 [[CMP]], float [[VAL_FIRST]], float [[KNOWN_MIN]]
+; CHECK-NEXT: br i1 [[LOOP_CONTINUE]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %indvars.iv.i = phi i64 [ 1, %entry ], [ %indvars.iv.next.i, %loop ]
+ %loop.counter = phi i64 [ 0, %entry ], [ %loop.counter.next, %loop ]
+ %min.idx = phi i32 [ %initial_min_idx, %entry ], [ %min.idx.next, %loop ]
+ %indvars.iv.next.i = add nsw i64 %indvars.iv.i, -1
+ %ptr.float.iv = getelementptr float, ptr %0, i64 %indvars.iv.i
+ %ptr.first.load = getelementptr i8, ptr %ptr.float.iv, i64 -8
+ %val.first = load float, ptr %ptr.first.load, align 4
+ %min.idx.ext = sext i32 %min.idx to i64
+ %ptr.float.min = getelementptr float, ptr %0, i64 %min.idx.ext
+ %ptr.second.load = getelementptr i8, ptr %ptr.float.min, i64 -4
+ %val.current.min = load float, ptr %ptr.second.load, align 4
+ %cmp = fcmp contract ogt float %val.current.min, %val.first
+ %next.idx.trunc = trunc nsw i64 %indvars.iv.next.i to i32
+ %min.idx.next = select i1 %cmp, i32 %min.idx, i32 %next.idx.trunc
+ %loop.counter.next = add nsw i64 %loop.counter, -1
+ %loop.continue = icmp samesign ugt i64 %loop.counter, 1
+ br i1 %loop.continue, label %loop, label %exit
+
+exit:
+ ret void
+}
>From 463e949d927f9f66b2d2a4942f3ae5b50a662e66 Mon Sep 17 00:00:00 2001
From: Madhur Amilkanthwar <madhura at nvidia.com>
Date: Sun, 3 May 2026 23:51:03 -0700
Subject: [PATCH 12/13] fixup! simplify the signature.
---
llvm/include/llvm/Transforms/Scalar/GVN.h | 1 -
llvm/lib/Transforms/Scalar/GVN.cpp | 15 +++++++++------
2 files changed, 9 insertions(+), 7 deletions(-)
diff --git a/llvm/include/llvm/Transforms/Scalar/GVN.h b/llvm/include/llvm/Transforms/Scalar/GVN.h
index 7ccddc0ed7b30..0af6e4aa074ba 100644
--- a/llvm/include/llvm/Transforms/Scalar/GVN.h
+++ b/llvm/include/llvm/Transforms/Scalar/GVN.h
@@ -411,7 +411,6 @@ class GVNPass : public PassInfoMixin<GVNPass> {
bool recognizeMinFindingSelectPattern(SelectInst *Select);
bool transformMinFindingSelectPattern(Loop *L, Type *LoadType,
BasicBlock *Preheader, BasicBlock *BB,
- Value *LHS, Value *RHS,
CmpInst *Comparison, SelectInst *Select,
Value *BasePtr, PHINode *IndexValPhi,
Value *OffsetVal);
diff --git a/llvm/lib/Transforms/Scalar/GVN.cpp b/llvm/lib/Transforms/Scalar/GVN.cpp
index e7ac3c55385c6..1460cb156ce10 100644
--- a/llvm/lib/Transforms/Scalar/GVN.cpp
+++ b/llvm/lib/Transforms/Scalar/GVN.cpp
@@ -3433,14 +3433,20 @@ static bool canHoistLoadWithMD(Loop *L, LoadInst *Load,
/// ...
/// br i1 %cond, label %loop, label %exit
bool GVNPass::transformMinFindingSelectPattern(
- Loop *L, Type *LoadType, BasicBlock *Preheader, BasicBlock *BB, Value *LHS,
- Value *LoadVal, CmpInst *Comparison, SelectInst *Select, Value *BasePtr,
+ Loop *L, Type *LoadType, BasicBlock *Preheader, BasicBlock *BB,
+ CmpInst *Comparison, SelectInst *Select, Value *BasePtr,
PHINode *IndexValPhi, Value *OffsetVal) {
assert(BasePtr && "BasePtr is null");
assert(OffsetVal && "OffsetVal is null");
assert(IndexValPhi && "IndexValPhi is null");
+ // The recognizer canonicalizes the matched (hoistable) load to operand(1)
+ // of the compare. Capture both operands now, before we mutate the compare
+ // below.
+ Value *LHS = Comparison->getOperand(0);
+ Value *LoadVal = Comparison->getOperand(1);
+
// Check if any instruction in the loop clobbers this location. Require MSSA
// or MD to perform the transformation.
bool CanHoist = false;
@@ -3626,14 +3632,11 @@ bool GVNPass::recognizeMinFindingSelectPattern(SelectInst *Select) {
Comparison->swapOperands();
}
- Value *LHS = Comparison->getOperand(0);
- Value *LoadVal = Comparison->getOperand(1);
-
LLVM_DEBUG(dbgs() << "GVN: Found minimum finding pattern in Block: "
<< Select->getParent()->getName() << ".\n");
return transformMinFindingSelectPattern(
- L, LoadType, Preheader, BB, LHS, LoadVal, Comparison, Select,
+ L, LoadType, Preheader, BB, Comparison, Select,
TypedGEP->getPointerOperand(), IndexValPhi, OffsetVal);
}
>From 1fae365ed911a5a8ddb2c4d8d700e303da5563fc Mon Sep 17 00:00:00 2001
From: Madhur Amilkanthwar <madhura at nvidia.com>
Date: Mon, 4 May 2026 00:13:00 -0700
Subject: [PATCH 13/13] fixup! factor load-hoist check into shared LoopUtils
helper
Pull the load-only logic out of LICM's canSinkOrHoistInst into a new
canHoistLoad utility in LoopUtils, and have GVN's min-finding select
transform call it instead of its own canHoistLoadWithMSSA. LICM
delegates to the new helper for the load case, so its behavior is
unchanged. The MDA fallback (canHoistLoadWithMD) is retained for the
no-MemorySSA path.
---
.../include/llvm/Transforms/Utils/LoopUtils.h | 12 +++
llvm/lib/Transforms/Scalar/GVN.cpp | 33 +++-----
llvm/lib/Transforms/Scalar/LICM.cpp | 78 +++++++++++--------
3 files changed, 66 insertions(+), 57 deletions(-)
diff --git a/llvm/include/llvm/Transforms/Utils/LoopUtils.h b/llvm/include/llvm/Transforms/Utils/LoopUtils.h
index 1e48eeca72952..48355a7173b63 100644
--- a/llvm/include/llvm/Transforms/Utils/LoopUtils.h
+++ b/llvm/include/llvm/Transforms/Utils/LoopUtils.h
@@ -451,6 +451,18 @@ LLVM_ABI bool canSinkOrHoistInst(Instruction &I, AAResults *AA,
SinkAndHoistLICMFlags &LICMFlags,
OptimizationRemarkEmitter *ORE = nullptr);
+/// Returns true if it is legal to hoist \p LI out of \p CurLoop. This is the
+/// load-specific subset of \c canSinkOrHoistInst: it rejects volatile or
+/// ordered loads, allows constant-memory / invariant.load / invariant.start-
+/// dominated loads unconditionally, and otherwise queries MemorySSA for an
+/// in-loop clobber. \p TargetExecutesOncePerLoop has the same meaning as in
+/// \c canSinkOrHoistInst (set to true when hoisting to the preheader).
+LLVM_ABI bool canHoistLoad(LoadInst &LI, AAResults *AA, DominatorTree *DT,
+ Loop *CurLoop, MemorySSAUpdater &MSSAU,
+ bool TargetExecutesOncePerLoop,
+ SinkAndHoistLICMFlags &LICMFlags,
+ OptimizationRemarkEmitter *ORE = nullptr);
+
/// Returns the llvm.vector.reduce intrinsic that corresponds to the recurrence
/// kind.
LLVM_ABI constexpr Intrinsic::ID getReductionIntrinsicID(RecurKind RK);
diff --git a/llvm/lib/Transforms/Scalar/GVN.cpp b/llvm/lib/Transforms/Scalar/GVN.cpp
index 1460cb156ce10..357cf3d93571f 100644
--- a/llvm/lib/Transforms/Scalar/GVN.cpp
+++ b/llvm/lib/Transforms/Scalar/GVN.cpp
@@ -72,6 +72,7 @@
#include "llvm/Transforms/Utils/AssumeBundleBuilder.h"
#include "llvm/Transforms/Utils/BasicBlockUtils.h"
#include "llvm/Transforms/Utils/Local.h"
+#include "llvm/Transforms/Utils/LoopUtils.h"
#include "llvm/Transforms/Utils/SSAUpdater.h"
#include "llvm/Transforms/Utils/VNCoercion.h"
#include <algorithm>
@@ -3342,24 +3343,6 @@ void GVNPass::assignValNumForDeadCode() {
}
}
-/// Return true if the load can be hoisted to the loop preheader (no clobber
-/// in the loop) using MemorySSA's clobbering access.
-static bool canHoistLoadWithMSSA(Loop *L, Instruction *LoadInst,
- MemorySSA *MSSA) {
- MemoryAccess *MA = MSSA->getMemoryAccess(LoadInst);
- assert(MA && "MemoryAccess expected when MemorySSA is available");
- MemoryAccess *Clobber =
- MSSA->getSkipSelfWalker()->getClobberingMemoryAccess(MA);
- assert(Clobber && "getClobberingMemoryAccess should never return null");
- if (MSSA->isLiveOnEntryDef(Clobber))
- return true;
- if (!L->contains(Clobber->getBlock()))
- return true;
- LLVM_DEBUG(dbgs() << "GVN: Cannot hoist - clobbered in loop by " << *Clobber
- << "\n");
- return false;
-}
-
/// Return true if the load can be hoisted to the loop preheader (no clobber
/// in the loop) using MemoryDependenceResults.
static bool canHoistLoadWithMD(Loop *L, LoadInst *Load,
@@ -3448,13 +3431,17 @@ bool GVNPass::transformMinFindingSelectPattern(
Value *LoadVal = Comparison->getOperand(1);
// Check if any instruction in the loop clobbers this location. Require MSSA
- // or MD to perform the transformation.
+ // or MD to perform the transformation. With MSSA we delegate to the common
+ // utility canHoistLoad(); with MD we use a local dependency walk.
bool CanHoist = false;
- if (MSSAU)
- CanHoist = canHoistLoadWithMSSA(L, dyn_cast<Instruction>(LoadVal),
- MSSAU->getMemorySSA());
- else if (MD)
+ if (MSSAU) {
+ SinkAndHoistLICMFlags Flags(/*IsSink=*/false, *L, *MSSAU->getMemorySSA());
+ CanHoist =
+ canHoistLoad(*cast<LoadInst>(LoadVal), VN.getAliasAnalysis(), DT, L,
+ *MSSAU, /*TargetExecutesOncePerLoop=*/true, Flags, ORE);
+ } else if (MD) {
CanHoist = canHoistLoadWithMD(L, cast<LoadInst>(LoadVal), MD);
+ }
if (!CanHoist) {
LLVM_DEBUG(dbgs() << "GVN: Cannot hoist - may be clobbered by some "
diff --git a/llvm/lib/Transforms/Scalar/LICM.cpp b/llvm/lib/Transforms/Scalar/LICM.cpp
index 02ece8bdf31db..93e1639db3ef8 100644
--- a/llvm/lib/Transforms/Scalar/LICM.cpp
+++ b/llvm/lib/Transforms/Scalar/LICM.cpp
@@ -1163,6 +1163,48 @@ static MemoryAccess *getClobberingMemoryAccess(MemorySSA &MSSA,
return Source;
}
+bool llvm::canHoistLoad(LoadInst &LI, AAResults *AA, DominatorTree *DT,
+ Loop *CurLoop, MemorySSAUpdater &MSSAU,
+ bool TargetExecutesOncePerLoop,
+ SinkAndHoistLICMFlags &Flags,
+ OptimizationRemarkEmitter *ORE) {
+ if (!LI.isUnordered())
+ return false; // Don't sink/hoist volatile or ordered atomic loads!
+
+ // Loads from constant memory are always safe to move, even if they end up
+ // in the same alias set as something that ends up being modified.
+ if (!isModSet(AA->getModRefInfoMask(LI.getOperand(0))))
+ return true;
+ if (LI.hasMetadata(LLVMContext::MD_invariant_load))
+ return true;
+
+ if (LI.isAtomic() && !TargetExecutesOncePerLoop)
+ return false; // Don't risk duplicating unordered loads
+
+ // This checks for an invariant.start dominating the load.
+ if (isLoadInvariantInLoop(&LI, DT, CurLoop))
+ return true;
+
+ MemorySSA *MSSA = MSSAU.getMemorySSA();
+ auto *MU = cast<MemoryUse>(MSSA->getMemoryAccess(&LI));
+
+ bool InvariantGroup = LI.hasMetadata(LLVMContext::MD_invariant_group);
+
+ bool Invalidated =
+ pointerInvalidatedByLoop(MSSA, MU, CurLoop, LI, Flags, InvariantGroup);
+ // Check loop-invariant address because this may also be a sinkable load
+ // whose address is not necessarily loop-invariant.
+ if (ORE && Invalidated && CurLoop->isLoopInvariant(LI.getPointerOperand()))
+ ORE->emit([&]() {
+ return OptimizationRemarkMissed(
+ DEBUG_TYPE, "LoadWithLoopInvariantAddressInvalidated", &LI)
+ << "failed to move load with loop-invariant address "
+ "because the loop may invalidate its value";
+ });
+
+ return !Invalidated;
+}
+
bool llvm::canSinkOrHoistInst(Instruction &I, AAResults *AA, DominatorTree *DT,
Loop *CurLoop, MemorySSAUpdater &MSSAU,
bool TargetExecutesOncePerLoop,
@@ -1175,40 +1217,8 @@ bool llvm::canSinkOrHoistInst(Instruction &I, AAResults *AA, DominatorTree *DT,
MemorySSA *MSSA = MSSAU.getMemorySSA();
// Loads have extra constraints we have to verify before we can hoist them.
if (LoadInst *LI = dyn_cast<LoadInst>(&I)) {
- if (!LI->isUnordered())
- return false; // Don't sink/hoist volatile or ordered atomic loads!
-
- // Loads from constant memory are always safe to move, even if they end up
- // in the same alias set as something that ends up being modified.
- if (!isModSet(AA->getModRefInfoMask(LI->getOperand(0))))
- return true;
- if (LI->hasMetadata(LLVMContext::MD_invariant_load))
- return true;
-
- if (LI->isAtomic() && !TargetExecutesOncePerLoop)
- return false; // Don't risk duplicating unordered loads
-
- // This checks for an invariant.start dominating the load.
- if (isLoadInvariantInLoop(LI, DT, CurLoop))
- return true;
-
- auto MU = cast<MemoryUse>(MSSA->getMemoryAccess(LI));
-
- bool InvariantGroup = LI->hasMetadata(LLVMContext::MD_invariant_group);
-
- bool Invalidated = pointerInvalidatedByLoop(
- MSSA, MU, CurLoop, I, Flags, InvariantGroup);
- // Check loop-invariant address because this may also be a sinkable load
- // whose address is not necessarily loop-invariant.
- if (ORE && Invalidated && CurLoop->isLoopInvariant(LI->getPointerOperand()))
- ORE->emit([&]() {
- return OptimizationRemarkMissed(
- DEBUG_TYPE, "LoadWithLoopInvariantAddressInvalidated", LI)
- << "failed to move load with loop-invariant address "
- "because the loop may invalidate its value";
- });
-
- return !Invalidated;
+ return canHoistLoad(*LI, AA, DT, CurLoop, MSSAU, TargetExecutesOncePerLoop,
+ Flags, ORE);
} else if (CallInst *CI = dyn_cast<CallInst>(&I)) {
// Don't sink calls which can throw.
if (CI->mayThrow())
More information about the llvm-commits
mailing list