[llvm] 0f50e20 - [LoopVersioning] Preserve MemorySSA if requested. (#220290)
via llvm-commits
llvm-commits at lists.llvm.org
Wed Sep 2 02:55:35 PDT 2026
Author: Florian Hahn
Date: 2026-09-02T10:55:30+01:00
New Revision: 0f50e2015fc8b260d876fba1f7649e97d14ab8bd
URL: https://github.com/llvm/llvm-project/commit/0f50e2015fc8b260d876fba1f7649e97d14ab8bd
DIFF: https://github.com/llvm/llvm-project/commit/0f50e2015fc8b260d876fba1f7649e97d14ab8bd.diff
LOG: [LoopVersioning] Preserve MemorySSA if requested. (#220290)
Update LoopVersioning to take an option MemorySSAUpdater. If provided,
update MemorySSA in the cloned loop, as well as for the new edges
connecting the new loop.
This fixes a MemorySSA verification failure with expensive checks in the
LoopFlatten pass which marks MemorySSA as preserved and is using
LoopVersioning.
Exposed by https://github.com/llvm/llvm-project/pull/219663.
PR: https://github.com/llvm/llvm-project/pull/220290
Added:
llvm/test/Transforms/LoopVersioning/preserve-memoryssa.ll
Modified:
llvm/include/llvm/Transforms/Utils/LoopVersioning.h
llvm/lib/Transforms/Scalar/LoopFlatten.cpp
llvm/lib/Transforms/Utils/LoopVersioning.cpp
Removed:
################################################################################
diff --git a/llvm/include/llvm/Transforms/Utils/LoopVersioning.h b/llvm/include/llvm/Transforms/Utils/LoopVersioning.h
index c440ea19e9e5c..2243e3b5f9e38 100644
--- a/llvm/include/llvm/Transforms/Utils/LoopVersioning.h
+++ b/llvm/include/llvm/Transforms/Utils/LoopVersioning.h
@@ -26,6 +26,7 @@ class SCEVPredicate;
class ScalarEvolution;
class LoopAccessInfo;
class LoopInfo;
+class MemorySSAUpdater;
struct RuntimeCheckingPtrGroup;
typedef std::pair<const RuntimeCheckingPtrGroup *,
const RuntimeCheckingPtrGroup *>
@@ -46,10 +47,12 @@ class LoopVersioning {
/// object having no checks and we expect the user to add them.
LLVM_ABI LoopVersioning(const LoopAccessInfo &LAI,
ArrayRef<RuntimePointerCheck> Checks, Loop *L,
- LoopInfo *LI, DominatorTree *DT, ScalarEvolution *SE);
+ LoopInfo *LI, DominatorTree *DT, ScalarEvolution *SE,
+ MemorySSAUpdater *MSSAU = nullptr);
/// Performs the CFG manipulation part of versioning the loop including
- /// the DominatorTree and LoopInfo updates.
+ /// the DominatorTree, LoopInfo and, if a MemorySSAUpdater was provided,
+ /// MemorySSA updates.
///
/// The loop that was used to construct the class will be the "versioned" loop
/// i.e. the loop that will receive control if all the memchecks pass.
@@ -148,6 +151,10 @@ class LoopVersioning {
LoopInfo *LI;
DominatorTree *DT;
ScalarEvolution *SE;
+
+ /// Updater to keep MemorySSA up to date, or nullptr if MemorySSA does not
+ /// need updating.
+ MemorySSAUpdater *MSSAU;
};
/// Expose LoopVersioning as a pass. Currently this is only used for
diff --git a/llvm/lib/Transforms/Scalar/LoopFlatten.cpp b/llvm/lib/Transforms/Scalar/LoopFlatten.cpp
index e48c47f1b4b89..6d81fa935b491 100644
--- a/llvm/lib/Transforms/Scalar/LoopFlatten.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopFlatten.cpp
@@ -967,7 +967,7 @@ static bool FlattenLoopPair(FlattenInfo &FI, DominatorTree *DT, LoopInfo *LI,
// emit 'false' as the branch condition, and add our own check afterwards.
BasicBlock *CheckBlock = FI.OuterLoop->getLoopPreheader();
ArrayRef<RuntimePointerCheck> Checks(nullptr, nullptr);
- LoopVersioning LVer(LAI, Checks, FI.OuterLoop, LI, DT, SE);
+ LoopVersioning LVer(LAI, Checks, FI.OuterLoop, LI, DT, SE, MSSAU);
LVer.versionLoop();
// Check for overflow by calculating the new tripcount using
diff --git a/llvm/lib/Transforms/Utils/LoopVersioning.cpp b/llvm/lib/Transforms/Utils/LoopVersioning.cpp
index 5dcbc5b2f3334..f6e88a72a2877 100644
--- a/llvm/lib/Transforms/Utils/LoopVersioning.cpp
+++ b/llvm/lib/Transforms/Utils/LoopVersioning.cpp
@@ -18,6 +18,9 @@
#include "llvm/Analysis/InstSimplifyFolder.h"
#include "llvm/Analysis/LoopAccessAnalysis.h"
#include "llvm/Analysis/LoopInfo.h"
+#include "llvm/Analysis/LoopIterator.h"
+#include "llvm/Analysis/MemorySSA.h"
+#include "llvm/Analysis/MemorySSAUpdater.h"
#include "llvm/Analysis/ScalarEvolution.h"
#include "llvm/Analysis/TargetLibraryInfo.h"
#include "llvm/IR/Dominators.h"
@@ -29,6 +32,7 @@
#include "llvm/Transforms/Utils/Cloning.h"
#include "llvm/Transforms/Utils/LoopUtils.h"
#include "llvm/Transforms/Utils/ScalarEvolutionExpander.h"
+#include <memory>
using namespace llvm;
@@ -43,9 +47,9 @@ static cl::opt<bool>
LoopVersioning::LoopVersioning(const LoopAccessInfo &LAI,
ArrayRef<RuntimePointerCheck> Checks, Loop *L,
LoopInfo *LI, DominatorTree *DT,
- ScalarEvolution *SE)
+ ScalarEvolution *SE, MemorySSAUpdater *MSSAU)
: VersionedLoop(L), AliasChecks(Checks), Preds(LAI.getPSE().getPredicate()),
- LAI(LAI), LI(LI), DT(DT), SE(SE) {}
+ LAI(LAI), LI(LI), DT(DT), SE(SE), MSSAU(MSSAU) {}
void LoopVersioning::versionLoop(
const SmallVectorImpl<Instruction *> &DefsUsedOutside) {
@@ -95,8 +99,8 @@ void LoopVersioning::versionLoop(
// Create empty preheader for the loop (and after cloning for the
// non-versioned loop).
BasicBlock *PH =
- SplitBlock(RuntimeCheckBB, RuntimeCheckBB->getTerminator(), DT, LI,
- nullptr, VersionedLoop->getHeader()->getName() + ".ph");
+ SplitBlock(RuntimeCheckBB, RuntimeCheckBB->getTerminator(), DT, LI, MSSAU,
+ VersionedLoop->getHeader()->getName() + ".ph");
// Clone the loop including the preheader.
//
@@ -123,11 +127,29 @@ void LoopVersioning::versionLoop(
// memchecking block.
DT->changeImmediateDominator(VersionedLoop->getExitBlock(), RuntimeCheckBB);
+ // Update MemorySSA, if it is needed.
+ if (MSSAU) {
+ LoopBlocksRPO LoopRPOT(VersionedLoop);
+ LoopRPOT.perform(LI);
+ MSSAU->updateForClonedLoop(LoopRPOT, /*ExitBlocks=*/{}, VMap);
+
+ // Update MemorySSA for new CFG edges reaching the new NonVersionedLoop as
+ // well as exiting it.
+ SmallVector<CFGUpdate, 4> Updates;
+ Updates.push_back({cfg::UpdateKind::Insert, RuntimeCheckBB,
+ NonVersionedLoop->getLoopPreheader()});
+ SmallVector<LoopInfo::Edge, 4> ExitEdges;
+ LI->getExitEdges(*NonVersionedLoop, ExitEdges);
+ for (auto [Exiting, Exit] : ExitEdges)
+ Updates.push_back({cfg::UpdateKind::Insert, Exiting, Exit});
+ MSSAU->applyInsertUpdates(Updates, *DT);
+ }
+
// Adds the necessary PHI nodes for the versioned loops based on the
// loop-defined values used outside of the loop.
addPHINodes(DefsUsedOutside);
- formDedicatedExitBlocks(NonVersionedLoop, DT, LI, nullptr, true);
- formDedicatedExitBlocks(VersionedLoop, DT, LI, nullptr, true);
+ formDedicatedExitBlocks(NonVersionedLoop, DT, LI, MSSAU, true);
+ formDedicatedExitBlocks(VersionedLoop, DT, LI, MSSAU, true);
assert(NonVersionedLoop->isLoopSimplifyForm() &&
VersionedLoop->isLoopSimplifyForm() &&
"The versioned loops should be in simplify form.");
@@ -274,7 +296,7 @@ void LoopVersioning::annotateInstWithNoAlias(Instruction *VersionedInst,
namespace {
bool runImpl(LoopInfo *LI, LoopAccessInfoManager &LAIs, DominatorTree *DT,
- ScalarEvolution *SE) {
+ ScalarEvolution *SE, MemorySSAUpdater *MSSAU) {
// Build up a worklist of inner-loops to version. This is necessary as the
// act of versioning a loop creates new loops and can invalidate iterators
// across the loops.
@@ -301,7 +323,7 @@ bool runImpl(LoopInfo *LI, LoopAccessInfoManager &LAIs, DominatorTree *DT,
formLCSSARecursively(*L, *DT, LI, SE);
LoopVersioning LVer(LAI, LAI.getRuntimePointerChecking()->getChecks(), L,
- LI, DT, SE);
+ LI, DT, SE, MSSAU);
LVer.versionLoop();
LVer.annotateLoopWithNoAlias();
Changed = true;
@@ -320,7 +342,16 @@ PreservedAnalyses LoopVersioningPass::run(Function &F,
LoopAccessInfoManager &LAIs = AM.getResult<LoopAccessAnalysis>(F);
auto &DT = AM.getResult<DominatorTreeAnalysis>(F);
- if (runImpl(&LI, LAIs, &DT, &SE))
- return PreservedAnalyses::none();
- return PreservedAnalyses::all();
+ // Keep MemorySSA up to date if it is available.
+ auto *MSSAAnalysis = AM.getCachedResult<MemorySSAAnalysis>(F);
+ std::unique_ptr<MemorySSAUpdater> MSSAU;
+ if (MSSAAnalysis)
+ MSSAU = std::make_unique<MemorySSAUpdater>(&MSSAAnalysis->getMSSA());
+
+ if (!runImpl(&LI, LAIs, &DT, &SE, MSSAU.get()))
+ return PreservedAnalyses::all();
+
+ if (MSSAAnalysis && VerifyMemorySSA)
+ MSSAAnalysis->getMSSA().verifyMemorySSA();
+ return PreservedAnalyses::none();
}
diff --git a/llvm/test/Transforms/LoopVersioning/preserve-memoryssa.ll b/llvm/test/Transforms/LoopVersioning/preserve-memoryssa.ll
new file mode 100644
index 0000000000000..b04e53e2136de
--- /dev/null
+++ b/llvm/test/Transforms/LoopVersioning/preserve-memoryssa.ll
@@ -0,0 +1,154 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
+; RUN: opt -passes='require<memoryssa>,loop-versioning' -verify-memoryssa -S %s | FileCheck %s
+
+define i32 @inner_cfg(ptr %a, ptr %b, i64 %n, i1 %c) {
+; CHECK-LABEL: define i32 @inner_cfg(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]], i1 [[C:%.*]]) {
+; CHECK-NEXT: [[LOOP_LVER_CHECK:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = shl i64 [[N]], 2
+; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP0]]
+; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP0]]
+; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[B]], [[SCEVGEP1]]
+; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[A]], [[SCEVGEP]]
+; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], label %[[LOOP_PH_LVER_ORIG:.*]], label %[[LOOP_PH:.*]]
+; CHECK: [[LOOP_PH_LVER_ORIG]]:
+; CHECK-NEXT: br label %[[LOOP_LVER_ORIG:.*]]
+; CHECK: [[LOOP_LVER_ORIG]]:
+; CHECK-NEXT: [[IV_LVER_ORIG:%.*]] = phi i64 [ 0, %[[LOOP_PH_LVER_ORIG]] ], [ [[IV_NEXT_LVER_ORIG:%.*]], %[[LATCH_LVER_ORIG:.*]] ]
+; CHECK-NEXT: [[GEP_A_LVER_ORIG:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV_LVER_ORIG]]
+; CHECK-NEXT: [[L_LVER_ORIG:%.*]] = load i32, ptr [[GEP_A_LVER_ORIG]], align 4
+; CHECK-NEXT: br i1 [[C]], label %[[THEN_LVER_ORIG:.*]], label %[[LATCH_LVER_ORIG]]
+; CHECK: [[THEN_LVER_ORIG]]:
+; CHECK-NEXT: [[GEP_B_LVER_ORIG:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV_LVER_ORIG]]
+; CHECK-NEXT: store i32 [[L_LVER_ORIG]], ptr [[GEP_B_LVER_ORIG]], align 4
+; CHECK-NEXT: br label %[[LATCH_LVER_ORIG]]
+; CHECK: [[LATCH_LVER_ORIG]]:
+; CHECK-NEXT: [[IV_NEXT_LVER_ORIG]] = add nuw nsw i64 [[IV_LVER_ORIG]], 1
+; CHECK-NEXT: [[EC_LVER_ORIG:%.*]] = icmp eq i64 [[IV_NEXT_LVER_ORIG]], [[N]]
+; CHECK-NEXT: br i1 [[EC_LVER_ORIG]], label %[[EXIT_LOOPEXIT:.*]], label %[[LOOP_LVER_ORIG]]
+; CHECK: [[LOOP_PH]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[LOOP_PH]] ], [ [[IV_NEXT:%.*]], %[[LATCH:.*]] ]
+; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT: [[L:%.*]] = load i32, ptr [[GEP_A]], align 4, !alias.scope [[META5:![0-9]+]]
+; CHECK-NEXT: br i1 [[C]], label %[[THEN:.*]], label %[[LATCH]]
+; CHECK: [[THEN]]:
+; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT: store i32 [[L]], ptr [[GEP_B]], align 4, !alias.scope [[META8:![0-9]+]], !noalias [[META5]]
+; CHECK-NEXT: br label %[[LATCH]]
+; CHECK: [[LATCH]]:
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT_LOOPEXIT2:.*]], label %[[LOOP]]
+; CHECK: [[EXIT_LOOPEXIT]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT_LOOPEXIT2]]:
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[LV:%.*]] = load i32, ptr [[A]], align 4
+; CHECK-NEXT: ret i32 [[LV]]
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %latch ]
+ %gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
+ %l = load i32, ptr %gep.a, align 4
+ br i1 %c, label %then, label %latch
+
+then:
+ %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
+ store i32 %l, ptr %gep.b, align 4
+ br label %latch
+
+latch:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %n
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ %lv = load i32, ptr %a, align 4
+ ret i32 %lv
+}
+
+; Versioning the inner loop of a nest: the exit block is inside the outer loop,
+; so the new exit edges feed a MemoryPhi that already has other incoming values.
+define void @nested(ptr %a, ptr %b, i64 %n) {
+; CHECK-LABEL: define void @nested(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = shl i64 [[N]], 2
+; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP0]]
+; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP0]]
+; CHECK-NEXT: br label %[[INNER_LVER_CHECK:.*]]
+; CHECK: [[INNER_LVER_CHECK]]:
+; CHECK-NEXT: [[J:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[J_NEXT:%.*]], %[[OUTER_LATCH:.*]] ]
+; CHECK-NEXT: store i32 0, ptr [[A]], align 4
+; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[B]], [[SCEVGEP1]]
+; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[A]], [[SCEVGEP]]
+; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], label %[[INNER_PH_LVER_ORIG:.*]], label %[[INNER_PH:.*]]
+; CHECK: [[INNER_PH_LVER_ORIG]]:
+; CHECK-NEXT: br label %[[INNER_LVER_ORIG:.*]]
+; CHECK: [[INNER_LVER_ORIG]]:
+; CHECK-NEXT: [[IV_LVER_ORIG:%.*]] = phi i64 [ 0, %[[INNER_PH_LVER_ORIG]] ], [ [[IV_NEXT_LVER_ORIG:%.*]], %[[INNER_LVER_ORIG]] ]
+; CHECK-NEXT: [[GEP_A_LVER_ORIG:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV_LVER_ORIG]]
+; CHECK-NEXT: [[L_LVER_ORIG:%.*]] = load i32, ptr [[GEP_A_LVER_ORIG]], align 4
+; CHECK-NEXT: [[GEP_B_LVER_ORIG:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV_LVER_ORIG]]
+; CHECK-NEXT: store i32 [[L_LVER_ORIG]], ptr [[GEP_B_LVER_ORIG]], align 4
+; CHECK-NEXT: [[IV_NEXT_LVER_ORIG]] = add nuw nsw i64 [[IV_LVER_ORIG]], 1
+; CHECK-NEXT: [[EC_LVER_ORIG:%.*]] = icmp eq i64 [[IV_NEXT_LVER_ORIG]], [[N]]
+; CHECK-NEXT: br i1 [[EC_LVER_ORIG]], label %[[OUTER_LATCH_LOOPEXIT:.*]], label %[[INNER_LVER_ORIG]]
+; CHECK: [[INNER_PH]]:
+; CHECK-NEXT: br label %[[INNER:.*]]
+; CHECK: [[INNER]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[INNER_PH]] ], [ [[IV_NEXT:%.*]], %[[INNER]] ]
+; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT: [[L:%.*]] = load i32, ptr [[GEP_A]], align 4, !alias.scope [[META10:![0-9]+]]
+; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT: store i32 [[L]], ptr [[GEP_B]], align 4, !alias.scope [[META13:![0-9]+]], !noalias [[META10]]
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EC]], label %[[OUTER_LATCH_LOOPEXIT2:.*]], label %[[INNER]]
+; CHECK: [[OUTER_LATCH_LOOPEXIT]]:
+; CHECK-NEXT: br label %[[OUTER_LATCH]]
+; CHECK: [[OUTER_LATCH_LOOPEXIT2]]:
+; CHECK-NEXT: br label %[[OUTER_LATCH]]
+; CHECK: [[OUTER_LATCH]]:
+; CHECK-NEXT: store i32 1, ptr [[B]], align 4
+; CHECK-NEXT: [[J_NEXT]] = add nuw nsw i64 [[J]], 1
+; CHECK-NEXT: [[EC_O:%.*]] = icmp eq i64 [[J_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EC_O]], label %[[EXIT:.*]], label %[[INNER_LVER_CHECK]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %outer
+
+outer:
+ %j = phi i64 [ 0, %entry ], [ %j.next, %outer.latch ]
+ store i32 0, ptr %a, align 4
+ br label %inner
+
+inner:
+ %iv = phi i64 [ 0, %outer ], [ %iv.next, %inner ]
+ %gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
+ %l = load i32, ptr %gep.a, align 4
+ %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
+ store i32 %l, ptr %gep.b, align 4
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %n
+ br i1 %ec, label %outer.latch, label %inner
+
+outer.latch:
+ store i32 1, ptr %b, align 4
+ %j.next = add nuw nsw i64 %j, 1
+ %ec.o = icmp eq i64 %j.next, %n
+ br i1 %ec.o, label %exit, label %outer
+
+exit:
+ ret void
+}
More information about the llvm-commits
mailing list