[llvm-branch-commits] [llvm] [Transforms] Preserve wave profiles across CFG rewrites (PR #226185)
Yaxun Liu via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Sun Nov 1 17:43:17 PST 2026
https://github.com/yxsamliu updated https://github.com/llvm/llvm-project/pull/226185
>From de63b815cf3874bbf03d7c20554315826e89cfe9 Mon Sep 17 00:00:00 2001
From: "Yaxun (Sam) Liu" <yaxun.liu at amd.com>
Date: Thu, 29 Oct 2026 10:37:23 +0000
Subject: [PATCH] [Transforms] Preserve wave profiles across CFG rewrites
GPU profiles record wave visits to individual blocks. Replacing or cloning
terminators can lose observations from unchanged blocks, or attach an old
count to a different execution event. For example, inserting a loop
preheader should retain counts on the original blocks while leaving the
new preheader unmeasured.
Preserve measured wave counts through switch lowering, structurization,
loop rotation, fixed-trip full unrolling and single-predecessor edge
rewrites when the original execution event remains unchanged. Retain
unaffected counts and the entry normalization, leave new or changed events
unmeasured, and do not revive previously invalid observations.
Clear imported wave identities during inlining: a helper's counts aggregate
all its calls and cannot describe one inlined copy. Invalidate split call
continuations and newly exposed unwind paths conservatively, and discard
recursive caller profiles whose invocation population has changed.
Batch supported edge splits, empty-block removals and block merges in
repeated CFG cleanup. Capture the validated profile once and rebuild its
metadata after the caller finishes, avoiding a whole-function scan for
every edit. Check for unhandled changes before recording an edit and before
applying the batch, so those changes cannot make old counts valid again.
Transfer branch hints only across equivalent decisions. Preserve switch
weights only where the destination of default traffic is known, including
the expected-weight marker.
RFC: https://discourse.llvm.org/t/rfc-gpu-wave-counts-for-spill-placement-costs/91927
---
llvm/include/llvm/IR/ProfDataUtils.h | 42 +++
.../llvm/Transforms/Utils/BasicBlockUtils.h | 11 +-
llvm/include/llvm/Transforms/Utils/Local.h | 8 +-
.../llvm/Transforms/Utils/LoopSimplify.h | 5 +-
.../include/llvm/Transforms/Utils/LoopUtils.h | 13 +-
llvm/lib/CodeGen/CodeGenPrepare.cpp | 23 +-
llvm/lib/IR/ProfDataUtils.cpp | 162 +++++++++-
.../lib/Transforms/Scalar/SimplifyCFGPass.cpp | 5 +-
llvm/lib/Transforms/Scalar/StructurizeCFG.cpp | 34 ++-
llvm/lib/Transforms/Utils/BasicBlockUtils.cpp | 75 ++++-
llvm/lib/Transforms/Utils/InlineFunction.cpp | 25 ++
llvm/lib/Transforms/Utils/Local.cpp | 19 +-
.../Transforms/Utils/LoopRotationUtils.cpp | 15 +
llvm/lib/Transforms/Utils/LoopSimplify.cpp | 47 ++-
llvm/lib/Transforms/Utils/LoopUnroll.cpp | 17 ++
llvm/lib/Transforms/Utils/LoopUtils.cpp | 12 +-
llvm/lib/Transforms/Utils/LowerSwitch.cpp | 123 +++++++-
llvm/lib/Transforms/Utils/SimplifyCFG.cpp | 30 +-
llvm/test/CodeGen/AMDGPU/itofp.i128.bf.ll | 12 +-
llvm/test/CodeGen/AMDGPU/itofp.i128.ll | 36 +--
.../CodeGenPrepare/wave-profile-edge.ll | 72 +++++
.../Transforms/LoopRotate/wave-profile.ll | 41 +++
.../LoopSimplify/block-wave-profile.ll | 49 +++
.../Transforms/LoopUnroll/wave-profile.ll | 141 +++++++++
.../Transforms/LowerSwitch/profile-weights.ll | 157 ++++++++++
.../Transforms/LowerSwitch/wave-profile.ll | 56 ++++
.../SimplifyCFG/wave-profile-edge.ll | 82 +++++
.../StructurizeCFG/branch-profile-transfer.ll | 27 ++
.../StructurizeCFG/uniformity-profile.ll | 4 +-
.../wave-profile-loop-prefix.ll | 53 ++++
.../Transforms/StructurizeCFG/wave-profile.ll | 50 ++++
llvm/unittests/IR/ProfDataUtilsTest.cpp | 224 ++++++++++++++
.../Transforms/Utils/CloningTest.cpp | 279 ++++++++++++++++++
.../Utils/LoopRotationUtilsTest.cpp | 68 +++++
34 files changed, 1895 insertions(+), 122 deletions(-)
create mode 100644 llvm/test/Transforms/CodeGenPrepare/wave-profile-edge.ll
create mode 100644 llvm/test/Transforms/LoopRotate/wave-profile.ll
create mode 100644 llvm/test/Transforms/LoopSimplify/block-wave-profile.ll
create mode 100644 llvm/test/Transforms/LoopUnroll/wave-profile.ll
create mode 100644 llvm/test/Transforms/LowerSwitch/profile-weights.ll
create mode 100644 llvm/test/Transforms/LowerSwitch/wave-profile.ll
create mode 100644 llvm/test/Transforms/SimplifyCFG/wave-profile-edge.ll
create mode 100644 llvm/test/Transforms/StructurizeCFG/branch-profile-transfer.ll
create mode 100644 llvm/test/Transforms/StructurizeCFG/wave-profile-loop-prefix.ll
create mode 100644 llvm/test/Transforms/StructurizeCFG/wave-profile.ll
diff --git a/llvm/include/llvm/IR/ProfDataUtils.h b/llvm/include/llvm/IR/ProfDataUtils.h
index c962fa528a9ef7..c4019cd3ec7299 100644
--- a/llvm/include/llvm/IR/ProfDataUtils.h
+++ b/llvm/include/llvm/IR/ProfDataUtils.h
@@ -16,6 +16,7 @@
#define LLVM_IR_PROFDATAUTILS_H
#include "llvm/ADT/BitVector.h"
+#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/STLFunctionalExtras.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/IR/Metadata.h"
@@ -24,6 +25,7 @@
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Compiler.h"
#include <cstddef>
+#include <optional>
#include <type_traits>
namespace llvm {
@@ -100,6 +102,7 @@ class LLVM_ABI BlockWaveCountPreserver {
TrackingMDNodeRef Profile;
SmallVector<Metadata *, 8> ProfileOperands;
SmallVector<BlockProfile> Blocks;
+ DenseMap<const BasicBlock *, unsigned> BlockIndices;
bool Invalidated = false;
public:
@@ -113,6 +116,45 @@ class LLVM_ABI BlockWaveCountPreserver {
void restore();
};
+/// Batch wave-profile updates in a caller that repeatedly splits or removes
+/// single-predecessor edges, or merges blocks into an unconditional
+/// predecessor. Record each supported change immediately before redirecting its
+/// edges. Other CFG changes are checked before a recorded change can hide them
+/// and again at apply(). Counts invalidated by those changes remain unmeasured.
+/// Metadata is rebuilt once, at apply(), which consumes the batch. Do not read
+/// wave counts mid-batch. Capture the profile before creating a split block.
+class LLVM_ABI BlockWaveCountUpdater {
+ struct BlockState {
+ WeakVH Block;
+ WeakVH Terminator;
+ TrackingMDNodeRef Metadata;
+ SmallVector<llvm::Metadata *, 6> Operands;
+ SmallVector<WeakVH, 2> Successors;
+ SmallVector<WeakVH, 2> Controls;
+ bool Removed = false;
+ };
+ Function &F;
+ std::optional<BlockWaveCountPreserver> Profile;
+ TrackingMDNodeRef Table;
+ DenseMap<const BasicBlock *, BlockState> Blocks;
+ bool Changed = false;
+ bool ConflictingMetadata = false;
+
+ void initialize();
+ BlockState capture(BasicBlock &BB);
+ void check(BasicBlock &BB);
+ void invalidateSuccessors(const BlockState &State);
+
+public:
+ explicit BlockWaveCountUpdater(Function &F) : F(F) {}
+ /// Whether the lazily captured profile is valid.
+ bool hasProfile();
+ void splitEdge(BasicBlock &Pred, BasicBlock &Succ, BasicBlock &NewBB);
+ void removeBlock(BasicBlock &BB);
+ void mergeBlock(BasicBlock &BB, BasicBlock &Pred);
+ void apply();
+};
+
/// Profile-based loop metadata that should be accessed only by using
/// \c llvm::getLoopEstimatedTripCount and \c llvm::setLoopEstimatedTripCount.
LLVM_ABI extern const char *LLVMLoopEstimatedTripCount;
diff --git a/llvm/include/llvm/Transforms/Utils/BasicBlockUtils.h b/llvm/include/llvm/Transforms/Utils/BasicBlockUtils.h
index 72dc2890ed27cb..0bd969c65dfa35 100644
--- a/llvm/include/llvm/Transforms/Utils/BasicBlockUtils.h
+++ b/llvm/include/llvm/Transforms/Utils/BasicBlockUtils.h
@@ -25,6 +25,8 @@
#include <cassert>
namespace llvm {
+
+class BlockWaveCountUpdater;
class CondBrInst;
class CycleInfo;
class LandingPadInst;
@@ -109,7 +111,8 @@ LLVM_ABI bool MergeBlockIntoPredecessor(
BasicBlock *BB, DomTreeUpdater *DTU = nullptr, LoopInfo *LI = nullptr,
MemorySSAUpdater *MSSAU = nullptr,
MemoryDependenceResults *MemDep = nullptr,
- bool PredecessorWithTwoSuccessors = false, DominatorTree *DT = nullptr);
+ bool PredecessorWithTwoSuccessors = false, DominatorTree *DT = nullptr,
+ BlockWaveCountUpdater *WPU = nullptr);
/// Merge block(s) sucessors, if possible. Return true if at least two
/// of the blocks were merged together.
@@ -389,7 +392,8 @@ inline BasicBlock *splitBlockBefore(BasicBlock *Old, Instruction *SplitPt,
LLVM_ABI BasicBlock *SplitBlockPredecessors(
BasicBlock *BB, ArrayRef<BasicBlock *> Preds, const char *Suffix,
DominatorTree *DT, LoopInfo *LI = nullptr,
- MemorySSAUpdater *MSSAU = nullptr, bool PreserveLCSSA = false);
+ MemorySSAUpdater *MSSAU = nullptr, bool PreserveLCSSA = false,
+ BlockWaveCountUpdater *WPU = nullptr);
/// This method introduces at least one new basic block into the function and
/// moves some of the predecessors of BB to be predecessors of the new block.
@@ -408,7 +412,8 @@ LLVM_ABI BasicBlock *SplitBlockPredecessors(
LLVM_ABI BasicBlock *SplitBlockPredecessors(
BasicBlock *BB, ArrayRef<BasicBlock *> Preds, const char *Suffix,
DomTreeUpdater *DTU = nullptr, LoopInfo *LI = nullptr,
- MemorySSAUpdater *MSSAU = nullptr, bool PreserveLCSSA = false);
+ MemorySSAUpdater *MSSAU = nullptr, bool PreserveLCSSA = false,
+ BlockWaveCountUpdater *WPU = nullptr);
/// This method transforms the landing pad, OrigBB, by introducing two new basic
/// blocks into the function. One of those new basic blocks gets the
diff --git a/llvm/include/llvm/Transforms/Utils/Local.h b/llvm/include/llvm/Transforms/Utils/Local.h
index 214baf905fb24d..f2c14a5c9c6fc7 100644
--- a/llvm/include/llvm/Transforms/Utils/Local.h
+++ b/llvm/include/llvm/Transforms/Utils/Local.h
@@ -24,6 +24,8 @@
namespace llvm {
+class BlockWaveCountUpdater;
+
class DataLayout;
class Value;
class WeakTrackingVH;
@@ -152,7 +154,8 @@ LLVM_ABI void MergeBasicBlockIntoOnlyPred(BasicBlock *BB,
/// successor block and return true. If we can't transform, return false.
LLVM_ABI bool
TryToSimplifyUncondBranchFromEmptyBlock(BasicBlock *BB,
- DomTreeUpdater *DTU = nullptr);
+ DomTreeUpdater *DTU = nullptr,
+ BlockWaveCountUpdater *WPU = nullptr);
/// Check for and eliminate duplicate PHI nodes in this block. This doesn't try
/// to be clever about PHI nodes which differ only in the order of the incoming
@@ -179,7 +182,8 @@ LLVM_ABI extern cl::opt<bool> RequireAndPreserveDomTree;
LLVM_ABI bool simplifyCFG(BasicBlock *BB, const TargetTransformInfo &TTI,
DomTreeUpdater *DTU = nullptr,
const SimplifyCFGOptions &Options = {},
- ArrayRef<WeakVH> LoopHeaders = {});
+ ArrayRef<WeakVH> LoopHeaders = {},
+ BlockWaveCountUpdater *WPU = nullptr);
/// This function is used to flatten a CFG. For example, it uses parallel-and
/// and parallel-or mode to collapse if-conditions and merge if-regions with
diff --git a/llvm/include/llvm/Transforms/Utils/LoopSimplify.h b/llvm/include/llvm/Transforms/Utils/LoopSimplify.h
index 722406110b3853..5495aae442dcd3 100644
--- a/llvm/include/llvm/Transforms/Utils/LoopSimplify.h
+++ b/llvm/include/llvm/Transforms/Utils/LoopSimplify.h
@@ -43,6 +43,8 @@
namespace llvm {
+class BlockWaveCountUpdater;
+
class AssumptionCache;
class DominatorTree;
class Loop;
@@ -64,7 +66,8 @@ class LoopSimplifyPass : public OptionalPassInfoMixin<LoopSimplifyPass> {
/// analyses if they're non-null, and LCSSA if \c PreserveLCSSA is true.
LLVM_ABI bool simplifyLoop(Loop *L, DominatorTree *DT, LoopInfo *LI,
ScalarEvolution *SE, AssumptionCache *AC,
- MemorySSAUpdater *MSSAU, bool PreserveLCSSA);
+ MemorySSAUpdater *MSSAU, bool PreserveLCSSA,
+ BlockWaveCountUpdater *WPU = nullptr);
} // end namespace llvm
diff --git a/llvm/include/llvm/Transforms/Utils/LoopUtils.h b/llvm/include/llvm/Transforms/Utils/LoopUtils.h
index 5b539ee515ca78..600f31ec8440af 100644
--- a/llvm/include/llvm/Transforms/Utils/LoopUtils.h
+++ b/llvm/include/llvm/Transforms/Utils/LoopUtils.h
@@ -19,6 +19,8 @@
namespace llvm {
+class BlockWaveCountUpdater;
+
template <typename T> class DomTreeNodeBase;
using DomTreeNode = DomTreeNodeBase<BasicBlock>;
class AssumptionCache;
@@ -54,10 +56,10 @@ typedef std::pair<const RuntimeCheckingPtrGroup *,
template <typename T, unsigned N> class SmallSetVector;
template <typename T, unsigned N> class SmallPriorityWorklist;
-LLVM_ABI BasicBlock *InsertPreheaderForLoop(Loop *L, DominatorTree *DT,
- LoopInfo *LI,
- MemorySSAUpdater *MSSAU,
- bool PreserveLCSSA);
+LLVM_ABI BasicBlock *
+InsertPreheaderForLoop(Loop *L, DominatorTree *DT, LoopInfo *LI,
+ MemorySSAUpdater *MSSAU, bool PreserveLCSSA,
+ BlockWaveCountUpdater *WPU = nullptr);
/// Ensure that all exit blocks of the loop are dedicated exits.
///
@@ -66,7 +68,8 @@ LLVM_ABI BasicBlock *InsertPreheaderForLoop(Loop *L, DominatorTree *DT,
/// tree and loop info if provided, and will preserve LCSSA if requested.
LLVM_ABI bool formDedicatedExitBlocks(Loop *L, DominatorTree *DT, LoopInfo *LI,
MemorySSAUpdater *MSSAU,
- bool PreserveLCSSA);
+ bool PreserveLCSSA,
+ BlockWaveCountUpdater *WPU = nullptr);
/// Ensures LCSSA form for every instruction from the Worklist in the scope of
/// innermost containing loop.
diff --git a/llvm/lib/CodeGen/CodeGenPrepare.cpp b/llvm/lib/CodeGen/CodeGenPrepare.cpp
index cc6865d02d87f3..69f4f2afb8f827 100644
--- a/llvm/lib/CodeGen/CodeGenPrepare.cpp
+++ b/llvm/lib/CodeGen/CodeGenPrepare.cpp
@@ -279,7 +279,7 @@ class CodeGenPrepare {
bool eliminateMostlyEmptyBlocks(Function &F, bool &ResetLI);
BasicBlock *findDestBlockOfMergeableEmptyBlock(BasicBlock *BB);
bool canMergeBlocks(const BasicBlock *BB, const BasicBlock *DestBB) const;
- bool eliminateMostlyEmptyBlock(BasicBlock *BB);
+ bool eliminateMostlyEmptyBlock(BasicBlock *BB, BlockWaveCountUpdater &WPU);
bool isMergingEmptyBlockProfitable(BasicBlock *BB, BasicBlock *DestBB,
bool isPreheader);
bool makeBitReverse(Instruction &I);
@@ -730,6 +730,7 @@ void CodeGenPrepare::removeAllAssertingVHReferences(Value *V) {
bool CodeGenPrepare::eliminateFallThrough(Function &F) {
bool Changed = false;
SmallPtrSet<BasicBlock *, 8> Preds;
+ BlockWaveCountUpdater WPU(F);
// Scan all of the blocks in the function, except for the entry block.
for (auto &Block : llvm::drop_begin(F)) {
auto *BB = &Block;
@@ -748,7 +749,8 @@ bool CodeGenPrepare::eliminateFallThrough(Function &F) {
LLVM_DEBUG(dbgs() << "To merge:\n" << *BB << "\n\n\n");
// Merge BB into SinglePred and delete it.
- MergeBlockIntoPredecessor(BB, DTU, LI);
+ MergeBlockIntoPredecessor(BB, DTU, LI, nullptr, nullptr, false, nullptr,
+ &WPU);
Preds.insert(SinglePred);
if (IsHugeFunc) {
@@ -759,6 +761,8 @@ bool CodeGenPrepare::eliminateFallThrough(Function &F) {
}
}
+ WPU.apply();
+
// (Repeatedly) merging blocks into their predecessors can create redundant
// debug intrinsics.
for (auto *Pred : Preds)
@@ -811,6 +815,7 @@ bool CodeGenPrepare::eliminateMostlyEmptyBlocks(Function &F, bool &ResetLI) {
ResetLI = false;
bool MadeChange = false;
+ BlockWaveCountUpdater WPU(F);
SmallPtrSet<PHINode *, 32> KnownNonDeadPHIs;
// Note that this intentionally skips the entry block.
for (auto &Block : llvm::drop_begin(F)) {
@@ -828,9 +833,10 @@ bool CodeGenPrepare::eliminateMostlyEmptyBlocks(Function &F, bool &ResetLI) {
!isMergingEmptyBlockProfitable(BB, DestBB, Preheaders.count(BB)))
continue;
- ResetLI |= eliminateMostlyEmptyBlock(BB);
+ ResetLI |= eliminateMostlyEmptyBlock(BB, WPU);
MadeChange = true;
}
+ WPU.apply();
return MadeChange;
}
@@ -1007,7 +1013,8 @@ static void replaceAllUsesWith(Value *Old, Value *New,
/// Eliminate a basic block that has only phi's and an unconditional branch in
/// it.
/// Indicate that the LoopInfo was modified only if it wasn't updated.
-bool CodeGenPrepare::eliminateMostlyEmptyBlock(BasicBlock *BB) {
+bool CodeGenPrepare::eliminateMostlyEmptyBlock(BasicBlock *BB,
+ BlockWaveCountUpdater &WPU) {
UncondBrInst *BI = cast<UncondBrInst>(BB->getTerminator());
BasicBlock *DestBB = BI->getSuccessor();
@@ -1021,7 +1028,8 @@ bool CodeGenPrepare::eliminateMostlyEmptyBlock(BasicBlock *BB) {
assert(SinglePred == BB &&
"Single predecessor not the same as predecessor");
// Merge DestBB into SinglePred/BB and delete it.
- MergeBlockIntoPredecessor(DestBB, DTU, LI);
+ MergeBlockIntoPredecessor(DestBB, DTU, LI, nullptr, nullptr, false,
+ nullptr, &WPU);
// Note: BB(=SinglePred) will not be deleted on this path.
// DestBB(=its single successor) is the one that was deleted.
LLVM_DEBUG(dbgs() << "AFTER:\n" << *SinglePred << "\n\n\n");
@@ -1035,6 +1043,11 @@ bool CodeGenPrepare::eliminateMostlyEmptyBlock(BasicBlock *BB) {
}
}
+ // Removing a block on a single predecessor's edge preserves the surrounding
+ // wave events. Multiple predecessors may change wave reconvergence.
+ if (BB->getSinglePredecessor())
+ WPU.removeBlock(*BB);
+
// Otherwise, we have multiple predecessors of BB. Update the PHIs in DestBB
// to handle the new incoming edges it is about to have.
for (PHINode &PN : DestBB->phis()) {
diff --git a/llvm/lib/IR/ProfDataUtils.cpp b/llvm/lib/IR/ProfDataUtils.cpp
index 401c8bed12621a..7790654d8a725d 100644
--- a/llvm/lib/IR/ProfDataUtils.cpp
+++ b/llvm/lib/IR/ProfDataUtils.cpp
@@ -358,24 +358,26 @@ BlockWaveCountPreserver::BlockWaveCountPreserver(Function &F) : F(F) {
SmallVector<Metadata *, 6> Operands;
if (MD)
Operands.assign(MD->op_begin(), MD->op_end());
+ BlockIndices.try_emplace(&BB, Blocks.size());
Blocks.push_back({&BB, Term, TrackingMDNodeRef(MD), std::move(Operands),
Mapping->BlockIds[Index], Mapping->HasCounts[Index]});
}
}
void BlockWaveCountPreserver::invalidate(const BasicBlock &BB) {
- for (BlockProfile &Block : Blocks)
- if (Block.Block == &BB) {
- Block.HasCount = false;
- Invalidated = true;
- }
+ auto It = BlockIndices.find(&BB);
+ if (It != BlockIndices.end() && Blocks[It->second].Block == &BB) {
+ Blocks[It->second].HasCount = false;
+ Invalidated = true;
+ }
}
void BlockWaveCountPreserver::forget(const BasicBlock &BB) {
- unsigned Size = Blocks.size();
- llvm::erase_if(Blocks,
- [&](const BlockProfile &Block) { return Block.Block == &BB; });
- Invalidated |= Blocks.size() != Size;
+ auto It = BlockIndices.find(&BB);
+ if (It != BlockIndices.end() && Blocks[It->second].Block == &BB) {
+ Blocks[It->second].Block = nullptr;
+ Invalidated = true;
+ }
}
void BlockWaveCountPreserver::restore() {
@@ -433,6 +435,148 @@ void BlockWaveCountPreserver::restore() {
writeWaveProfile(F, Table, Ids, HasCounts);
}
+bool BlockWaveCountUpdater::hasProfile() {
+ initialize();
+ return Profile->hasProfile();
+}
+
+BlockWaveCountUpdater::BlockState
+BlockWaveCountUpdater::capture(BasicBlock &BB) {
+ Instruction *Term = BB.getTerminator();
+ MDNode *MD = Term->getMetadata(LLVMContext::MD_wave_profile_block);
+ BlockState State{&BB, Term, TrackingMDNodeRef(MD), {}, {}, {}};
+ if (MD)
+ State.Operands.assign(MD->op_begin(), MD->op_end());
+ for (BasicBlock *Succ : successors(&BB))
+ State.Successors.push_back(Succ);
+ for (Value *Op : Term->operand_values())
+ if (!isa<BasicBlock>(Op))
+ State.Controls.push_back(Op);
+ return State;
+}
+
+void BlockWaveCountUpdater::initialize() {
+ if (Profile)
+ return;
+ Profile.emplace(F);
+ Table.reset(F.getMetadata(LLVMContext::MD_wave_profile));
+ if (Profile->hasProfile())
+ for (BasicBlock &BB : F)
+ Blocks.try_emplace(&BB, capture(BB));
+}
+
+void BlockWaveCountUpdater::invalidateSuccessors(const BlockState &State) {
+ for (const WeakVH &Succ : State.Successors)
+ if (auto *BB = dyn_cast_or_null<BasicBlock>(Succ))
+ Profile->invalidate(*BB);
+}
+
+void BlockWaveCountUpdater::check(BasicBlock &BB) {
+ BlockState Current = capture(BB);
+ auto It = Blocks.find(&BB);
+ if (It == Blocks.end() || It->second.Block != &BB || It->second.Removed) {
+ // An unrecorded new block has no measured event. Do not reinterpret copied
+ // or independently supplied metadata as one of our original identities.
+ ConflictingMetadata |= Current.Metadata.get() != nullptr;
+ if (It != Blocks.end() && !It->second.Removed)
+ invalidateSuccessors(It->second);
+ invalidateSuccessors(Current);
+ Blocks[&BB] = std::move(Current);
+ return;
+ }
+
+ BlockState &Saved = It->second;
+ if (Saved.Terminator != Current.Terminator ||
+ Saved.Metadata != Current.Metadata ||
+ (Saved.Metadata &&
+ !hasWaveProfileOperands(Saved.Metadata, Saved.Operands)) ||
+ Saved.Successors != Current.Successors ||
+ Saved.Controls != Current.Controls) {
+ Profile->invalidate(BB);
+ invalidateSuccessors(Saved);
+ invalidateSuccessors(Current);
+ }
+ Saved = std::move(Current);
+}
+
+void BlockWaveCountUpdater::splitEdge(BasicBlock &Pred, BasicBlock &Succ,
+ BasicBlock &NewBB) {
+ if (!hasProfile())
+ return;
+ assert(Pred.getParent() == &F && Succ.getParent() == &F &&
+ NewBB.getParent() == &F && "wave update belongs to another function");
+ check(Pred);
+ auto It = Blocks.find(&NewBB);
+ if (It != Blocks.end() && !It->second.Removed)
+ invalidateSuccessors(It->second);
+ Blocks[&NewBB] = capture(NewBB);
+ for (WeakVH &Target : Blocks.find(&Pred)->second.Successors)
+ if (Target == &Succ)
+ Target = &NewBB;
+ Changed = true;
+}
+
+void BlockWaveCountUpdater::removeBlock(BasicBlock &BB) {
+ if (!hasProfile())
+ return;
+ BasicBlock *Pred = BB.getSinglePredecessor();
+ assert(BB.getParent() == &F && "wave update belongs to another function");
+ assert(Pred && BB.getSingleSuccessor() &&
+ "expected a single-predecessor edge");
+ check(*Pred);
+ check(BB);
+ for (WeakVH &Target : Blocks.find(Pred)->second.Successors)
+ if (Target == &BB)
+ Target = BB.getSingleSuccessor();
+ Blocks.find(&BB)->second.Removed = true;
+ Profile->forget(BB);
+ Changed = true;
+}
+
+void BlockWaveCountUpdater::mergeBlock(BasicBlock &BB, BasicBlock &Pred) {
+ if (!hasProfile())
+ return;
+ assert(BB.getParent() == &F && Pred.getParent() == &F &&
+ "wave update belongs to another function");
+ check(Pred);
+ check(BB);
+ BlockState &SavedPred = Blocks.find(&Pred)->second;
+ BlockState &SavedBB = Blocks.find(&BB)->second;
+ SavedPred.Terminator = SavedBB.Terminator;
+ SavedPred.Metadata.reset();
+ SavedPred.Operands.clear();
+ SavedPred.Successors = SavedBB.Successors;
+ SavedPred.Controls = SavedBB.Controls;
+ SavedBB.Removed = true;
+ Profile->forget(BB);
+ Changed = true;
+}
+
+void BlockWaveCountUpdater::apply() {
+ if (!Changed)
+ return;
+ Changed = false;
+ // A newer profile takes precedence, subject to pending invalidations in the
+ // snapshot. Do not compare its block records against the old graph.
+ if (F.getMetadata(LLVMContext::MD_wave_profile) != Table) {
+ Profile->restore();
+ return;
+ }
+ // Check every block before rebuilding any record, so an unhandled rewrite
+ // cannot acquire fresh edges while retaining an old measured count.
+ for (BasicBlock &BB : F)
+ check(BB);
+ for (const auto &Entry : Blocks) {
+ const BlockState &State = Entry.second;
+ if (!State.Block && !State.Removed)
+ invalidateSuccessors(State);
+ }
+ if (ConflictingMetadata)
+ clearBlockWaveCounts(F);
+ else
+ Profile->restore();
+}
+
// MD_prof nodes have the following layout
//
// In general:
diff --git a/llvm/lib/Transforms/Scalar/SimplifyCFGPass.cpp b/llvm/lib/Transforms/Scalar/SimplifyCFGPass.cpp
index 647130deb8457d..e4c16affda7680 100644
--- a/llvm/lib/Transforms/Scalar/SimplifyCFGPass.cpp
+++ b/llvm/lib/Transforms/Scalar/SimplifyCFGPass.cpp
@@ -34,6 +34,7 @@
#include "llvm/IR/CFG.h"
#include "llvm/IR/Dominators.h"
#include "llvm/IR/Instructions.h"
+#include "llvm/IR/ProfDataUtils.h"
#include "llvm/IR/ValueHandle.h"
#include "llvm/InitializePasses.h"
#include "llvm/Pass.h"
@@ -203,6 +204,7 @@ static bool iterativelySimplifyCFG(Function &F, const TargetTransformInfo &TTI,
SmallVector<WeakVH, 16> LoopHeaders(UniqueLoopHeaders.begin(),
UniqueLoopHeaders.end());
+ BlockWaveCountUpdater WPU(F);
unsigned IterCnt = 0;
(void)IterCnt;
while (LocalChange) {
@@ -221,13 +223,14 @@ static bool iterativelySimplifyCFG(Function &F, const TargetTransformInfo &TTI,
while (BBIt != F.end() && DTU->isBBPendingDeletion(&*BBIt))
++BBIt;
}
- if (simplifyCFG(&BB, TTI, DTU, Options, LoopHeaders)) {
+ if (simplifyCFG(&BB, TTI, DTU, Options, LoopHeaders, &WPU)) {
LocalChange = true;
++NumSimpl;
}
}
Changed |= LocalChange;
}
+ WPU.apply();
return Changed;
}
diff --git a/llvm/lib/Transforms/Scalar/StructurizeCFG.cpp b/llvm/lib/Transforms/Scalar/StructurizeCFG.cpp
index f6004c3795e1b1..9514b823efe07d 100644
--- a/llvm/lib/Transforms/Scalar/StructurizeCFG.cpp
+++ b/llvm/lib/Transforms/Scalar/StructurizeCFG.cpp
@@ -81,8 +81,10 @@ using MaybeCondBranchWeights = std::optional<class CondBranchWeights>;
class CondBranchWeights {
uint32_t TrueWeight;
uint32_t FalseWeight;
+ bool IsExpected;
- CondBranchWeights(uint32_t T, uint32_t F) : TrueWeight(T), FalseWeight(F) {}
+ CondBranchWeights(uint32_t T, uint32_t F, bool E)
+ : TrueWeight(T), FalseWeight(F), IsExpected(E) {}
public:
static MaybeCondBranchWeights tryParse(const CondBrInst &Br) {
@@ -90,7 +92,7 @@ class CondBranchWeights {
if (!extractBranchWeights(Br, T, F))
return std::nullopt;
- return CondBranchWeights(T, F);
+ return CondBranchWeights(T, F, hasBranchWeightOrigin(Br));
}
static void setMetadata(CondBrInst &Br,
@@ -98,17 +100,18 @@ class CondBranchWeights {
if (!Weights)
return;
uint32_t Arr[] = {Weights->TrueWeight, Weights->FalseWeight};
- setBranchWeights(Br, Arr, false);
+ setBranchWeights(Br, Arr, Weights->IsExpected);
}
CondBranchWeights invert() const {
- return CondBranchWeights{FalseWeight, TrueWeight};
+ return CondBranchWeights{FalseWeight, TrueWeight, IsExpected};
}
};
struct PredInfo {
Value *Pred;
MaybeCondBranchWeights Weights;
+ MDNode *Uniformity = nullptr;
};
using BBPredicates = DenseMap<BasicBlock *, PredInfo>;
@@ -269,12 +272,14 @@ class StructurizeCFG {
const TargetTransformInfo *TTI;
Function *Func;
Region *ParentRegion;
+ BlockWaveCountPreserver *WaveProfile = nullptr;
DominatorTree *DT;
SmallVector<RegionNode *, 8> Order;
BBSet Visited;
BBSet FlowSet;
+ BBSet ChangedExecutionBlocks;
// The terminator carries a profile of the block's execution, independently
// of its branch condition. Keep it when replacing the terminator of the
@@ -555,7 +560,8 @@ PredInfo StructurizeCFG::buildCondition(CondBrInst *Term, unsigned Idx,
if (Weights)
Weights = Weights->invert();
}
- return {Cond, Weights};
+ return {Cond, Weights,
+ Term->getMetadata(LLVMContext::MD_branch_uniformity_profile)};
}
/// Analyze the predecessors of each block and build up predicates
@@ -672,7 +678,11 @@ void StructurizeCFG::insertConditions(bool Loops, SSAUpdaterBulk &PhiInserter) {
if (ParentInfo.Pred) {
Term->setCondition(ParentInfo.Pred);
- CondBranchWeights::setMetadata(*Term, ParentInfo.Weights);
+ if (!ChangedExecutionBlocks.contains(Parent)) {
+ CondBranchWeights::setMetadata(*Term, ParentInfo.Weights);
+ Term->setMetadata(LLVMContext::MD_branch_uniformity_profile,
+ ParentInfo.Uniformity);
+ }
} else {
if (!Dominator.resultIsRememberedBlock())
PhiInserter.AddAvailableValue(Variable, Dominator.result(), Default);
@@ -1082,8 +1092,11 @@ std::pair<BasicBlock *, DebugLoc> StructurizeCFG::needPrefix(bool NeedEmpty) {
if (!NeedEmpty || Entry->getFirstInsertionPt() == Entry->end()) {
// An empty prefix reused as a loop header also executes on backedges.
// Its old execution profile does not describe those additional visits.
- if (NeedEmpty)
+ if (NeedEmpty) {
BlockUniformityProfiles.erase(Entry);
+ WaveProfile->invalidate(*Entry);
+ ChangedExecutionBlocks.insert(Entry);
+ }
return {Entry, DL};
}
}
@@ -1345,6 +1358,9 @@ bool StructurizeCFG::run(Region *R, DominatorTree *DT,
this->TTI = TTI;
Func = R->getEntry()->getParent();
+ BlockWaveCountPreserver SavedWaveProfile(*Func);
+ WaveProfile = &SavedWaveProfile;
+
ParentRegion = R;
orderNodes();
@@ -1366,6 +1382,9 @@ bool StructurizeCFG::run(Region *R, DominatorTree *DT,
BB->getTerminator()->setMetadata(LLVMContext::MD_block_uniformity_profile,
MD);
+ SavedWaveProfile.restore();
+ WaveProfile = nullptr;
+
// Cleanup
BlockUniformityProfiles.clear();
Order.clear();
@@ -1378,6 +1397,7 @@ bool StructurizeCFG::run(Region *R, DominatorTree *DT,
LoopPreds.clear();
LoopConds.clear();
FlowSet.clear();
+ ChangedExecutionBlocks.clear();
return true;
}
diff --git a/llvm/lib/Transforms/Utils/BasicBlockUtils.cpp b/llvm/lib/Transforms/Utils/BasicBlockUtils.cpp
index c40a5e32be3135..610fa198a82770 100644
--- a/llvm/lib/Transforms/Utils/BasicBlockUtils.cpp
+++ b/llvm/lib/Transforms/Utils/BasicBlockUtils.cpp
@@ -34,6 +34,7 @@
#include "llvm/IR/Instruction.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/LLVMContext.h"
+#include "llvm/IR/ProfDataUtils.h"
#include "llvm/IR/Type.h"
#include "llvm/IR/User.h"
#include "llvm/IR/Value.h"
@@ -45,6 +46,7 @@
#include "llvm/Transforms/Utils/Local.h"
#include <cassert>
#include <cstdint>
+#include <optional>
#include <string>
#include <utility>
#include <vector>
@@ -238,7 +240,8 @@ bool llvm::MergeBlockIntoPredecessor(BasicBlock *BB, DomTreeUpdater *DTU,
LoopInfo *LI, MemorySSAUpdater *MSSAU,
MemoryDependenceResults *MemDep,
bool PredecessorWithTwoSuccessors,
- DominatorTree *DT) {
+ DominatorTree *DT,
+ BlockWaveCountUpdater *WPU) {
if (BB->hasAddressTaken())
return false;
@@ -288,6 +291,16 @@ bool llvm::MergeBlockIntoPredecessor(BasicBlock *BB, DomTreeUpdater *DTU,
LLVM_DEBUG(dbgs() << "Merging: " << BB->getName() << " into "
<< PredBB->getName() << "\n");
+ std::optional<BlockWaveCountPreserver> WaveProfile;
+ if (!PredecessorWithTwoSuccessors) {
+ if (WPU)
+ WPU->mergeBlock(*BB, *PredBB);
+ else {
+ WaveProfile.emplace(*BB->getParent());
+ WaveProfile->forget(*BB);
+ }
+ }
+
// Begin by getting rid of unneeded PHIs.
SmallVector<AssertingVH<Value>, 4> IncomingValues;
if (isa<PHINode>(BB->front())) {
@@ -365,6 +378,9 @@ bool llvm::MergeBlockIntoPredecessor(BasicBlock *BB, DomTreeUpdater *DTU,
// The merged block retains the predecessor's executions. Its terminator
// changes, but its block-uniformity classification still applies.
STI->copyMetadata(*PTI, {LLVMContext::MD_block_uniformity_profile});
+ // The transplanted terminator must not replace the predecessor's event.
+ if (WPU ? WPU->hasProfile() : WaveProfile->hasProfile())
+ STI->setMetadata(LLVMContext::MD_wave_profile_block, nullptr);
// Delete the unconditional branch from the predecessor.
PredBB->back().eraseFromParent();
@@ -403,6 +419,9 @@ bool llvm::MergeBlockIntoPredecessor(BasicBlock *BB, DomTreeUpdater *DTU,
// Finally, erase the old block and update dominator info.
DeleteDeadBlock(BB, DTU);
+ if (WaveProfile)
+ WaveProfile->restore();
+
return true;
}
@@ -412,13 +431,18 @@ bool llvm::MergeBlockSuccessorsIntoGivenBlocks(
assert(!MergeBlocks.empty() && "MergeBlocks should not be empty");
bool BlocksHaveBeenMerged = false;
+ DenseMap<Function *, std::unique_ptr<BlockWaveCountUpdater>> WaveUpdates;
while (!MergeBlocks.empty()) {
BasicBlock *BB = *MergeBlocks.begin();
BasicBlock *Dest = BB->getSingleSuccessor();
if (Dest && (!L || L->contains(Dest))) {
BasicBlock *Fold = Dest->getUniquePredecessor();
(void)Fold;
- if (MergeBlockIntoPredecessor(Dest, DTU, LI)) {
+ auto &WPU = WaveUpdates[BB->getParent()];
+ if (!WPU)
+ WPU = std::make_unique<BlockWaveCountUpdater>(*BB->getParent());
+ if (MergeBlockIntoPredecessor(Dest, DTU, LI, nullptr, nullptr, false,
+ nullptr, WPU.get())) {
assert(Fold == BB &&
"Expecting BB to be unique predecessor of the Dest block");
MergeBlocks.erase(Dest);
@@ -428,6 +452,8 @@ bool llvm::MergeBlockSuccessorsIntoGivenBlocks(
} else
MergeBlocks.erase(BB);
}
+ for (auto &Entry : WaveUpdates)
+ Entry.second->apply();
return BlocksHaveBeenMerged;
}
@@ -1311,11 +1337,10 @@ static void SplitLandingPadPredecessorsImpl(
DomTreeUpdater *DTU, DominatorTree *DT, LoopInfo *LI,
MemorySSAUpdater *MSSAU, bool PreserveLCSSA);
-static BasicBlock *
-SplitBlockPredecessorsImpl(BasicBlock *BB, ArrayRef<BasicBlock *> Preds,
- const char *Suffix, DomTreeUpdater *DTU,
- DominatorTree *DT, LoopInfo *LI,
- MemorySSAUpdater *MSSAU, bool PreserveLCSSA) {
+static BasicBlock *SplitBlockPredecessorsImpl(
+ BasicBlock *BB, ArrayRef<BasicBlock *> Preds, const char *Suffix,
+ DomTreeUpdater *DTU, DominatorTree *DT, LoopInfo *LI,
+ MemorySSAUpdater *MSSAU, bool PreserveLCSSA, BlockWaveCountUpdater *WPU) {
// Do not attempt to split that which cannot be split.
if (!BB->canSplitPredecessors())
return nullptr;
@@ -1331,6 +1356,17 @@ SplitBlockPredecessorsImpl(BasicBlock *BB, ArrayRef<BasicBlock *> Preds,
return NewBBs[0];
}
+ // A new unconditional block on one predecessor's edge does not change
+ // existing wave events. The new block remains unmeasured.
+ std::optional<BlockWaveCountPreserver> WaveProfile;
+ if (Preds.size() == 1) {
+ // Capture the old graph before creating the split block.
+ if (WPU)
+ WPU->hasProfile();
+ else
+ WaveProfile.emplace(*BB->getParent());
+ }
+
// Create new basic block, insert right before the original block.
BasicBlock *NewBB = BasicBlock::Create(
BB->getContext(), BB->getName() + Suffix, BB->getParent(), BB);
@@ -1355,6 +1391,9 @@ SplitBlockPredecessorsImpl(BasicBlock *BB, ArrayRef<BasicBlock *> Preds,
} else
BI->setDebugLoc(BB->getFirstNonPHIOrPseudoProbe()->getDebugLoc());
+ if (WPU && Preds.size() == 1)
+ WPU->splitEdge(*Preds.front(), *BB, *NewBB);
+
// Move the edges from Preds to point to NewBB instead of BB.
for (BasicBlock *Pred : Preds) {
// This is slightly more strict than necessary; the minimum requirement
@@ -1398,6 +1437,9 @@ SplitBlockPredecessorsImpl(BasicBlock *BB, ArrayRef<BasicBlock *> Preds,
}
}
+ if (WaveProfile)
+ WaveProfile->restore();
+
return NewBB;
}
@@ -1405,18 +1447,19 @@ BasicBlock *llvm::SplitBlockPredecessors(BasicBlock *BB,
ArrayRef<BasicBlock *> Preds,
const char *Suffix, DominatorTree *DT,
LoopInfo *LI, MemorySSAUpdater *MSSAU,
- bool PreserveLCSSA) {
+ bool PreserveLCSSA,
+ BlockWaveCountUpdater *WPU) {
return SplitBlockPredecessorsImpl(BB, Preds, Suffix, /*DTU=*/nullptr, DT, LI,
- MSSAU, PreserveLCSSA);
+ MSSAU, PreserveLCSSA, WPU);
}
-BasicBlock *llvm::SplitBlockPredecessors(BasicBlock *BB,
- ArrayRef<BasicBlock *> Preds,
- const char *Suffix,
- DomTreeUpdater *DTU, LoopInfo *LI,
- MemorySSAUpdater *MSSAU,
- bool PreserveLCSSA) {
+BasicBlock *
+llvm::SplitBlockPredecessors(BasicBlock *BB, ArrayRef<BasicBlock *> Preds,
+ const char *Suffix, DomTreeUpdater *DTU,
+ LoopInfo *LI, MemorySSAUpdater *MSSAU,
+ bool PreserveLCSSA, BlockWaveCountUpdater *WPU) {
return SplitBlockPredecessorsImpl(BB, Preds, Suffix, DTU,
- /*DT=*/nullptr, LI, MSSAU, PreserveLCSSA);
+ /*DT=*/nullptr, LI, MSSAU, PreserveLCSSA,
+ WPU);
}
static void SplitLandingPadPredecessorsImpl(
diff --git a/llvm/lib/Transforms/Utils/InlineFunction.cpp b/llvm/lib/Transforms/Utils/InlineFunction.cpp
index e8d630491dd750..0bc8e7ad8e2838 100644
--- a/llvm/lib/Transforms/Utils/InlineFunction.cpp
+++ b/llvm/lib/Transforms/Utils/InlineFunction.cpp
@@ -2730,6 +2730,18 @@ void llvm::InlineFunctionImpl(CallBase &CB, InlineFunctionInfo &IFI,
assert(CalledFunc && !CalledFunc->isDeclaration() &&
"CanInlineCallSite should have verified direct call to definition");
+ // Recursive inlining changes the invocation population of the caller itself.
+ if (Caller == CalledFunc && Caller->getMetadata(LLVMContext::MD_wave_profile))
+ clearBlockWaveCounts(*Caller);
+ if (auto *II = dyn_cast<InvokeInst>(&CB)) {
+ // Newly exposed unwind paths may change the wave populations at either
+ // continuation; do not infer their visits from scalar return counts.
+ II->getNormalDest()->getTerminator()->setMetadata(
+ LLVMContext::MD_wave_profile_block, nullptr);
+ II->getUnwindDest()->getTerminator()->setMetadata(
+ LLVMContext::MD_wave_profile_block, nullptr);
+ }
+
// Determine if we are dealing with a call in an EHPad which does not unwind
// to caller.
bool EHPadForCallUnwindsLocally = false;
@@ -2847,6 +2859,12 @@ void llvm::InlineFunctionImpl(CallBase &CB, InlineFunctionInfo &IFI,
// Remember the first block that is newly cloned over.
FirstNewBlock = LastBlock; ++FirstNewBlock;
+ // Callee counts aggregate all its invocations, not this callsite. Imported
+ // blocks remain unmeasured in the caller.
+ for (BasicBlock &BB : make_range(FirstNewBlock, Caller->end()))
+ BB.getTerminator()->setMetadata(LLVMContext::MD_wave_profile_block,
+ nullptr);
+
// Insert retainRV/clainRV runtime calls.
objcarc::ARCInstKind RVCallKind = objcarc::getAttachedARCFunctionKind(&CB);
if (RVCallKind != objcarc::ARCInstKind::None)
@@ -3403,6 +3421,13 @@ void llvm::InlineFunctionImpl(CallBase &CB, InlineFunctionInfo &IFI,
// We want to clone the entire callee function into the hole between the
// "starter" and "ender" blocks. How we accomplish this depends on whether
// this is an invoke instruction or a call instruction.
+ // Splitting the call block moves its terminator to the continuation, whose
+ // wave population is not the original block-entry event. Drop that
+ // association and let the reader conservatively invalidate its changed
+ // neighborhood.
+ OrigBB->getTerminator()->setMetadata(LLVMContext::MD_wave_profile_block,
+ nullptr);
+
BasicBlock *AfterCallBB;
UncondBrInst *CreatedBranchToNormalDest = nullptr;
if (InvokeInst *II = dyn_cast<InvokeInst>(&CB)) {
diff --git a/llvm/lib/Transforms/Utils/Local.cpp b/llvm/lib/Transforms/Utils/Local.cpp
index 7bdce3e64a55f9..6417a358823786 100644
--- a/llvm/lib/Transforms/Utils/Local.cpp
+++ b/llvm/lib/Transforms/Utils/Local.cpp
@@ -1149,7 +1149,8 @@ static void redirectValuesFromPredecessorsToPhi(BasicBlock *BB,
}
bool llvm::TryToSimplifyUncondBranchFromEmptyBlock(BasicBlock *BB,
- DomTreeUpdater *DTU) {
+ DomTreeUpdater *DTU,
+ BlockWaveCountUpdater *WPU) {
assert(BB != &BB->getParent()->getEntryBlock() &&
"TryToSimplifyUncondBranchFromEmptyBlock called on entry block!");
@@ -1280,6 +1281,19 @@ bool llvm::TryToSimplifyUncondBranchFromEmptyBlock(BasicBlock *BB,
if (PredTI->hasNonDebugLocLoopMetadata())
return false;
+ // Removing an unconditional block on a single predecessor's edge does not
+ // change the surrounding wave events. Multi-predecessor reconvergence is
+ // not covered by this transfer.
+ std::optional<BlockWaveCountPreserver> WaveProfile;
+ if (BBKillable && BB->getSinglePredecessor()) {
+ if (WPU)
+ WPU->removeBlock(*BB);
+ else {
+ WaveProfile.emplace(*BB->getParent());
+ WaveProfile->forget(*BB);
+ }
+ }
+
if (BBKillable)
LLVM_DEBUG(dbgs() << "Killing Trivial BB: \n" << *BB);
else if (BBPhisMergeable)
@@ -1381,6 +1395,9 @@ bool llvm::TryToSimplifyUncondBranchFromEmptyBlock(BasicBlock *BB,
if (BBKillable)
DeleteDeadBlock(BB, DTU);
+ if (WaveProfile)
+ WaveProfile->restore();
+
return true;
}
diff --git a/llvm/lib/Transforms/Utils/LoopRotationUtils.cpp b/llvm/lib/Transforms/Utils/LoopRotationUtils.cpp
index 5a6462d85a39a5..9c4aac2398374a 100644
--- a/llvm/lib/Transforms/Utils/LoopRotationUtils.cpp
+++ b/llvm/lib/Transforms/Utils/LoopRotationUtils.cpp
@@ -11,6 +11,7 @@
//===----------------------------------------------------------------------===//
#include "llvm/Transforms/Utils/LoopRotationUtils.h"
+#include "llvm/ADT/SmallPtrSet.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/Analysis/AssumptionCache.h"
#include "llvm/Analysis/CodeMetrics.h"
@@ -456,6 +457,12 @@ bool LoopRotate::rotateLoop(Loop *L, bool SimplifiedLatch) {
assert(L->contains(NewHeader) && !L->contains(Exit) &&
"Unable to determine loop header and exit blocks");
+ Function &F = *OrigHeader->getParent();
+ BlockWaveCountPreserver WaveProfile(F);
+ bool HasWaveProfile = WaveProfile.hasProfile();
+ if (HasWaveProfile)
+ WaveProfile.invalidate(*OrigHeader);
+
// This code assumes that the new header has exactly one predecessor.
// Remove any single-entry PHI nodes in it.
assert(NewHeader->getSinglePredecessor() &&
@@ -813,6 +820,14 @@ bool LoopRotate::rotateLoop(Loop *L, bool SimplifiedLatch) {
if (MSSAU && VerifyMemorySSA)
MSSAU->getMemorySSA()->verifyMemorySSA();
+ if (HasWaveProfile) {
+ // The preheader clone inherits the old header's metadata. Restore the
+ // rotated events before the merge performs its own profile transfer.
+ OrigPreheader->getTerminator()->setMetadata(
+ LLVMContext::MD_wave_profile_block, nullptr);
+ WaveProfile.restore();
+ }
+
// Now that the CFG and DomTree are in a consistent state again, try to merge
// the OrigHeader block into OrigLatch. This will succeed if they are
// connected by an unconditional branch. This is just a cleanup so the
diff --git a/llvm/lib/Transforms/Utils/LoopSimplify.cpp b/llvm/lib/Transforms/Utils/LoopSimplify.cpp
index 1a73dba362554e..61dca1b24f05f6 100644
--- a/llvm/lib/Transforms/Utils/LoopSimplify.cpp
+++ b/llvm/lib/Transforms/Utils/LoopSimplify.cpp
@@ -61,6 +61,7 @@
#include "llvm/IR/Instructions.h"
#include "llvm/IR/LLVMContext.h"
#include "llvm/IR/Module.h"
+#include "llvm/IR/ProfDataUtils.h"
#include "llvm/InitializePasses.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/raw_ostream.h"
@@ -114,7 +115,8 @@ static void placeSplitBlockCarefully(BasicBlock *NewBB,
///
BasicBlock *llvm::InsertPreheaderForLoop(Loop *L, DominatorTree *DT,
LoopInfo *LI, MemorySSAUpdater *MSSAU,
- bool PreserveLCSSA) {
+ bool PreserveLCSSA,
+ BlockWaveCountUpdater *WPU) {
BasicBlock *Header = L->getHeader();
// Compute the set of predecessors of the loop that are not in the loop.
@@ -135,7 +137,7 @@ BasicBlock *llvm::InsertPreheaderForLoop(Loop *L, DominatorTree *DT,
// Split out the loop pre-header.
BasicBlock *PreheaderBB;
PreheaderBB = SplitBlockPredecessors(Header, OutsideBlocks, ".preheader", DT,
- LI, MSSAU, PreserveLCSSA);
+ LI, MSSAU, PreserveLCSSA, WPU);
if (!PreheaderBB)
return nullptr;
@@ -212,7 +214,8 @@ static PHINode *findPHIToPartitionLoops(Loop *L, DominatorTree *DT,
static Loop *separateNestedLoop(Loop *L, BasicBlock *Preheader,
DominatorTree *DT, LoopInfo *LI,
ScalarEvolution *SE, bool PreserveLCSSA,
- AssumptionCache *AC, MemorySSAUpdater *MSSAU) {
+ AssumptionCache *AC, MemorySSAUpdater *MSSAU,
+ BlockWaveCountUpdater *WPU) {
// Don't try to separate loops without a preheader.
if (!Preheader)
return nullptr;
@@ -267,7 +270,7 @@ static Loop *separateNestedLoop(Loop *L, BasicBlock *Preheader,
SE->forgetLoop(L);
BasicBlock *NewBB = SplitBlockPredecessors(Header, OuterLoopPreds, ".outer",
- DT, LI, MSSAU, PreserveLCSSA);
+ DT, LI, MSSAU, PreserveLCSSA, WPU);
// Make sure that NewBB is put someplace intelligent, which doesn't mess up
// code layout too horribly.
@@ -325,7 +328,7 @@ static Loop *separateNestedLoop(Loop *L, BasicBlock *Preheader,
// Split edges to exit blocks from the inner loop, if they emerged in the
// process of separating the outer one.
- formDedicatedExitBlocks(L, DT, LI, MSSAU, PreserveLCSSA);
+ formDedicatedExitBlocks(L, DT, LI, MSSAU, PreserveLCSSA, WPU);
if (PreserveLCSSA) {
// Fix LCSSA form for L. Some values, which previously were only used inside
@@ -472,7 +475,8 @@ static BasicBlock *insertUniqueBackedgeBlock(Loop *L, BasicBlock *Preheader,
static bool simplifyOneLoop(Loop *L, SmallVectorImpl<Loop *> &Worklist,
DominatorTree *DT, LoopInfo *LI,
ScalarEvolution *SE, AssumptionCache *AC,
- MemorySSAUpdater *MSSAU, bool PreserveLCSSA) {
+ MemorySSAUpdater *MSSAU, bool PreserveLCSSA,
+ BlockWaveCountUpdater *WPU) {
bool Changed = false;
if (MSSAU && VerifyMemorySSA)
MSSAU->getMemorySSA()->verifyMemorySSA();
@@ -532,7 +536,7 @@ static bool simplifyOneLoop(Loop *L, SmallVectorImpl<Loop *> &Worklist,
// Does the loop already have a preheader? If so, don't insert one.
BasicBlock *Preheader = L->getLoopPreheader();
if (!Preheader) {
- Preheader = InsertPreheaderForLoop(L, DT, LI, MSSAU, PreserveLCSSA);
+ Preheader = InsertPreheaderForLoop(L, DT, LI, MSSAU, PreserveLCSSA, WPU);
if (Preheader)
Changed = true;
}
@@ -541,7 +545,7 @@ static bool simplifyOneLoop(Loop *L, SmallVectorImpl<Loop *> &Worklist,
// predecessors that are inside of the loop. This check guarantees that the
// loop preheader/header will dominate the exit blocks. If the exit block has
// predecessors from outside of the loop, split the edge now.
- if (formDedicatedExitBlocks(L, DT, LI, MSSAU, PreserveLCSSA))
+ if (formDedicatedExitBlocks(L, DT, LI, MSSAU, PreserveLCSSA, WPU))
Changed = true;
if (MSSAU && VerifyMemorySSA)
@@ -556,7 +560,7 @@ static bool simplifyOneLoop(Loop *L, SmallVectorImpl<Loop *> &Worklist,
// common backedge instead.
if (L->getNumBackEdges() < 8) {
if (Loop *OuterL = separateNestedLoop(L, Preheader, DT, LI, SE,
- PreserveLCSSA, AC, MSSAU)) {
+ PreserveLCSSA, AC, MSSAU, WPU)) {
++NumNested;
// Enqueue the outer loop as it should be processed next in our
// depth-first nest walk.
@@ -691,7 +695,8 @@ static bool simplifyOneLoop(Loop *L, SmallVectorImpl<Loop *> &Worklist,
bool llvm::simplifyLoop(Loop *L, DominatorTree *DT, LoopInfo *LI,
ScalarEvolution *SE, AssumptionCache *AC,
- MemorySSAUpdater *MSSAU, bool PreserveLCSSA) {
+ MemorySSAUpdater *MSSAU, bool PreserveLCSSA,
+ BlockWaveCountUpdater *WPU) {
bool Changed = false;
#ifndef NDEBUG
@@ -705,6 +710,11 @@ bool llvm::simplifyLoop(Loop *L, DominatorTree *DT, LoopInfo *LI,
}
#endif
+ BlockWaveCountUpdater LocalWPU(*L->getHeader()->getParent());
+ bool OwnWaveUpdates = !WPU;
+ if (!WPU)
+ WPU = &LocalWPU;
+
// Worklist maintains our depth-first queue of loops in this nest to process.
SmallVector<Loop *, 4> Worklist;
Worklist.push_back(L);
@@ -719,7 +729,10 @@ bool llvm::simplifyLoop(Loop *L, DominatorTree *DT, LoopInfo *LI,
while (!Worklist.empty())
Changed |= simplifyOneLoop(Worklist.pop_back_val(), Worklist, DT, LI, SE,
- AC, MSSAU, PreserveLCSSA);
+ AC, MSSAU, PreserveLCSSA, WPU);
+
+ if (OwnWaveUpdates)
+ WPU->apply();
// Changing exit conditions for blocks may affect exit counts of this loop and
// any of its parents, so we must invalidate the entire subtree if we've made
@@ -800,9 +813,13 @@ bool LoopSimplify::runOnFunction(Function &F) {
bool PreserveLCSSA = mustPreserveAnalysisID(LCSSAID);
+ BlockWaveCountUpdater WPU(F);
+
// Simplify each loop nest in the function.
for (auto *L : *LI)
- Changed |= simplifyLoop(L, DT, LI, SE, AC, MSSAU.get(), PreserveLCSSA);
+ Changed |=
+ simplifyLoop(L, DT, LI, SE, AC, MSSAU.get(), PreserveLCSSA, &WPU);
+ WPU.apply();
#ifndef NDEBUG
if (PreserveLCSSA) {
@@ -828,12 +845,14 @@ PreservedAnalyses LoopSimplifyPass::run(Function &F,
MSSAU = std::make_unique<MemorySSAUpdater>(MSSA);
}
+ BlockWaveCountUpdater WPU(F);
// Note that we don't preserve LCSSA in the new PM, if you need it run LCSSA
// after simplifying the loops. MemorySSA is preserved if it exists.
for (auto *L : *LI)
- Changed |=
- simplifyLoop(L, DT, LI, SE, AC, MSSAU.get(), /*PreserveLCSSA*/ false);
+ Changed |= simplifyLoop(L, DT, LI, SE, AC, MSSAU.get(),
+ /*PreserveLCSSA*/ false, &WPU);
+ WPU.apply();
if (!Changed)
return PreservedAnalyses::all();
diff --git a/llvm/lib/Transforms/Utils/LoopUnroll.cpp b/llvm/lib/Transforms/Utils/LoopUnroll.cpp
index db44f074359ac6..f210bdc725dec7 100644
--- a/llvm/lib/Transforms/Utils/LoopUnroll.cpp
+++ b/llvm/lib/Transforms/Utils/LoopUnroll.cpp
@@ -48,6 +48,7 @@
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/Metadata.h"
#include "llvm/IR/PatternMatch.h"
+#include "llvm/IR/ProfDataUtils.h"
#include "llvm/IR/Use.h"
#include "llvm/IR/User.h"
#include "llvm/IR/ValueHandle.h"
@@ -68,6 +69,7 @@
#include <assert.h>
#include <cmath>
#include <numeric>
+#include <optional>
#include <vector>
namespace llvm {
@@ -1054,6 +1056,18 @@ llvm::UnrollLoop(Loop *L, UnrollLoopOptions ULO, LoopInfo *LI,
}
}
+ // Fixed-trip full unrolling preserves surrounding execution events. The
+ // loop bodies and exit can inherit one another's terminators when merged;
+ // discard those identities rather than reusing their aggregate wave counts.
+ std::optional<BlockWaveCountPreserver> WaveProfile;
+ if (CompletelyUnroll && ExitingBlocks.size() == 1 && ExitBlocks.size() == 1 &&
+ SE->getSmallConstantTripCount(L) == ULO.Count) {
+ WaveProfile.emplace(*Header->getParent());
+ for (BasicBlock *BB : L->getBlocks())
+ WaveProfile->forget(*BB);
+ WaveProfile->forget(*ExitBlocks.front());
+ }
+
using namespace ore;
// Determine whether this loop originated from the vectorizer so we can
@@ -1561,6 +1575,9 @@ llvm::UnrollLoop(Loop *L, UnrollLoopOptions ULO, LoopInfo *LI,
CondLatches.reserve(Latches.size());
CondLatchNexts.reserve(Latches.size());
+ if (WaveProfile)
+ WaveProfile->restore();
+
// Merge adjacent basic blocks, if possible.
for (auto [I, Latch] : enumerate(Latches)) {
++IterCounts.back();
diff --git a/llvm/lib/Transforms/Utils/LoopUtils.cpp b/llvm/lib/Transforms/Utils/LoopUtils.cpp
index 2968f4600fd9a7..d24a6f2de737fa 100644
--- a/llvm/lib/Transforms/Utils/LoopUtils.cpp
+++ b/llvm/lib/Transforms/Utils/LoopUtils.cpp
@@ -58,9 +58,13 @@ static const char *LLVMLoopDisableNonforced = "llvm.loop.disable_nonforced";
static const char *LLVMLoopDisableLICM = "llvm.licm.disable";
bool llvm::formDedicatedExitBlocks(Loop *L, DominatorTree *DT, LoopInfo *LI,
- MemorySSAUpdater *MSSAU,
- bool PreserveLCSSA) {
+ MemorySSAUpdater *MSSAU, bool PreserveLCSSA,
+ BlockWaveCountUpdater *WPU) {
bool Changed = false;
+ BlockWaveCountUpdater LocalWPU(*L->getHeader()->getParent());
+ bool OwnWaveUpdates = !WPU;
+ if (!WPU)
+ WPU = &LocalWPU;
// We re-use a vector for the in-loop predecesosrs.
SmallVector<BasicBlock *, 4> InLoopPredecessors;
@@ -91,7 +95,7 @@ bool llvm::formDedicatedExitBlocks(Loop *L, DominatorTree *DT, LoopInfo *LI,
return false;
auto *NewExitBB = SplitBlockPredecessors(
- BB, InLoopPredecessors, ".loopexit", DT, LI, MSSAU, PreserveLCSSA);
+ BB, InLoopPredecessors, ".loopexit", DT, LI, MSSAU, PreserveLCSSA, WPU);
if (!NewExitBB)
LLVM_DEBUG(
@@ -119,6 +123,8 @@ bool llvm::formDedicatedExitBlocks(Loop *L, DominatorTree *DT, LoopInfo *LI,
Changed |= RewriteExit(SuccBB);
}
+ if (OwnWaveUpdates)
+ WPU->apply();
return Changed;
}
diff --git a/llvm/lib/Transforms/Utils/LowerSwitch.cpp b/llvm/lib/Transforms/Utils/LowerSwitch.cpp
index 9313f313cb02d5..fc3afca99973cd 100644
--- a/llvm/lib/Transforms/Utils/LowerSwitch.cpp
+++ b/llvm/lib/Transforms/Utils/LowerSwitch.cpp
@@ -28,6 +28,7 @@
#include "llvm/IR/InstrTypes.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/PassManager.h"
+#include "llvm/IR/ProfDataUtils.h"
#include "llvm/IR/Value.h"
#include "llvm/InitializePasses.h"
#include "llvm/Pass.h"
@@ -147,13 +148,101 @@ void FixPhis(BasicBlock *SuccBB, BasicBlock *OrigBB, BasicBlock *NewBB,
}
}
+// A default weight describes all values outside the explicit cases. Preserve
+// a generated branch's weights only when that mass has one possible destination
+// (including bypassing the branch); splitting it would require value profiling.
+class SwitchProfile {
+ SmallVector<APInt> Values;
+ SmallVector<uint64_t> PrefixWeights;
+ uint32_t DefaultWeight = 0;
+ APInt Lower, Upper;
+ bool IsExpected;
+
+ std::pair<unsigned, unsigned> getRange(const APInt &Lo,
+ const APInt &Hi) const {
+ auto Less = [](const APInt &A, const APInt &B) { return A.slt(B); };
+ auto Begin = llvm::lower_bound(Values, Lo, Less);
+ auto End = llvm::upper_bound(Values, Hi, Less);
+ return {Begin - Values.begin(), End - Values.begin()};
+ }
+
+ bool hasDefaultValues(const APInt &Lo, const APInt &Hi) const {
+ APInt Size =
+ Hi.sext(Hi.getBitWidth() + 1) - Lo.sext(Lo.getBitWidth() + 1) + 1;
+ auto [Begin, End] = getRange(Lo, Hi);
+ return Size.ugt(End - Begin);
+ }
+
+ uint64_t getWeight(const APInt &Lo, const APInt &Hi) const {
+ auto [Begin, End] = getRange(Lo, Hi);
+ return PrefixWeights[End] - PrefixWeights[Begin];
+ }
+
+public:
+ explicit SwitchProfile(const SwitchInst &SI)
+ : Lower(APInt::getSignedMinValue(
+ SI.getCondition()->getType()->getIntegerBitWidth())),
+ Upper(APInt::getSignedMaxValue(Lower.getBitWidth())),
+ IsExpected(hasBranchWeightOrigin(SI)) {
+ SmallVector<uint32_t> Weights;
+ if (!hasValidBranchWeightMD(SI) || !extractBranchWeights(SI, Weights))
+ return;
+ DefaultWeight = Weights[0];
+ SmallVector<std::pair<APInt, uint32_t>> Cases;
+ for (auto [Index, Case] : enumerate(SI.cases()))
+ Cases.emplace_back(Case.getCaseValue()->getValue(), Weights[Index + 1]);
+ llvm::sort(Cases, [](const auto &A, const auto &B) {
+ return A.first.slt(B.first);
+ });
+ PrefixWeights.push_back(0);
+ for (const auto &[Value, Weight] : Cases) {
+ Values.push_back(Value);
+ PrefixWeights.push_back(PrefixWeights.back() + Weight);
+ }
+ }
+
+ void setBounds(const APInt &Lo, const APInt &Hi, bool DefaultUnreachable) {
+ Lower = Lo;
+ Upper = Hi;
+ if (DefaultUnreachable)
+ DefaultWeight = 0;
+ }
+
+ void setMetadata(CondBrInst &Br, const APInt &Lo, const APInt &Hi,
+ const APInt &TrueLo, const APInt &TrueHi) const {
+ if (PrefixWeights.empty())
+ return;
+ uint64_t TrueWeight = getWeight(TrueLo, TrueHi);
+ uint64_t FalseWeight = getWeight(Lo, Hi) - TrueWeight;
+ if (DefaultWeight) {
+ bool TrueDefault = hasDefaultValues(TrueLo, TrueHi);
+ bool FalseDefault =
+ (Lo.slt(TrueLo) && hasDefaultValues(Lo, TrueLo - 1)) ||
+ (TrueHi.slt(Hi) && hasDefaultValues(TrueHi + 1, Hi));
+ bool OutsideDefault =
+ (Lower.slt(Lo) && hasDefaultValues(Lower, Lo - 1)) ||
+ (Hi.slt(Upper) && hasDefaultValues(Hi + 1, Upper));
+ if (unsigned(TrueDefault) + unsigned(FalseDefault) +
+ unsigned(OutsideDefault) >
+ 1)
+ return;
+ if (TrueDefault)
+ TrueWeight += DefaultWeight;
+ else if (FalseDefault)
+ FalseWeight += DefaultWeight;
+ }
+ setFittedBranchWeights(Br, {TrueWeight, FalseWeight}, IsExpected,
+ /*ElideAllZero=*/true);
+ }
+};
+
/// Create a new leaf block for the binary lookup tree. It checks if the
/// switch's value == the case's value. If not, then it jumps to the default
/// branch. At this point in the tree, the value can't be another valid case
/// value, so the jump to the "default" branch is warranted.
BasicBlock *NewLeafBlock(CaseRange &Leaf, Value *Val, ConstantInt *LowerBound,
ConstantInt *UpperBound, BasicBlock *OrigBlock,
- BasicBlock *Default) {
+ BasicBlock *Default, const SwitchProfile &Profile) {
Function *F = OrigBlock->getParent();
BasicBlock *NewLeaf = BasicBlock::Create(Val->getContext(), "LeafBlock");
F->insert(++OrigBlock->getIterator(), NewLeaf);
@@ -191,7 +280,9 @@ BasicBlock *NewLeafBlock(CaseRange &Leaf, Value *Val, ConstantInt *LowerBound,
// Make the conditional branch...
BasicBlock *Succ = Leaf.BB;
- CondBrInst::Create(Comp, Succ, Default, NewLeaf);
+ CondBrInst *Br = CondBrInst::Create(Comp, Succ, Default, NewLeaf);
+ Profile.setMetadata(*Br, LowerBound->getValue(), UpperBound->getValue(),
+ Leaf.Low->getValue(), Leaf.High->getValue());
// Update the PHI incoming value/block for the default.
for (auto &I : Default->phis()) {
@@ -227,7 +318,8 @@ BasicBlock *SwitchConvert(CaseItr Begin, CaseItr End, ConstantInt *LowerBound,
ConstantInt *UpperBound, Value *Val,
BasicBlock *Predecessor, BasicBlock *OrigBlock,
BasicBlock *Default,
- const std::vector<IntRange> &UnreachableRanges) {
+ const std::vector<IntRange> &UnreachableRanges,
+ const SwitchProfile &Profile) {
assert(LowerBound && UpperBound && "Bounds must be initialized");
unsigned Size = End - Begin;
@@ -241,8 +333,8 @@ BasicBlock *SwitchConvert(CaseItr Begin, CaseItr End, ConstantInt *LowerBound,
FixPhis(Begin->BB, OrigBlock, Predecessor, NumMergedCases);
return Begin->BB;
}
- return NewLeafBlock(*Begin, Val, LowerBound, UpperBound, OrigBlock,
- Default);
+ return NewLeafBlock(*Begin, Val, LowerBound, UpperBound, OrigBlock, Default,
+ Profile);
}
unsigned Mid = Size / 2;
@@ -289,15 +381,17 @@ BasicBlock *SwitchConvert(CaseItr Begin, CaseItr End, ConstantInt *LowerBound,
BasicBlock *LBranch =
SwitchConvert(LHS.begin(), LHS.end(), LowerBound, NewUpperBound, Val,
- NewNode, OrigBlock, Default, UnreachableRanges);
+ NewNode, OrigBlock, Default, UnreachableRanges, Profile);
BasicBlock *RBranch =
SwitchConvert(RHS.begin(), RHS.end(), NewLowerBound, UpperBound, Val,
- NewNode, OrigBlock, Default, UnreachableRanges);
+ NewNode, OrigBlock, Default, UnreachableRanges, Profile);
F->insert(++OrigBlock->getIterator(), NewNode);
Comp->insertInto(NewNode, NewNode->end());
- CondBrInst::Create(Comp, LBranch, RBranch, NewNode);
+ CondBrInst *Br = CondBrInst::Create(Comp, LBranch, RBranch, NewNode);
+ Profile.setMetadata(*Br, LowerBound->getValue(), UpperBound->getValue(),
+ LowerBound->getValue(), Pivot.Low->getValue() - 1);
return NewNode;
}
@@ -333,7 +427,6 @@ unsigned Clusterify(CaseVector &Cases, SwitchInst *SI) {
"Cases should be strictly ascending");
if ((nextValue == currentValue + 1) && (currentBB == nextBB)) {
I->High = J->High;
- // FIXME: Combine branch weights.
} else if (++I != J) {
*I = *J;
}
@@ -352,6 +445,7 @@ void ProcessSwitchInst(SwitchInst *SI,
BasicBlock *OrigBlock = SI->getParent();
Function *F = OrigBlock->getParent();
Value *Val = SI->getCondition(); // The value we are switching on...
+ SwitchProfile Profile(*SI);
BasicBlock *Default = SI->getDefaultDest();
// Don't handle unreachable blocks. If there are successors with phis, this
@@ -508,9 +602,11 @@ void ProcessSwitchInst(SwitchInst *SI,
Val = SI->getCondition();
}
+ Profile.setBounds(LowerBound->getValue(), UpperBound->getValue(),
+ DefaultIsUnreachableFromSwitch);
BasicBlock *SwitchBlock =
SwitchConvert(Cases.begin(), Cases.end(), LowerBound, UpperBound, Val,
- OrigBlock, OrigBlock, Default, UnreachableRanges);
+ OrigBlock, OrigBlock, Default, UnreachableRanges, Profile);
// We have added incoming values for newly-created predecessors in
// NewLeafBlock(). The only meaningful work we offload to FixPhis() is to
@@ -534,6 +630,7 @@ void ProcessSwitchInst(SwitchInst *SI,
bool LowerSwitch(Function &F, LazyValueInfo *LVI, AssumptionCache *AC) {
bool Changed = false;
+ BlockWaveCountPreserver WaveProfile(F);
SmallPtrSet<BasicBlock *, 8> DeleteList;
// We use make_early_inc_range here so that we don't traverse new blocks.
@@ -545,7 +642,11 @@ bool LowerSwitch(Function &F, LazyValueInfo *LVI, AssumptionCache *AC) {
if (SwitchInst *SI = dyn_cast<SwitchInst>(Cur.getTerminator())) {
Changed = true;
+ MDNode *Uniformity =
+ SI->getMetadata(LLVMContext::MD_block_uniformity_profile);
ProcessSwitchInst(SI, DeleteList, AC, LVI);
+ Cur.getTerminator()->setMetadata(LLVMContext::MD_block_uniformity_profile,
+ Uniformity);
}
}
@@ -554,6 +655,8 @@ bool LowerSwitch(Function &F, LazyValueInfo *LVI, AssumptionCache *AC) {
DeleteDeadBlock(BB);
}
+ if (Changed)
+ WaveProfile.restore();
return Changed;
}
diff --git a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
index 19f3ee8cb5137c..1ab51d9c32cb61 100644
--- a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
+++ b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
@@ -324,13 +324,16 @@ class SimplifyCFGOpt {
public:
SimplifyCFGOpt(const TargetTransformInfo &TTI, DomTreeUpdater *DTU,
const DataLayout &DL, ArrayRef<WeakVH> LoopHeaders,
- const SimplifyCFGOptions &Opts)
- : TTI(TTI), DTU(DTU), DL(DL), LoopHeaders(LoopHeaders), Options(Opts) {
+ const SimplifyCFGOptions &Opts, BlockWaveCountUpdater *WPU)
+ : TTI(TTI), DTU(DTU), DL(DL), LoopHeaders(LoopHeaders), Options(Opts),
+ WPU(WPU) {
assert((!DTU || !DTU->hasPostDomTree()) &&
"SimplifyCFG is not yet capable of maintaining validity of a "
"PostDomTree, so don't ask for it.");
}
+ BlockWaveCountUpdater *WPU;
+
bool simplifyOnce(BasicBlock *BB);
bool run(BasicBlock *BB);
@@ -3662,7 +3665,7 @@ static bool reachesUncontrolledConvergentCallBeforeBlock(BasicBlock *From,
/// from the predecessor to their ultimate destination.
static std::optional<bool> foldCondBranchOnValueKnownInPredecessorImpl(
CondBrInst *BI, const TargetTransformInfo &TTI, DomTreeUpdater *DTU,
- AssumptionCache *AC, const DataLayout &DL) {
+ AssumptionCache *AC, const DataLayout &DL, BlockWaveCountUpdater *WPU) {
SmallMapVector<ConstantInt *, SmallSetVector<BasicBlock *, 2>, 2> KnownValues;
BasicBlock *BB = BI->getParent();
Value *Cond = BI->getCondition();
@@ -3750,7 +3753,8 @@ static std::optional<bool> foldCondBranchOnValueKnownInPredecessorImpl(
// Split the predecessors we are threading into a new edge block. We'll
// clone the instructions into this block, and then redirect it to RealDest.
- BasicBlock *EdgeBB = SplitBlockPredecessors(BB, PredBBs, ".critedge", DTU);
+ BasicBlock *EdgeBB = SplitBlockPredecessors(BB, PredBBs, ".critedge", DTU,
+ nullptr, nullptr, false, WPU);
if (!EdgeBB)
continue;
@@ -3841,7 +3845,8 @@ static std::optional<bool> foldCondBranchOnValueKnownInPredecessorImpl(
// it back into the predecessor if possible. This not only avoids
// unnecessary SimplifyCFG iterations, but also makes sure that we don't
// bypass the check for trivial cycles above.
- MergeBlockIntoPredecessor(EdgeBB, DTU);
+ MergeBlockIntoPredecessor(EdgeBB, DTU, nullptr, nullptr, nullptr, false,
+ nullptr, WPU);
// Signal repeat, simplifying any other constants.
return std::nullopt;
@@ -3862,7 +3867,7 @@ bool SimplifyCFGOpt::foldCondBranchOnValueKnownInPredecessor(CondBrInst *BI) {
do {
// Note that None means "we changed things, but recurse further."
Result = foldCondBranchOnValueKnownInPredecessorImpl(BI, TTI, DTU,
- Options.AC, DL);
+ Options.AC, DL, WPU);
EverChanged |= Result == std::nullopt || *Result;
} while (Result == std::nullopt);
return EverChanged;
@@ -8886,7 +8891,8 @@ bool SimplifyCFGOpt::simplifyUncondBranch(UncondBrInst *BI,
(is_contained(LoopHeaders, BB) || is_contained(LoopHeaders, Succ)));
BasicBlock::iterator I = BB->getFirstNonPHIOrPseudoProbe();
if (I->isTerminator() && BB != &BB->getParent()->getEntryBlock() &&
- !NeedCanonicalLoop && TryToSimplifyUncondBranchFromEmptyBlock(BB, DTU))
+ !NeedCanonicalLoop &&
+ TryToSimplifyUncondBranchFromEmptyBlock(BB, DTU, WPU))
return true;
// If the only instruction in the block is a seteq/setne comparison against a
@@ -9401,7 +9407,8 @@ bool SimplifyCFGOpt::simplifyOnce(BasicBlock *BB) {
// Merge basic blocks into their predecessor if there is only one distinct
// pred, and if there is only one distinct successor of the predecessor, and
// if there are no PHI nodes.
- if (MergeBlockIntoPredecessor(BB, DTU))
+ if (MergeBlockIntoPredecessor(BB, DTU, nullptr, nullptr, nullptr, false,
+ nullptr, WPU))
return true;
if (SinkCommon && Options.SinkCommonInsts) {
@@ -9477,8 +9484,9 @@ bool SimplifyCFGOpt::run(BasicBlock *BB) {
bool llvm::simplifyCFG(BasicBlock *BB, const TargetTransformInfo &TTI,
DomTreeUpdater *DTU, const SimplifyCFGOptions &Options,
- ArrayRef<WeakVH> LoopHeaders) {
- return SimplifyCFGOpt(TTI, DTU, BB->getDataLayout(), LoopHeaders,
- Options)
+ ArrayRef<WeakVH> LoopHeaders,
+ BlockWaveCountUpdater *WPU) {
+ return SimplifyCFGOpt(TTI, DTU, BB->getDataLayout(), LoopHeaders, Options,
+ WPU)
.run(BB);
}
diff --git a/llvm/test/CodeGen/AMDGPU/itofp.i128.bf.ll b/llvm/test/CodeGen/AMDGPU/itofp.i128.bf.ll
index 4d9a7fc63331c8..c75805339c93b9 100644
--- a/llvm/test/CodeGen/AMDGPU/itofp.i128.bf.ll
+++ b/llvm/test/CodeGen/AMDGPU/itofp.i128.bf.ll
@@ -52,7 +52,6 @@ define bfloat @sitofp_i128_to_bf16(i128 %x) {
; SDAG-NEXT: ; %bb.4: ; %LeafBlock
; SDAG-NEXT: v_cmp_ne_u32_e32 vcc, 26, v2
; SDAG-NEXT: s_and_saveexec_b64 s[12:13], vcc
-; SDAG-NEXT: s_cbranch_execz .LBB0_6
; SDAG-NEXT: ; %bb.5: ; %itofp-sw-default
; SDAG-NEXT: v_sub_u32_e32 v12, 0x66, v7
; SDAG-NEXT: v_sub_u32_e32 v10, 64, v12
@@ -91,7 +90,7 @@ define bfloat @sitofp_i128_to_bf16(i128 %x) {
; SDAG-NEXT: v_or_b32_e32 v8, v15, v0
; SDAG-NEXT: v_mov_b32_e32 v0, v8
; SDAG-NEXT: v_mov_b32_e32 v1, v9
-; SDAG-NEXT: .LBB0_6: ; %Flow1
+; SDAG-NEXT: ; %bb.6: ; %Flow1
; SDAG-NEXT: s_or_b64 exec, exec, s[12:13]
; SDAG-NEXT: .LBB0_7: ; %Flow2
; SDAG-NEXT: s_andn2_saveexec_b64 s[4:5], s[10:11]
@@ -191,7 +190,6 @@ define bfloat @sitofp_i128_to_bf16(i128 %x) {
; GISEL-NEXT: ; %bb.4: ; %LeafBlock
; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 26, v7
; GISEL-NEXT: s_and_saveexec_b64 s[12:13], vcc
-; GISEL-NEXT: s_cbranch_execz .LBB0_6
; GISEL-NEXT: ; %bb.5: ; %itofp-sw-default
; GISEL-NEXT: v_sub_u32_e32 v4, 0x66, v5
; GISEL-NEXT: v_sub_u32_e32 v11, 64, v4
@@ -234,7 +232,7 @@ define bfloat @sitofp_i128_to_bf16(i128 %x) {
; GISEL-NEXT: v_mov_b32_e32 v1, v4
; GISEL-NEXT: v_mov_b32_e32 v2, v5
; GISEL-NEXT: v_mov_b32_e32 v3, v6
-; GISEL-NEXT: .LBB0_6: ; %Flow1
+; GISEL-NEXT: ; %bb.6: ; %Flow1
; GISEL-NEXT: s_or_b64 exec, exec, s[12:13]
; GISEL-NEXT: .LBB0_7: ; %Flow2
; GISEL-NEXT: s_andn2_saveexec_b64 s[4:5], s[10:11]
@@ -330,7 +328,6 @@ define bfloat @uitofp_i128_to_bf16(i128 %x) {
; SDAG-NEXT: ; %bb.4: ; %LeafBlock
; SDAG-NEXT: v_cmp_ne_u32_e32 vcc, 26, v4
; SDAG-NEXT: s_and_saveexec_b64 s[12:13], vcc
-; SDAG-NEXT: s_cbranch_execz .LBB1_6
; SDAG-NEXT: ; %bb.5: ; %itofp-sw-default
; SDAG-NEXT: v_sub_u32_e32 v11, 0x66, v6
; SDAG-NEXT: v_sub_u32_e32 v9, 64, v11
@@ -369,7 +366,7 @@ define bfloat @uitofp_i128_to_bf16(i128 %x) {
; SDAG-NEXT: v_or_b32_e32 v7, v14, v0
; SDAG-NEXT: v_mov_b32_e32 v0, v7
; SDAG-NEXT: v_mov_b32_e32 v1, v8
-; SDAG-NEXT: .LBB1_6: ; %Flow1
+; SDAG-NEXT: ; %bb.6: ; %Flow1
; SDAG-NEXT: s_or_b64 exec, exec, s[12:13]
; SDAG-NEXT: .LBB1_7: ; %Flow2
; SDAG-NEXT: s_andn2_saveexec_b64 s[4:5], s[10:11]
@@ -459,7 +456,6 @@ define bfloat @uitofp_i128_to_bf16(i128 %x) {
; GISEL-NEXT: ; %bb.4: ; %LeafBlock
; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 26, v6
; GISEL-NEXT: s_and_saveexec_b64 s[12:13], vcc
-; GISEL-NEXT: s_cbranch_execz .LBB1_6
; GISEL-NEXT: ; %bb.5: ; %itofp-sw-default
; GISEL-NEXT: v_sub_u32_e32 v4, 0x66, v5
; GISEL-NEXT: v_sub_u32_e32 v10, 64, v4
@@ -502,7 +498,7 @@ define bfloat @uitofp_i128_to_bf16(i128 %x) {
; GISEL-NEXT: v_mov_b32_e32 v1, v4
; GISEL-NEXT: v_mov_b32_e32 v2, v5
; GISEL-NEXT: v_mov_b32_e32 v3, v6
-; GISEL-NEXT: .LBB1_6: ; %Flow1
+; GISEL-NEXT: ; %bb.6: ; %Flow1
; GISEL-NEXT: s_or_b64 exec, exec, s[12:13]
; GISEL-NEXT: .LBB1_7: ; %Flow2
; GISEL-NEXT: s_andn2_saveexec_b64 s[4:5], s[10:11]
diff --git a/llvm/test/CodeGen/AMDGPU/itofp.i128.ll b/llvm/test/CodeGen/AMDGPU/itofp.i128.ll
index 54f16e2a586ca4..73c0574ffc19d9 100644
--- a/llvm/test/CodeGen/AMDGPU/itofp.i128.ll
+++ b/llvm/test/CodeGen/AMDGPU/itofp.i128.ll
@@ -52,7 +52,6 @@ define float @sitofp_i128_to_f32(i128 %x) {
; SDAG-NEXT: ; %bb.4: ; %LeafBlock
; SDAG-NEXT: v_cmp_ne_u32_e32 vcc, 26, v2
; SDAG-NEXT: s_and_saveexec_b64 s[12:13], vcc
-; SDAG-NEXT: s_cbranch_execz .LBB0_6
; SDAG-NEXT: ; %bb.5: ; %itofp-sw-default
; SDAG-NEXT: v_sub_u32_e32 v12, 0x66, v7
; SDAG-NEXT: v_sub_u32_e32 v10, 64, v12
@@ -91,7 +90,7 @@ define float @sitofp_i128_to_f32(i128 %x) {
; SDAG-NEXT: v_or_b32_e32 v8, v15, v0
; SDAG-NEXT: v_mov_b32_e32 v0, v8
; SDAG-NEXT: v_mov_b32_e32 v1, v9
-; SDAG-NEXT: .LBB0_6: ; %Flow1
+; SDAG-NEXT: ; %bb.6: ; %Flow1
; SDAG-NEXT: s_or_b64 exec, exec, s[12:13]
; SDAG-NEXT: .LBB0_7: ; %Flow2
; SDAG-NEXT: s_andn2_saveexec_b64 s[4:5], s[10:11]
@@ -184,7 +183,6 @@ define float @sitofp_i128_to_f32(i128 %x) {
; GISEL-NEXT: ; %bb.4: ; %LeafBlock
; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 26, v7
; GISEL-NEXT: s_and_saveexec_b64 s[12:13], vcc
-; GISEL-NEXT: s_cbranch_execz .LBB0_6
; GISEL-NEXT: ; %bb.5: ; %itofp-sw-default
; GISEL-NEXT: v_sub_u32_e32 v4, 0x66, v5
; GISEL-NEXT: v_sub_u32_e32 v11, 64, v4
@@ -227,7 +225,7 @@ define float @sitofp_i128_to_f32(i128 %x) {
; GISEL-NEXT: v_mov_b32_e32 v1, v4
; GISEL-NEXT: v_mov_b32_e32 v2, v5
; GISEL-NEXT: v_mov_b32_e32 v3, v6
-; GISEL-NEXT: .LBB0_6: ; %Flow1
+; GISEL-NEXT: ; %bb.6: ; %Flow1
; GISEL-NEXT: s_or_b64 exec, exec, s[12:13]
; GISEL-NEXT: .LBB0_7: ; %Flow2
; GISEL-NEXT: s_andn2_saveexec_b64 s[4:5], s[10:11]
@@ -316,7 +314,6 @@ define float @uitofp_i128_to_f32(i128 %x) {
; SDAG-NEXT: ; %bb.4: ; %LeafBlock
; SDAG-NEXT: v_cmp_ne_u32_e32 vcc, 26, v4
; SDAG-NEXT: s_and_saveexec_b64 s[12:13], vcc
-; SDAG-NEXT: s_cbranch_execz .LBB1_6
; SDAG-NEXT: ; %bb.5: ; %itofp-sw-default
; SDAG-NEXT: v_sub_u32_e32 v11, 0x66, v6
; SDAG-NEXT: v_sub_u32_e32 v9, 64, v11
@@ -355,7 +352,7 @@ define float @uitofp_i128_to_f32(i128 %x) {
; SDAG-NEXT: v_or_b32_e32 v7, v14, v0
; SDAG-NEXT: v_mov_b32_e32 v0, v7
; SDAG-NEXT: v_mov_b32_e32 v1, v8
-; SDAG-NEXT: .LBB1_6: ; %Flow1
+; SDAG-NEXT: ; %bb.6: ; %Flow1
; SDAG-NEXT: s_or_b64 exec, exec, s[12:13]
; SDAG-NEXT: .LBB1_7: ; %Flow2
; SDAG-NEXT: s_andn2_saveexec_b64 s[4:5], s[10:11]
@@ -438,7 +435,6 @@ define float @uitofp_i128_to_f32(i128 %x) {
; GISEL-NEXT: ; %bb.4: ; %LeafBlock
; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 26, v6
; GISEL-NEXT: s_and_saveexec_b64 s[12:13], vcc
-; GISEL-NEXT: s_cbranch_execz .LBB1_6
; GISEL-NEXT: ; %bb.5: ; %itofp-sw-default
; GISEL-NEXT: v_sub_u32_e32 v4, 0x66, v5
; GISEL-NEXT: v_sub_u32_e32 v10, 64, v4
@@ -481,7 +477,7 @@ define float @uitofp_i128_to_f32(i128 %x) {
; GISEL-NEXT: v_mov_b32_e32 v1, v4
; GISEL-NEXT: v_mov_b32_e32 v2, v5
; GISEL-NEXT: v_mov_b32_e32 v3, v6
-; GISEL-NEXT: .LBB1_6: ; %Flow1
+; GISEL-NEXT: ; %bb.6: ; %Flow1
; GISEL-NEXT: s_or_b64 exec, exec, s[12:13]
; GISEL-NEXT: .LBB1_7: ; %Flow2
; GISEL-NEXT: s_andn2_saveexec_b64 s[4:5], s[10:11]
@@ -581,7 +577,6 @@ define double @sitofp_i128_to_f64(i128 %x) {
; SDAG-NEXT: ; %bb.4: ; %LeafBlock
; SDAG-NEXT: v_cmp_ne_u32_e32 vcc, 55, v2
; SDAG-NEXT: s_and_saveexec_b64 s[12:13], vcc
-; SDAG-NEXT: s_cbranch_execz .LBB2_6
; SDAG-NEXT: ; %bb.5: ; %itofp-sw-default
; SDAG-NEXT: v_sub_u32_e32 v12, 0x49, v9
; SDAG-NEXT: v_sub_u32_e32 v10, 64, v12
@@ -624,7 +619,7 @@ define double @sitofp_i128_to_f64(i128 %x) {
; SDAG-NEXT: v_mov_b32_e32 v5, v1
; SDAG-NEXT: v_mov_b32_e32 v4, v0
; SDAG-NEXT: v_mov_b32_e32 v7, v11
-; SDAG-NEXT: .LBB2_6: ; %Flow1
+; SDAG-NEXT: ; %bb.6: ; %Flow1
; SDAG-NEXT: s_or_b64 exec, exec, s[12:13]
; SDAG-NEXT: .LBB2_7: ; %Flow2
; SDAG-NEXT: s_andn2_saveexec_b64 s[4:5], s[10:11]
@@ -730,7 +725,6 @@ define double @sitofp_i128_to_f64(i128 %x) {
; GISEL-NEXT: ; %bb.4: ; %LeafBlock
; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 55, v7
; GISEL-NEXT: s_and_saveexec_b64 s[12:13], vcc
-; GISEL-NEXT: s_cbranch_execz .LBB2_6
; GISEL-NEXT: ; %bb.5: ; %itofp-sw-default
; GISEL-NEXT: v_sub_u32_e32 v14, 0x49, v9
; GISEL-NEXT: v_sub_u32_e32 v10, 64, v14
@@ -775,7 +769,7 @@ define double @sitofp_i128_to_f64(i128 %x) {
; GISEL-NEXT: v_mov_b32_e32 v3, v10
; GISEL-NEXT: v_mov_b32_e32 v4, v11
; GISEL-NEXT: v_mov_b32_e32 v5, v12
-; GISEL-NEXT: .LBB2_6: ; %Flow1
+; GISEL-NEXT: ; %bb.6: ; %Flow1
; GISEL-NEXT: s_or_b64 exec, exec, s[12:13]
; GISEL-NEXT: .LBB2_7: ; %Flow2
; GISEL-NEXT: s_andn2_saveexec_b64 s[4:5], s[10:11]
@@ -877,7 +871,6 @@ define double @uitofp_i128_to_f64(i128 %x) {
; SDAG-NEXT: ; %bb.4: ; %LeafBlock
; SDAG-NEXT: v_cmp_ne_u32_e32 vcc, 55, v6
; SDAG-NEXT: s_and_saveexec_b64 s[12:13], vcc
-; SDAG-NEXT: s_cbranch_execz .LBB3_6
; SDAG-NEXT: ; %bb.5: ; %itofp-sw-default
; SDAG-NEXT: v_sub_u32_e32 v11, 0x49, v8
; SDAG-NEXT: v_sub_u32_e32 v9, 64, v11
@@ -920,7 +913,7 @@ define double @uitofp_i128_to_f64(i128 %x) {
; SDAG-NEXT: v_mov_b32_e32 v0, v4
; SDAG-NEXT: v_mov_b32_e32 v1, v5
; SDAG-NEXT: v_mov_b32_e32 v3, v10
-; SDAG-NEXT: .LBB3_6: ; %Flow1
+; SDAG-NEXT: ; %bb.6: ; %Flow1
; SDAG-NEXT: s_or_b64 exec, exec, s[12:13]
; SDAG-NEXT: .LBB3_7: ; %Flow2
; SDAG-NEXT: s_andn2_saveexec_b64 s[4:5], s[10:11]
@@ -1013,7 +1006,6 @@ define double @uitofp_i128_to_f64(i128 %x) {
; GISEL-NEXT: ; %bb.4: ; %LeafBlock
; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 55, v6
; GISEL-NEXT: s_and_saveexec_b64 s[12:13], vcc
-; GISEL-NEXT: s_cbranch_execz .LBB3_6
; GISEL-NEXT: ; %bb.5: ; %itofp-sw-default
; GISEL-NEXT: v_sub_u32_e32 v13, 0x49, v8
; GISEL-NEXT: v_sub_u32_e32 v9, 64, v13
@@ -1059,7 +1051,7 @@ define double @uitofp_i128_to_f64(i128 %x) {
; GISEL-NEXT: v_mov_b32_e32 v1, v9
; GISEL-NEXT: v_mov_b32_e32 v2, v10
; GISEL-NEXT: v_mov_b32_e32 v3, v11
-; GISEL-NEXT: .LBB3_6: ; %Flow1
+; GISEL-NEXT: ; %bb.6: ; %Flow1
; GISEL-NEXT: s_or_b64 exec, exec, s[12:13]
; GISEL-NEXT: .LBB3_7: ; %Flow2
; GISEL-NEXT: s_andn2_saveexec_b64 s[4:5], s[10:11]
@@ -1174,7 +1166,6 @@ define half @sitofp_i128_to_f16(i128 %x) {
; SDAG-NEXT: ; %bb.4: ; %LeafBlock
; SDAG-NEXT: v_cmp_ne_u32_e32 vcc, 26, v2
; SDAG-NEXT: s_and_saveexec_b64 s[12:13], vcc
-; SDAG-NEXT: s_cbranch_execz .LBB4_6
; SDAG-NEXT: ; %bb.5: ; %itofp-sw-default
; SDAG-NEXT: v_sub_u32_e32 v12, 0x66, v7
; SDAG-NEXT: v_sub_u32_e32 v10, 64, v12
@@ -1213,7 +1204,7 @@ define half @sitofp_i128_to_f16(i128 %x) {
; SDAG-NEXT: v_or_b32_e32 v8, v15, v0
; SDAG-NEXT: v_mov_b32_e32 v0, v8
; SDAG-NEXT: v_mov_b32_e32 v1, v9
-; SDAG-NEXT: .LBB4_6: ; %Flow1
+; SDAG-NEXT: ; %bb.6: ; %Flow1
; SDAG-NEXT: s_or_b64 exec, exec, s[12:13]
; SDAG-NEXT: .LBB4_7: ; %Flow2
; SDAG-NEXT: s_andn2_saveexec_b64 s[4:5], s[10:11]
@@ -1307,7 +1298,6 @@ define half @sitofp_i128_to_f16(i128 %x) {
; GISEL-NEXT: ; %bb.4: ; %LeafBlock
; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 26, v7
; GISEL-NEXT: s_and_saveexec_b64 s[12:13], vcc
-; GISEL-NEXT: s_cbranch_execz .LBB4_6
; GISEL-NEXT: ; %bb.5: ; %itofp-sw-default
; GISEL-NEXT: v_sub_u32_e32 v4, 0x66, v5
; GISEL-NEXT: v_sub_u32_e32 v11, 64, v4
@@ -1350,7 +1340,7 @@ define half @sitofp_i128_to_f16(i128 %x) {
; GISEL-NEXT: v_mov_b32_e32 v1, v4
; GISEL-NEXT: v_mov_b32_e32 v2, v5
; GISEL-NEXT: v_mov_b32_e32 v3, v6
-; GISEL-NEXT: .LBB4_6: ; %Flow1
+; GISEL-NEXT: ; %bb.6: ; %Flow1
; GISEL-NEXT: s_or_b64 exec, exec, s[12:13]
; GISEL-NEXT: .LBB4_7: ; %Flow2
; GISEL-NEXT: s_andn2_saveexec_b64 s[4:5], s[10:11]
@@ -1440,7 +1430,6 @@ define half @uitofp_i128_to_f16(i128 %x) {
; SDAG-NEXT: ; %bb.4: ; %LeafBlock
; SDAG-NEXT: v_cmp_ne_u32_e32 vcc, 26, v4
; SDAG-NEXT: s_and_saveexec_b64 s[12:13], vcc
-; SDAG-NEXT: s_cbranch_execz .LBB5_6
; SDAG-NEXT: ; %bb.5: ; %itofp-sw-default
; SDAG-NEXT: v_sub_u32_e32 v11, 0x66, v6
; SDAG-NEXT: v_sub_u32_e32 v9, 64, v11
@@ -1479,7 +1468,7 @@ define half @uitofp_i128_to_f16(i128 %x) {
; SDAG-NEXT: v_or_b32_e32 v7, v14, v0
; SDAG-NEXT: v_mov_b32_e32 v0, v7
; SDAG-NEXT: v_mov_b32_e32 v1, v8
-; SDAG-NEXT: .LBB5_6: ; %Flow1
+; SDAG-NEXT: ; %bb.6: ; %Flow1
; SDAG-NEXT: s_or_b64 exec, exec, s[12:13]
; SDAG-NEXT: .LBB5_7: ; %Flow2
; SDAG-NEXT: s_andn2_saveexec_b64 s[4:5], s[10:11]
@@ -1563,7 +1552,6 @@ define half @uitofp_i128_to_f16(i128 %x) {
; GISEL-NEXT: ; %bb.4: ; %LeafBlock
; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 26, v6
; GISEL-NEXT: s_and_saveexec_b64 s[12:13], vcc
-; GISEL-NEXT: s_cbranch_execz .LBB5_6
; GISEL-NEXT: ; %bb.5: ; %itofp-sw-default
; GISEL-NEXT: v_sub_u32_e32 v4, 0x66, v5
; GISEL-NEXT: v_sub_u32_e32 v10, 64, v4
@@ -1606,7 +1594,7 @@ define half @uitofp_i128_to_f16(i128 %x) {
; GISEL-NEXT: v_mov_b32_e32 v1, v4
; GISEL-NEXT: v_mov_b32_e32 v2, v5
; GISEL-NEXT: v_mov_b32_e32 v3, v6
-; GISEL-NEXT: .LBB5_6: ; %Flow1
+; GISEL-NEXT: ; %bb.6: ; %Flow1
; GISEL-NEXT: s_or_b64 exec, exec, s[12:13]
; GISEL-NEXT: .LBB5_7: ; %Flow2
; GISEL-NEXT: s_andn2_saveexec_b64 s[4:5], s[10:11]
diff --git a/llvm/test/Transforms/CodeGenPrepare/wave-profile-edge.ll b/llvm/test/Transforms/CodeGenPrepare/wave-profile-edge.ll
new file mode 100644
index 00000000000000..175f2b98784a84
--- /dev/null
+++ b/llvm/test/Transforms/CodeGenPrepare/wave-profile-edge.ll
@@ -0,0 +1,72 @@
+; NOTE: Do not autogenerate. Check wave event identities and measured/unknown\n; state across composed CFG rewrites without unrelated instruction checks.\n; REQUIRES: amdgpu-registered-target
+; RUN: split-file %s %t
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx950 -passes='require<profile-summary>,function(codegenprepare),verify' %t/edge.ll | FileCheck %s --check-prefix=CLEAN
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx950 -passes='require<profile-summary>,function(codegenprepare,codegenprepare),verify' %t/edge.ll | FileCheck %s --check-prefix=CLEAN
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx950 -passes='require<profile-summary>,function(codegenprepare),verify' %t/invalid.ll | FileCheck %s --check-prefix=INVALID
+
+; An empty block with one predecessor only subdivides an edge. Retain the
+; surrounding measurements when removing that block, including repeated
+; cleanup. Previously invalid measurements must stay invalid.
+;
+; CLEAN-LABEL: define amdgpu_kernel void @_Z14divergent_loopPVi(
+; CLEAN-SAME: !wave.profile [[TABLE:![0-9]+]]
+; CLEAN: br i1 %run, label %body, label %exit, !wave.profile.block [[ENTRY:![0-9]+]]
+; CLEAN: br i1 %more, label %body, label %exit, !wave.profile.block [[BODY:![0-9]+]]
+; CLEAN: ret void, !wave.profile.block [[EXIT:![0-9]+]]
+; CLEAN: [[TABLE]] = distinct !{i64 2, i64 2480672276464841217, i64 6, i64 66, i64 6, i64 6{{(, i64 0)*}}}
+; CLEAN: [[ENTRY]] = !{i64 2, i64 2480672276464841217, i64 0, i64 1, i64 1, i64 3}
+; CLEAN: [[BODY]] = !{i64 2, i64 2480672276464841217, i64 1, i64 1, i64 1, i64 3}
+; CLEAN: [[EXIT]] = !{i64 2, i64 2480672276464841217, i64 3, i64 1}
+;
+;--- edge.ll
+define amdgpu_kernel void @_Z14divergent_loopPVi(ptr addrspace(1) %out, i32 %n, i1 %run) !wave.profile !0 {
+entry:
+ br i1 %run, label %body, label %exit, !wave.profile.block !1
+body:
+ %i = phi i32 [0, %entry], [%next, %body]
+ store volatile i32 %i, ptr addrspace(1) %out
+ %next = add nuw nsw i32 %i, 1
+ %more = icmp ult i32 %next, %n
+ br i1 %more, label %body, label %cleanup, !wave.profile.block !2
+cleanup:
+ br label %exit, !wave.profile.block !3
+exit:
+ store volatile i32 42, ptr addrspace(1) %out
+ ret void, !wave.profile.block !4
+}
+!0 = !{i64 2, i64 2480672276464841217, i64 6, i64 66, i64 6, i64 6}
+!1 = !{i64 2, i64 2480672276464841217, i64 0, i64 1, i64 1, i64 3}
+!2 = !{i64 2, i64 2480672276464841217, i64 1, i64 1, i64 1, i64 2}
+!3 = !{i64 2, i64 2480672276464841217, i64 2, i64 1, i64 3}
+!4 = !{i64 2, i64 2480672276464841217, i64 3, i64 1}
+
+;--- invalid.ll
+; The body's old successor list already mismatches. Removing cleanup makes
+; those old IDs match again, but must not revive the rejected measurements.
+; INVALID-LABEL: define amdgpu_kernel void @_Z14divergent_loopPVi(
+; INVALID: br i1 %run, label %body, label %exit, !wave.profile.block [[ENTRY:![0-9]+]]
+; INVALID: br i1 %more, label %body, label %exit, !wave.profile.block [[BODY:![0-9]+]]
+; INVALID: ret void, !wave.profile.block [[EXIT:![0-9]+]]
+; INVALID: [[ENTRY]] = !{i64 2, i64 2480672276464841217, i64 0, i64 1, i64 1, i64 3}
+; INVALID: [[BODY]] = !{i64 2, i64 2480672276464841217, i64 1, i64 0, i64 1, i64 3}
+; INVALID: [[EXIT]] = !{i64 2, i64 2480672276464841217, i64 3, i64 0}
+define amdgpu_kernel void @_Z14divergent_loopPVi(ptr addrspace(1) %out, i32 %n, i1 %run) !wave.profile !0 {
+entry:
+ br i1 %run, label %body, label %exit, !wave.profile.block !1
+body:
+ %i = phi i32 [0, %entry], [%next, %body]
+ store volatile i32 %i, ptr addrspace(1) %out
+ %next = add nuw nsw i32 %i, 1
+ %more = icmp ult i32 %next, %n
+ br i1 %more, label %body, label %cleanup, !wave.profile.block !2
+cleanup:
+ br label %exit, !wave.profile.block !3
+exit:
+ store volatile i32 42, ptr addrspace(1) %out
+ ret void, !wave.profile.block !4
+}
+!0 = !{i64 2, i64 2480672276464841217, i64 6, i64 66, i64 6, i64 6}
+!1 = !{i64 2, i64 2480672276464841217, i64 0, i64 1, i64 1, i64 3}
+!2 = !{i64 2, i64 2480672276464841217, i64 1, i64 1, i64 1, i64 3}
+!3 = !{i64 2, i64 2480672276464841217, i64 2, i64 1, i64 3}
+!4 = !{i64 2, i64 2480672276464841217, i64 3, i64 1}
diff --git a/llvm/test/Transforms/LoopRotate/wave-profile.ll b/llvm/test/Transforms/LoopRotate/wave-profile.ll
new file mode 100644
index 00000000000000..bbec159506ea2d
--- /dev/null
+++ b/llvm/test/Transforms/LoopRotate/wave-profile.ll
@@ -0,0 +1,41 @@
+; RUN: opt -S -passes='loop-mssa(loop-rotate),verify' %s | FileCheck %s
+
+; The old header disappears. Keep the original identity space and the counts
+; for the blocks whose executions have not changed.
+
+define amdgpu_kernel void @_Z14divergent_loopPVi(ptr addrspace(1) %out) !wave.profile !0 {
+; CHECK-LABEL: define amdgpu_kernel void @_Z14divergent_loopPVi(
+; CHECK-SAME: !wave.profile [[PROFILE:![0-9]+]] {
+; CHECK: entry:
+; CHECK: br label %body, !wave.profile.block [[ENTRY:![0-9]+]]
+; CHECK: exit:
+; CHECK: ret void, !wave.profile.block [[EXIT:![0-9]+]]
+; CHECK: body:
+; CHECK: br i1 %cond, label %body, label %exit, !wave.profile.block [[BODY:![0-9]+]]
+;
+; CHECK: [[PROFILE]] = distinct !{i64 2, i64 2480672276464841217, i64 6, i64 30, i64 6, i64 24}
+; CHECK: [[ENTRY]] = !{i64 2, i64 2480672276464841217, i64 0, i64 1, i64 3}
+; CHECK: [[EXIT]] = !{i64 2, i64 2480672276464841217, i64 2, i64 1}
+; CHECK: [[BODY]] = !{i64 2, i64 2480672276464841217, i64 3, i64 1, i64 3, i64 2}
+entry:
+ br label %header, !wave.profile.block !1
+
+header:
+ %i = phi i32 [ 0, %entry ], [ %inc, %body ]
+ %cond = icmp ult i32 %i, 4
+ br i1 %cond, label %body, label %exit, !wave.profile.block !2
+
+exit:
+ ret void, !wave.profile.block !3
+
+body:
+ store volatile i32 %i, ptr addrspace(1) %out
+ %inc = add nuw nsw i32 %i, 1
+ br label %header, !wave.profile.block !4
+}
+
+!0 = !{i64 2, i64 2480672276464841217, i64 6, i64 30, i64 6, i64 24}
+!1 = !{i64 2, i64 2480672276464841217, i64 0, i64 1, i64 1}
+!2 = !{i64 2, i64 2480672276464841217, i64 1, i64 1, i64 3, i64 2}
+!3 = !{i64 2, i64 2480672276464841217, i64 2, i64 1}
+!4 = !{i64 2, i64 2480672276464841217, i64 3, i64 1, i64 1}
diff --git a/llvm/test/Transforms/LoopSimplify/block-wave-profile.ll b/llvm/test/Transforms/LoopSimplify/block-wave-profile.ll
new file mode 100644
index 00000000000000..7098a601d7e988
--- /dev/null
+++ b/llvm/test/Transforms/LoopSimplify/block-wave-profile.ll
@@ -0,0 +1,49 @@
+; NOTE: Do not autogenerate. Check stable measured identities and unmeasured
+; preheaders without depending on metadata node numbering.
+; RUN: opt -passes=loop-simplify,verify -S %s | FileCheck %s
+;
+; Each independent loop after h0 needs a preheader. All original observations
+; survive the batch, and newly created blocks have no measured count.
+;
+; CHECK-LABEL: define void @_Z14divergent_loopPVi(
+; CHECK-SAME: !wave.profile ![[TABLE:[0-9]+]]
+; CHECK: h0:
+; CHECK-NEXT: br i1 %repeat, label %h0, label %h1.preheader, !wave.profile.block ![[H0:[0-9]+]]
+; CHECK: h1.preheader:
+; CHECK-NEXT: br label %h1, !wave.profile.block ![[PRE1:[0-9]+]]
+; CHECK: h1:
+; CHECK-NEXT: br i1 %repeat, label %h1, label %h2.preheader, !wave.profile.block ![[H1:[0-9]+]]
+; CHECK: h2.preheader:
+; CHECK-NEXT: br label %h2, !wave.profile.block ![[PRE2:[0-9]+]]
+; CHECK: h2:
+; CHECK-NEXT: br i1 %repeat, label %h2, label %exit, !wave.profile.block ![[H2:[0-9]+]]
+; CHECK: exit:
+; CHECK-NEXT: ret void, !wave.profile.block ![[EXIT:[0-9]+]]
+
+define void @_Z14divergent_loopPVi(i1 %repeat) !wave.profile !0 {
+entry:
+ br label %h0, !wave.profile.block !1
+h0:
+ br i1 %repeat, label %h0, label %h1, !wave.profile.block !2
+h1:
+ br i1 %repeat, label %h1, label %h2, !wave.profile.block !3
+h2:
+ br i1 %repeat, label %h2, label %exit, !wave.profile.block !4
+exit:
+ ret void, !wave.profile.block !5
+}
+
+; CHECK: ![[TABLE]] = distinct !{i64 2, i64 2480672276464841217, i64 1, i64 1, i64 1, i64 1, i64 1, i64 0, i64 0}
+; CHECK: ![[H0]] = !{i64 2, i64 2480672276464841217, i64 1, i64 1, i64 1, i64 [[ID1:[0-9]+]]}
+; CHECK: ![[PRE1]] = !{i64 2, i64 2480672276464841217, i64 [[ID1]], i64 0, i64 2}
+; CHECK: ![[H1]] = !{i64 2, i64 2480672276464841217, i64 2, i64 1, i64 2, i64 [[ID2:[0-9]+]]}
+; CHECK: ![[PRE2]] = !{i64 2, i64 2480672276464841217, i64 [[ID2]], i64 0, i64 3}
+; CHECK: ![[H2]] = !{i64 2, i64 2480672276464841217, i64 3, i64 1, i64 3, i64 4}
+; CHECK: ![[EXIT]] = !{i64 2, i64 2480672276464841217, i64 4, i64 1}
+
+!0 = distinct !{i64 2, i64 2480672276464841217, i64 1, i64 1, i64 1, i64 1, i64 1}
+!1 = !{i64 2, i64 2480672276464841217, i64 0, i64 1, i64 1}
+!2 = !{i64 2, i64 2480672276464841217, i64 1, i64 1, i64 1, i64 2}
+!3 = !{i64 2, i64 2480672276464841217, i64 2, i64 1, i64 2, i64 3}
+!4 = !{i64 2, i64 2480672276464841217, i64 3, i64 1, i64 3, i64 4}
+!5 = !{i64 2, i64 2480672276464841217, i64 4, i64 1}
diff --git a/llvm/test/Transforms/LoopUnroll/wave-profile.ll b/llvm/test/Transforms/LoopUnroll/wave-profile.ll
new file mode 100644
index 00000000000000..6df3175dde5cf0
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/wave-profile.ll
@@ -0,0 +1,141 @@
+; NOTE: Do not autogenerate. Check wave event identities and measured/unknown\n; state across composed CFG rewrites without unrelated instruction checks.\n; RUN: split-file %s %t
+; RUN: opt -S -passes='loop(loop-unroll-full),verify' %t/simple.ll | FileCheck %s --check-prefix=UNROLL
+; RUN: opt -S -passes='loop(loop-unroll-full),simplifycfg,verify' %t/simple.ll | FileCheck %s --check-prefix=MERGE
+; RUN: opt -S -passes='loop(loop-unroll-full),simplifycfg,verify' %t/nested.ll | FileCheck %s --check-prefix=NESTED
+; RUN: opt -S -passes='loop-unroll,verify' -unroll-count=2 %t/partial.ll | FileCheck %s --check-prefix=PARTIAL
+; RUN: opt -S -passes='loop-unroll,verify' -unroll-count=4 %t/early-exit.ll | FileCheck %s --check-prefix=EARLY
+
+;--- simple.ll
+;
+; Full unrolling replaces a loop event and transplants its cleanup terminator.
+; Keep the entry event and its normalization, without retaining the loop count.
+; Subsequent merging retains the predecessor's measured event.
+;
+; UNROLL-LABEL: define amdgpu_kernel void @_Z14divergent_loopPVi(
+; UNROLL-SAME: !wave.profile [[TABLE:![0-9]+]]
+; UNROLL: entry:
+; UNROLL: br label %body, !wave.profile.block [[ENTRY:![0-9]+]]
+; UNROLL: body:
+; UNROLL-COUNT-4: store volatile i32
+; UNROLL: ret void, !wave.profile.block [[BODY:![0-9]+]]
+; UNROLL: [[TABLE]] = distinct !{i64 2, i64 2480672276464841217, i64 6, i64 24, i64 0{{(, i64 0)+}}}
+; UNROLL: [[ENTRY]] = !{i64 2, i64 2480672276464841217, i64 0, i64 1, i64 3}
+; UNROLL: [[BODY]] = !{i64 2, i64 2480672276464841217, i64 3, i64 0}
+;
+; MERGE-LABEL: define amdgpu_kernel void @_Z14divergent_loopPVi(
+; MERGE: entry:
+; MERGE-COUNT-4: store volatile i32
+; MERGE: ret void, !wave.profile.block [[ENTRY:![0-9]+]]
+; MERGE: [[ENTRY]] = !{i64 2, i64 2480672276464841217, i64 0, i64 1}
+define amdgpu_kernel void @_Z14divergent_loopPVi(ptr addrspace(1) %out) !wave.profile !0 {
+entry:
+ br label %body, !wave.profile.block !1
+body:
+ %i = phi i32 [0, %entry], [%next, %body]
+ store volatile i32 %i, ptr addrspace(1) %out
+ %next = add nuw nsw i32 %i, 1
+ %more = icmp ult i32 %next, 4
+ br i1 %more, label %body, label %exit, !wave.profile.block !2
+exit:
+ ret void, !wave.profile.block !3
+}
+!0 = !{i64 2, i64 2480672276464841217, i64 6, i64 24, i64 0}
+!1 = !{i64 2, i64 2480672276464841217, i64 0, i64 1, i64 1}
+!2 = !{i64 2, i64 2480672276464841217, i64 1, i64 1, i64 1, i64 2}
+!3 = !{i64 2, i64 2480672276464841217, i64 2, i64 0}
+
+;--- nested.ll
+; Preserve the surrounding outer-loop event through inner full unroll and merge.
+; NESTED-LABEL: define amdgpu_kernel void @_Z14divergent_loopPVi(
+; NESTED-SAME: !wave.profile [[TABLE:![0-9]+]]
+; NESTED: entry:
+; NESTED: br label %outer, !wave.profile.block [[ENTRY:![0-9]+]]
+; NESTED: outer:
+; NESTED-COUNT-4: store volatile i32
+; NESTED: br i1 %outer.more, label %outer, label %exit, !llvm.loop {{![0-9]+}}, !wave.profile.block [[OUTER:![0-9]+]]
+; NESTED: [[TABLE]] = distinct !{i64 2, i64 2480672276464841217, i64 6, i64 66, i64 264, i64 66, i64 0{{(, i64 0)+}}}
+; NESTED: [[ENTRY]] = !{i64 2, i64 2480672276464841217, i64 0, i64 1, i64 1}
+; NESTED: [[OUTER]] = !{i64 2, i64 2480672276464841217, i64 1, i64 1, i64 1, i64 4}
+define amdgpu_kernel void @_Z14divergent_loopPVi(ptr addrspace(1) %out, i32 %n) !wave.profile !0 {
+entry:
+ br label %outer, !wave.profile.block !1
+outer:
+ %j = phi i32 [0, %entry], [%j.next, %outer.latch]
+ br label %body, !wave.profile.block !2
+body:
+ %i = phi i32 [0, %outer], [%next, %body]
+ store volatile i32 %i, ptr addrspace(1) %out
+ %next = add nuw nsw i32 %i, 1
+ %more = icmp ult i32 %next, 4
+ br i1 %more, label %body, label %outer.latch, !wave.profile.block !3
+outer.latch:
+ %j.next = add nuw nsw i32 %j, 1
+ %outer.more = icmp ult i32 %j.next, %n
+ br i1 %outer.more, label %outer, label %exit, !llvm.loop !6, !wave.profile.block !4
+exit:
+ ret void, !wave.profile.block !5
+}
+!0 = !{i64 2, i64 2480672276464841217, i64 6, i64 66, i64 264, i64 66, i64 0}
+!1 = !{i64 2, i64 2480672276464841217, i64 0, i64 1, i64 1}
+!2 = !{i64 2, i64 2480672276464841217, i64 1, i64 1, i64 2}
+!3 = !{i64 2, i64 2480672276464841217, i64 2, i64 1, i64 2, i64 3}
+!4 = !{i64 2, i64 2480672276464841217, i64 3, i64 1, i64 1, i64 4}
+!5 = !{i64 2, i64 2480672276464841217, i64 4, i64 0}
+!6 = distinct !{!6, !7}
+!7 = !{!"llvm.loop.unroll.disable"}
+
+;--- partial.ll
+; The original loop count does not describe the partially unrolled loop.
+; Do not revive it when the duplicated identities collapse during merging.
+; PARTIAL-LABEL: define amdgpu_kernel void @_Z14divergent_loopPVi(
+; PARTIAL: entry:
+; PARTIAL: br label %body, !wave.profile.block [[ENTRY:![0-9]+]]
+; PARTIAL: body:
+; PARTIAL-COUNT-2: store volatile i32
+; PARTIAL: br i1 %more.1, label %body, label %exit, !llvm.loop {{![0-9]+}}, !wave.profile.block [[BODY:![0-9]+]]
+; PARTIAL: [[ENTRY]] = !{i64 2, i64 2480672276464841217, i64 3, i64 0, i64 4}
+; PARTIAL: [[BODY]] = !{i64 2, i64 2480672276464841217, i64 4, i64 0, i64 4, i64 2}
+define amdgpu_kernel void @_Z14divergent_loopPVi(ptr addrspace(1) %out) !wave.profile !0 {
+entry:
+ br label %body, !wave.profile.block !1
+body:
+ %i = phi i32 [0, %entry], [%next, %body]
+ store volatile i32 %i, ptr addrspace(1) %out
+ %next = add nuw nsw i32 %i, 1
+ %more = icmp ult i32 %next, 8
+ br i1 %more, label %body, label %exit, !wave.profile.block !2
+exit:
+ ret void, !wave.profile.block !3
+}
+!0 = !{i64 2, i64 2480672276464841217, i64 6, i64 48, i64 0}
+!1 = !{i64 2, i64 2480672276464841217, i64 0, i64 1, i64 1}
+!2 = !{i64 2, i64 2480672276464841217, i64 1, i64 1, i64 1, i64 2}
+!3 = !{i64 2, i64 2480672276464841217, i64 2, i64 0}
+
+;--- early-exit.ll
+; A trip-count upper bound is not an exact trip count. Leave copied identities
+; ambiguous so the validator rejects them, rather than certifying new events.
+; EARLY-LABEL: define amdgpu_kernel void @_Z14divergent_loopPVi(
+; EARLY-SAME: !wave.profile [[TABLE:![0-9]+]]
+; EARLY: br i1 %keepGoing, label %body.1, label %exit, !wave.profile.block [[COPY:![0-9]+]]
+; EARLY: br i1 %keepGoing, label %body.2, label %exit, !wave.profile.block [[COPY]]
+; EARLY: br i1 %keepGoing, label %body.3, label %exit, !wave.profile.block [[COPY]]
+; EARLY: [[TABLE]] = !{i64 2, i64 2480672276464841217, i64 6, i64 24, i64 0}
+; EARLY: [[COPY]] = !{i64 2, i64 2480672276464841217, i64 1, i64 1, i64 1, i64 2}
+define amdgpu_kernel void @_Z14divergent_loopPVi(ptr addrspace(1) %out, i1 %keepGoing) !wave.profile !0 {
+entry:
+ br label %body, !wave.profile.block !1
+body:
+ %i = phi i32 [0, %entry], [%next, %body]
+ store volatile i32 %i, ptr addrspace(1) %out
+ %next = add nuw nsw i32 %i, 1
+ %more = icmp ult i32 %next, 4
+ %again = and i1 %more, %keepGoing
+ br i1 %again, label %body, label %exit, !wave.profile.block !2
+exit:
+ ret void, !wave.profile.block !3
+}
+!0 = !{i64 2, i64 2480672276464841217, i64 6, i64 24, i64 0}
+!1 = !{i64 2, i64 2480672276464841217, i64 0, i64 1, i64 1}
+!2 = !{i64 2, i64 2480672276464841217, i64 1, i64 1, i64 1, i64 2}
+!3 = !{i64 2, i64 2480672276464841217, i64 2, i64 0}
diff --git a/llvm/test/Transforms/LowerSwitch/profile-weights.ll b/llvm/test/Transforms/LowerSwitch/profile-weights.ll
new file mode 100644
index 00000000000000..3640e5f0864969
--- /dev/null
+++ b/llvm/test/Transforms/LowerSwitch/profile-weights.ll
@@ -0,0 +1,157 @@
+; RUN: opt -S -passes='lower-switch,verify' %s | FileCheck %s
+
+; With no default traffic, each comparison receives the sum of its cases.
+; CHECK-LABEL: define void @zero_default(
+; CHECK: br i1 %Pivot{{.*}}, label {{.*}}, label {{.*}}, !prof [[ROOT:![0-9]+]]
+; CHECK: br i1 %Pivot{{.*}}, label {{.*}}, label {{.*}}, !prof [[RIGHT:![0-9]+]]
+define void @zero_default(i32 %v) {
+entry:
+ switch i32 %v, label %exit [i32 0, label %a
+ i32 1, label %b
+ i32 2, label %c], !prof !0
+ a: call void @use(i32 0)
+ br label %exit
+ b: call void @use(i32 1)
+ br label %exit
+ c: call void @use(i32 2)
+ br label %exit
+exit: ret void
+}
+
+; The default values can reach either side of the tree. Their split is unknown.
+; CHECK-LABEL: define void @unknown_default_split(
+; CHECK-NOT: !prof
+; CHECK: ret void
+define void @unknown_default_split(i32 %v) {
+entry:
+ switch i32 %v, label %exit [i32 0, label %a
+ i32 2, label %b], !prof !1
+ a: call void @use(i32 0)
+ br label %exit
+ b: call void @use(i32 1)
+ br label %exit
+exit: ret void
+}
+
+; A single comparison can retain both weights, including the expected marker.
+; CHECK-LABEL: define void @single_case(
+; CHECK: br i1 %SwitchLeaf, label %a, label %exit, !prof [[SINGLE:![0-9]+]]
+define void @single_case(i32 %v) {
+entry:
+ switch i32 %v, label %exit [i32 7, label %a], !prof !2
+ a: call void @use(i32 0)
+ br label %exit
+exit: ret void
+}
+
+; An i2 has only four values, so all default traffic must take value 1.
+; CHECK-LABEL: define void @known_default_value(
+; CHECK: br i1 %Pivot{{.*}}, label {{.*}}, label {{.*}}, !prof [[KNOWN:![0-9]+]]
+define void @known_default_value(i2 %v) {
+entry:
+ switch i2 %v, label %exit [i2 -2, label %a
+ i2 -1, label %b
+ i2 0, label %c], !prof !3
+ a: call void @use(i32 0)
+ br label %exit
+ b: call void @use(i32 1)
+ br label %exit
+ c: call void @use(i32 2)
+ br label %exit
+exit: ret void
+}
+
+; Replacing an unreachable default with a popular destination retains all of
+; that destination's case weights, including noncontiguous case values.
+; CHECK-LABEL: define void @popular_default(
+; CHECK: br i1 %SwitchLeaf, label %b, label %a, !prof [[POPULAR:![0-9]+]]
+define void @popular_default(i32 %v) {
+entry:
+ switch i32 %v, label %dead [i32 0, label %a
+ i32 1, label %a
+ i32 2, label %b
+ i32 3, label %a], !prof !4
+ a: call void @use(i32 0)
+ br label %exit
+ b: call void @use(i32 1)
+ br label %exit
+exit: ret void
+dead: unreachable
+}
+
+; Explicit cases targeting the default still have individually known weights.
+; CHECK-LABEL: define void @explicit_default_case(
+; CHECK: br i1 %Pivot{{.*}}, label {{.*}}, label {{.*}}, !prof [[POPULAR]]
+define void @explicit_default_case(i2 %v) {
+entry:
+ switch i2 %v, label %exit [i2 -2, label %a
+ i2 -1, label %exit
+ i2 0, label %b], !prof !3
+ a: call void @use(i32 0)
+ br label %exit
+ b: call void @use(i32 1)
+ br label %exit
+exit: ret void
+}
+
+; Adjacent values sharing a destination retain their combined mass.
+; CHECK-LABEL: define void @clustered_cases(
+; CHECK: br i1 %Pivot{{.*}}, label {{.*}}, label {{.*}}, !prof [[POPULAR]]
+define void @clustered_cases(i32 %v) {
+entry:
+ switch i32 %v, label %exit [i32 0, label %a
+ i32 1, label %a
+ i32 3, label %b
+ i32 4, label %c], !prof !4
+ a: call void @use(i32 0)
+ br label %exit
+ b: call void @use(i32 1)
+ br label %exit
+ c: call void @use(i32 2)
+ br label %exit
+exit: ret void
+}
+
+; Sums use 64 bits and are scaled together when they do not fit in i32.
+; CHECK-LABEL: define void @large_weights(
+; CHECK: br i1 %Pivot{{.*}}, label {{.*}}, label {{.*}}, !prof [[LARGE:![0-9]+]]
+define void @large_weights(i32 %v) {
+entry:
+ switch i32 %v, label %exit [i32 0, label %a
+ i32 1, label %b
+ i32 2, label %c], !prof !5
+ a: call void @use(i32 0)
+ br label %exit
+ b: call void @use(i32 1)
+ br label %exit
+ c: call void @use(i32 2)
+ br label %exit
+exit: ret void
+}
+
+; CHECK-LABEL: define void @all_zero(
+; CHECK-NOT: !prof
+; CHECK: ret void
+define void @all_zero(i32 %v) {
+entry:
+ switch i32 %v, label %exit [i32 7, label %a], !prof !6
+ a: call void @use(i32 0)
+ br label %exit
+exit: ret void
+}
+
+declare void @use(i32)
+!0 = !{!"branch_weights", i32 0, i32 10, i32 20, i32 30}
+!1 = !{!"branch_weights", i32 100, i32 10, i32 20}
+!2 = !{!"branch_weights", !"expected", i32 40, i32 60}
+!3 = !{!"branch_weights", i32 40, i32 10, i32 20, i32 30}
+!4 = !{!"branch_weights", i32 0, i32 10, i32 20, i32 30, i32 40}
+!5 = !{!"branch_weights", i32 0, i32 -1, i32 -1, i32 -1}
+!6 = !{!"branch_weights", i32 0, i32 0}
+
+; CHECK-DAG: [[ROOT]] = !{!"branch_weights", i32 10, i32 50}
+; CHECK-DAG: [[RIGHT]] = !{!"branch_weights", i32 20, i32 30}
+; CHECK-DAG: [[SINGLE]] = !{!"branch_weights", !"expected", i32 60, i32 40}
+; CHECK-DAG: [[KNOWN]] = !{!"branch_weights", i32 10, i32 90}
+; CHECK-DAG: [[POPULAR]] = !{!"branch_weights", i32 30, i32 70}
+; CHECK-DAG: [[LARGE]] = !{!"branch_weights", i32 2147483647, i32 -1}
diff --git a/llvm/test/Transforms/LowerSwitch/wave-profile.ll b/llvm/test/Transforms/LowerSwitch/wave-profile.ll
new file mode 100644
index 00000000000000..2b625f90e4b7cf
--- /dev/null
+++ b/llvm/test/Transforms/LowerSwitch/wave-profile.ll
@@ -0,0 +1,56 @@
+; RUN: opt -S -passes='lower-switch,verify' %s | FileCheck %s --implicit-check-not=block.uniformity.profile
+; RUN: opt -S -passes='lower-switch,structurizecfg,verify' %s | FileCheck %s --check-prefix=PIPE
+
+; Keep the original invocation anchor even though original entry ID zero is
+; absent. A synthetic comparison receives an identity but no measured count.
+define void @if_else(i32 %v, ptr %out) !uniformity.profile !5 !wave.profile !0 {
+; CHECK-LABEL: define void @if_else(
+; CHECK-SAME: !wave.profile [[PROFILE:![0-9]+]]
+entry:
+; CHECK: entry:
+; CHECK: br label %LeafBlock, !block.uniformity.profile {{![0-9]+}}, !wave.profile.block [[ENTRY:![0-9]+]]
+ switch i32 %v, label %right [i32 0, label %left], !wave.profile.block !1, !block.uniformity.profile !5
+; CHECK: LeafBlock:
+; CHECK: br i1 %SwitchLeaf, label %left, label %right, !wave.profile.block [[LEAF:![0-9]+]]
+left:
+ store volatile i32 1, ptr %out
+ br label %exit, !wave.profile.block !2
+right:
+ store volatile i32 2, ptr %out
+ br label %exit, !wave.profile.block !3
+exit:
+ ret void, !wave.profile.block !4
+}
+!0 = !{i64 2, i64 2685589004101179296, i64 11, i64 10, i64 7, i64 3, i64 10}
+!1 = !{i64 2, i64 2685589004101179296, i64 1, i64 1, i64 3, i64 2}
+!2 = !{i64 2, i64 2685589004101179296, i64 2, i64 1, i64 4}
+!3 = !{i64 2, i64 2685589004101179296, i64 3, i64 1, i64 4}
+!4 = !{i64 2, i64 2685589004101179296, i64 4, i64 1}
+!5 = !{}
+; CHECK-DAG: [[PROFILE]] = distinct !{i64 2, i64 2685589004101179296, i64 11, i64 10, i64 7, i64 3, i64 10, i64 0}
+; CHECK-DAG: [[ENTRY]] = !{i64 2, i64 2685589004101179296, i64 1, i64 1, i64 5}
+; CHECK-DAG: [[LEAF]] = !{i64 2, i64 2685589004101179296, i64 5, i64 0, i64 2, i64 3}
+
+; The second CFG pass must retain measured identities and the original entry
+; anchor while assigning another unmeasured identity to its flow block.
+; PIPE-LABEL: define void @if_else(
+; PIPE-SAME: !wave.profile [[PIPE_PROFILE:![0-9]+]] {
+; PIPE: entry:
+; PIPE: br label %LeafBlock, !block.uniformity.profile {{![0-9]+}}, !wave.profile.block [[PIPE_ENTRY:![0-9]+]]
+; PIPE: LeafBlock:
+; PIPE: br i1 {{.*}}, label %right, label %Flow, !wave.profile.block [[PIPE_LEAF:![0-9]+]]
+; PIPE: Flow:
+; PIPE: br i1 {{.*}}, label %left, label %exit, !wave.profile.block [[PIPE_FLOW:![0-9]+]]
+; PIPE: left:
+; PIPE: br label %exit, !wave.profile.block [[PIPE_LEFT:![0-9]+]]
+; PIPE: right:
+; PIPE: br label %Flow, !wave.profile.block [[PIPE_RIGHT:![0-9]+]]
+; PIPE: exit:
+; PIPE: ret void, !wave.profile.block [[PIPE_EXIT:![0-9]+]]
+; PIPE: [[PIPE_PROFILE]] = distinct !{i64 2, i64 2685589004101179296, i64 11, i64 10, i64 7, i64 3, i64 10, i64 0, i64 0}
+; PIPE-DAG: [[PIPE_ENTRY]] = !{i64 2, i64 2685589004101179296, i64 1, i64 1, i64 5}
+; PIPE-DAG: [[PIPE_LEAF]] = !{i64 2, i64 2685589004101179296, i64 5, i64 0, i64 3, i64 6}
+; PIPE-DAG: [[PIPE_FLOW]] = !{i64 2, i64 2685589004101179296, i64 6, i64 0, i64 2, i64 4}
+; PIPE-DAG: [[PIPE_LEFT]] = !{i64 2, i64 2685589004101179296, i64 2, i64 1, i64 4}
+; PIPE-DAG: [[PIPE_RIGHT]] = !{i64 2, i64 2685589004101179296, i64 3, i64 1, i64 6}
+; PIPE-DAG: [[PIPE_EXIT]] = !{i64 2, i64 2685589004101179296, i64 4, i64 1}
diff --git a/llvm/test/Transforms/SimplifyCFG/wave-profile-edge.ll b/llvm/test/Transforms/SimplifyCFG/wave-profile-edge.ll
new file mode 100644
index 00000000000000..60a8ba4d0a22bf
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/wave-profile-edge.ll
@@ -0,0 +1,82 @@
+; NOTE: Do not autogenerate. Check wave event identities and measured/unknown\n; state across composed CFG rewrites without unrelated instruction checks.\n; RUN: split-file %s %t
+; RUN: opt -S -passes='simplifycfg,verify' %t/edge.ll | FileCheck %s --check-prefix=CLEAN
+; RUN: opt -S -passes='loop-simplify,verify' %t/edge.ll | FileCheck %s --check-prefix=SPLIT
+; RUN: opt -S -passes='simplifycfg,loop-simplify,simplifycfg,loop-simplify,simplifycfg,verify' %t/edge.ll | FileCheck %s --check-prefix=CLEAN
+; RUN: opt -S -passes='simplifycfg,verify' %t/invalid.ll | FileCheck %s --check-prefix=INVALID
+; RUN: opt -S -passes='simplifycfg,loop-simplify,simplifycfg,verify' %t/invalid.ll | FileCheck %s --check-prefix=INVALID
+
+; An empty block with one predecessor only subdivides an edge. Retain the
+; surrounding measurements when adding/removing that block, including repeated
+; loop canonicalization. New blocks remain unmeasured.
+;
+; CLEAN-LABEL: define amdgpu_kernel void @_Z14divergent_loopPVi(
+; CLEAN-SAME: !wave.profile [[TABLE:![0-9]+]]
+; CLEAN: br i1 %run, label %body, label %exit, !wave.profile.block [[ENTRY:![0-9]+]]
+; CLEAN: br i1 %more, label %body, label %exit, !wave.profile.block [[BODY:![0-9]+]]
+; CLEAN: ret void, !wave.profile.block [[EXIT:![0-9]+]]
+; CLEAN: [[TABLE]] = distinct !{i64 2, i64 2480672276464841217, i64 6, i64 66, i64 6, i64 6{{(, i64 0)*}}}
+; CLEAN: [[ENTRY]] = !{i64 2, i64 2480672276464841217, i64 0, i64 1, i64 1, i64 3}
+; CLEAN: [[BODY]] = !{i64 2, i64 2480672276464841217, i64 1, i64 1, i64 1, i64 3}
+; CLEAN: [[EXIT]] = !{i64 2, i64 2480672276464841217, i64 3, i64 1}
+;
+; SPLIT-LABEL: define amdgpu_kernel void @_Z14divergent_loopPVi(
+; SPLIT: br i1 %run, label %body.preheader, label %exit, !wave.profile.block [[ENTRY:![0-9]+]]
+; SPLIT: body.preheader:
+; SPLIT: br label %body, !wave.profile.block [[NEW:![0-9]+]]
+; SPLIT: br i1 %more, label %body, label %cleanup, !wave.profile.block [[BODY:![0-9]+]]
+; SPLIT: [[ENTRY]] = !{i64 2, i64 2480672276464841217, i64 0, i64 1, i64 4, i64 3}
+; SPLIT: [[NEW]] = !{i64 2, i64 2480672276464841217, i64 4, i64 0, i64 1}
+; SPLIT: [[BODY]] = !{i64 2, i64 2480672276464841217, i64 1, i64 1, i64 1, i64 2}
+
+;--- edge.ll
+define amdgpu_kernel void @_Z14divergent_loopPVi(ptr addrspace(1) %out, i32 %n, i1 %run) !wave.profile !0 {
+entry:
+ br i1 %run, label %body, label %exit, !wave.profile.block !1
+body:
+ %i = phi i32 [0, %entry], [%next, %body]
+ store volatile i32 %i, ptr addrspace(1) %out
+ %next = add nuw nsw i32 %i, 1
+ %more = icmp ult i32 %next, %n
+ br i1 %more, label %body, label %cleanup, !wave.profile.block !2
+cleanup:
+ br label %exit, !wave.profile.block !3
+exit:
+ store volatile i32 42, ptr addrspace(1) %out
+ ret void, !wave.profile.block !4
+}
+!0 = !{i64 2, i64 2480672276464841217, i64 6, i64 66, i64 6, i64 6}
+!1 = !{i64 2, i64 2480672276464841217, i64 0, i64 1, i64 1, i64 3}
+!2 = !{i64 2, i64 2480672276464841217, i64 1, i64 1, i64 1, i64 2}
+!3 = !{i64 2, i64 2480672276464841217, i64 2, i64 1, i64 3}
+!4 = !{i64 2, i64 2480672276464841217, i64 3, i64 1}
+
+;--- invalid.ll
+; The body's old successor list already mismatches. Removing cleanup makes
+; those old IDs match again, but must not revive the rejected measurements.
+; INVALID-LABEL: define amdgpu_kernel void @_Z14divergent_loopPVi(
+; INVALID: br i1 %run, label %body, label %exit, !wave.profile.block [[ENTRY:![0-9]+]]
+; INVALID: br i1 %more, label %body, label %exit, !wave.profile.block [[BODY:![0-9]+]]
+; INVALID: ret void, !wave.profile.block [[EXIT:![0-9]+]]
+; INVALID: [[ENTRY]] = !{i64 2, i64 2480672276464841217, i64 0, i64 1, i64 1, i64 3}
+; INVALID: [[BODY]] = !{i64 2, i64 2480672276464841217, i64 1, i64 0, i64 1, i64 3}
+; INVALID: [[EXIT]] = !{i64 2, i64 2480672276464841217, i64 3, i64 0}
+define amdgpu_kernel void @_Z14divergent_loopPVi(ptr addrspace(1) %out, i32 %n, i1 %run) !wave.profile !0 {
+entry:
+ br i1 %run, label %body, label %exit, !wave.profile.block !1
+body:
+ %i = phi i32 [0, %entry], [%next, %body]
+ store volatile i32 %i, ptr addrspace(1) %out
+ %next = add nuw nsw i32 %i, 1
+ %more = icmp ult i32 %next, %n
+ br i1 %more, label %body, label %cleanup, !wave.profile.block !2
+cleanup:
+ br label %exit, !wave.profile.block !3
+exit:
+ store volatile i32 42, ptr addrspace(1) %out
+ ret void, !wave.profile.block !4
+}
+!0 = !{i64 2, i64 2480672276464841217, i64 6, i64 66, i64 6, i64 6}
+!1 = !{i64 2, i64 2480672276464841217, i64 0, i64 1, i64 1, i64 3}
+!2 = !{i64 2, i64 2480672276464841217, i64 1, i64 1, i64 1, i64 3}
+!3 = !{i64 2, i64 2480672276464841217, i64 2, i64 1, i64 3}
+!4 = !{i64 2, i64 2480672276464841217, i64 3, i64 1}
diff --git a/llvm/test/Transforms/StructurizeCFG/branch-profile-transfer.ll b/llvm/test/Transforms/StructurizeCFG/branch-profile-transfer.ll
new file mode 100644
index 00000000000000..6bda7830e1074f
--- /dev/null
+++ b/llvm/test/Transforms/StructurizeCFG/branch-profile-transfer.ll
@@ -0,0 +1,27 @@
+; RUN: opt -S -passes='structurizecfg,verify' %s | FileCheck %s --implicit-check-not=branch.uniformity.profile
+
+; Inverting the decision in the same block preserves its uniformity and swaps
+; its weights, including the llvm.expect origin. The PHI-driven Flow decision
+; combines different incoming execution populations and receives neither hint.
+; CHECK-LABEL: define void @diamond(
+; CHECK: entry:
+; CHECK: br i1 {{.*}}, label %right, label %Flow, !prof [[WEIGHTS:![0-9]+]], !branch.uniformity.profile [[UNIFORM:![0-9]+]]
+; CHECK: Flow:
+; CHECK: br i1 {{.*}}, label %left, label %exit{{$}}
+define void @diamond(i1 %c, ptr %p) !uniformity.profile !1 {
+entry:
+ br i1 %c, label %left, label %right, !prof !0, !branch.uniformity.profile !1
+left:
+ store volatile i32 1, ptr %p
+ br label %exit
+right:
+ store volatile i32 2, ptr %p
+ br label %exit
+exit:
+ ret void
+}
+
+!0 = !{!"branch_weights", !"expected", i32 90, i32 10}
+!1 = !{}
+; CHECK-DAG: [[WEIGHTS]] = !{!"branch_weights", !"expected", i32 10, i32 90}
+; CHECK-DAG: [[UNIFORM]] = !{}
diff --git a/llvm/test/Transforms/StructurizeCFG/uniformity-profile.ll b/llvm/test/Transforms/StructurizeCFG/uniformity-profile.ll
index df78540888fdff..241d62735c4846 100644
--- a/llvm/test/Transforms/StructurizeCFG/uniformity-profile.ll
+++ b/llvm/test/Transforms/StructurizeCFG/uniformity-profile.ll
@@ -6,7 +6,7 @@
; CHECK-LABEL: define void @diamond(
; CHECK: entry:
-; CHECK: br i1 {{.*}}, label {{.*}}, label {{.*}}, !block.uniformity.profile ![[MD:[0-9]+]]{{$}}
+; CHECK: br i1 {{.*}}, label {{.*}}, label {{.*}}, !block.uniformity.profile ![[MD:[0-9]+]], !branch.uniformity.profile ![[MD]]{{$}}
; CHECK: then:
; CHECK-NEXT: store i32 1, ptr %out
; CHECK-NEXT: br label {{.*}}, !block.uniformity.profile ![[MD]]{{$}}
@@ -86,7 +86,7 @@ exit:
; Preserve the inner block profiles when a containing region is rewritten.
; CHECK-LABEL: define void @nested(
; CHECK: outer.then:
-; CHECK-NEXT: br i1 {{.*}}, label {{.*}}, label {{.*}}, !block.uniformity.profile ![[MD]]{{$}}
+; CHECK-NEXT: br i1 {{.*}}, label {{.*}}, label {{.*}}, !block.uniformity.profile ![[MD]], !branch.uniformity.profile ![[MD]]{{$}}
; CHECK: inner.then:
; CHECK-NEXT: store i32 5, ptr %out
; CHECK-NEXT: br label {{.*}}, !block.uniformity.profile ![[MD]]{{$}}
diff --git a/llvm/test/Transforms/StructurizeCFG/wave-profile-loop-prefix.ll b/llvm/test/Transforms/StructurizeCFG/wave-profile-loop-prefix.ll
new file mode 100644
index 00000000000000..a8eba3fd2b2d13
--- /dev/null
+++ b/llvm/test/Transforms/StructurizeCFG/wave-profile-loop-prefix.ll
@@ -0,0 +1,53 @@
+; RUN: opt -S -passes='structurizecfg,verify' %s | FileCheck %s --implicit-check-not="!prof" --implicit-check-not="!block.uniformity.profile" --implicit-check-not="!branch.uniformity.profile"
+
+; Reusing an empty prefix as a loop header changes its execution event. Keep
+; its identity but invalidate the old count, without losing the entry anchor.
+define void @if_else(i1 %enter, i1 %stop, i1 %again, ptr %out) !wave.profile !0 !uniformity.profile !9 {
+; CHECK-LABEL: define void @if_else(
+; CHECK-SAME: !wave.profile [[WAVE:![0-9]+]]
+entry:
+; CHECK: entry:
+; CHECK: br label %prefix, !wave.profile.block [[ENTRY:![0-9]+]]{{$}}
+ br label %prefix, !wave.profile.block !1
+prefix:
+; CHECK: prefix:
+; CHECK: br {{[^!]*}}!wave.profile.block [[PREFIX:![0-9]+]]{{$}}
+ br i1 %enter, label %a, label %exit.a, !prof !8, !block.uniformity.profile !9, !branch.uniformity.profile !9, !wave.profile.block !2
+a:
+; CHECK: a:
+; CHECK: br {{[^!]*}}!wave.profile.block [[A:![0-9]+]]{{$}}
+ store i32 3, ptr %out
+ br i1 %stop, label %exit.a, label %b, !wave.profile.block !3
+b:
+; CHECK: b:
+; CHECK: br {{[^!]*}}!wave.profile.block [[B:![0-9]+]]{{$}}
+ store i32 4, ptr %out
+ br i1 %again, label %a, label %exit.b, !wave.profile.block !4
+exit.a:
+ store i32 5, ptr %out
+ br label %exit, !wave.profile.block !5
+exit.b:
+ store i32 6, ptr %out
+ br label %exit, !wave.profile.block !6
+exit:
+; CHECK: exit:
+; CHECK: ret void, !wave.profile.block [[EXIT:![0-9]+]]{{$}}
+ ret void, !wave.profile.block !7
+}
+!0 = !{i64 2, i64 2685589004101179296, i64 10, i64 10, i64 30, i64 20, i64 10, i64 0, i64 10}
+!1 = !{i64 2, i64 2685589004101179296, i64 0, i64 1, i64 1}
+!2 = !{i64 2, i64 2685589004101179296, i64 1, i64 1, i64 2, i64 4}
+!3 = !{i64 2, i64 2685589004101179296, i64 2, i64 1, i64 4, i64 3}
+!4 = !{i64 2, i64 2685589004101179296, i64 3, i64 1, i64 2, i64 5}
+!5 = !{i64 2, i64 2685589004101179296, i64 4, i64 1, i64 6}
+!6 = !{i64 2, i64 2685589004101179296, i64 5, i64 1, i64 6}
+!7 = !{i64 2, i64 2685589004101179296, i64 6, i64 1}
+; CHECK: [[WAVE]] = distinct !{i64 2, i64 2685589004101179296, i64 10, i64 10, i64 30, i64 20, i64 10, i64 0, i64 10{{.*}}}
+; CHECK: [[ENTRY]] = !{i64 2, i64 2685589004101179296, i64 0, i64 1{{.*}}}
+; CHECK: [[PREFIX]] = !{i64 2, i64 2685589004101179296, i64 1, i64 0{{.*}}}
+; CHECK: [[A]] = !{i64 2, i64 2685589004101179296, i64 2, i64 1{{.*}}}
+; CHECK: [[B]] = !{i64 2, i64 2685589004101179296, i64 3, i64 1{{.*}}}
+; CHECK: [[EXIT]] = !{i64 2, i64 2685589004101179296, i64 6, i64 1}
+
+!8 = !{!"branch_weights", i32 90, i32 10}
+!9 = !{}
diff --git a/llvm/test/Transforms/StructurizeCFG/wave-profile.ll b/llvm/test/Transforms/StructurizeCFG/wave-profile.ll
new file mode 100644
index 00000000000000..b21d8e4198775c
--- /dev/null
+++ b/llvm/test/Transforms/StructurizeCFG/wave-profile.ll
@@ -0,0 +1,50 @@
+; RUN: opt -S -passes='structurizecfg,verify' %s | FileCheck %s
+
+target triple = "amdgcn-amd-amdhsa"
+
+; A known CFG transform preserves a partial profile's original identities on
+; surviving blocks and refreshes their successor identities. The synthesized
+; flow block gets an identity but no measurement. The absent original entry
+; retains its normalization count (11), distinct from the current entry (10).
+
+define amdgpu_kernel void @if_else(i1 %condition, ptr addrspace(1) %out) !wave.profile !0 {
+; CHECK-LABEL: define amdgpu_kernel void @if_else(
+; CHECK-SAME: !wave.profile [[PROFILE:![0-9]+]] {
+; CHECK: entry:
+; CHECK: br i1 {{.*}}, label %{{.*}}, label %[[FLOW:.*]], {{.*}}!wave.profile.block [[ENTRY_MD:![0-9]+]]
+; CHECK: [[FLOW]]:
+; CHECK: br i1 {{.*}}, label %left, label %exit, !wave.profile.block [[FLOW_MD:![0-9]+]]
+; CHECK: left:
+; CHECK: br label %{{.*}}, !wave.profile.block [[LEFT_MD:![0-9]+]]
+; CHECK: right:
+; CHECK: br label %{{.*}}, !wave.profile.block [[RIGHT_MD:![0-9]+]]
+; CHECK: exit:
+; CHECK: ret void, !wave.profile.block [[EXIT_MD:![0-9]+]]
+;
+entry:
+ br i1 %condition, label %left, label %right, !wave.profile.block !1
+
+left:
+ store volatile i32 1, ptr addrspace(1) %out
+ br label %exit, !wave.profile.block !2
+
+right:
+ store volatile i32 2, ptr addrspace(1) %out
+ br label %exit, !wave.profile.block !3
+
+exit:
+ ret void, !wave.profile.block !4
+}
+
+!0 = !{i64 2, i64 2685589004101179296, i64 11, i64 10, i64 7, i64 3, i64 10}
+!1 = !{i64 2, i64 2685589004101179296, i64 1, i64 1, i64 2, i64 3}
+!2 = !{i64 2, i64 2685589004101179296, i64 2, i64 1, i64 4}
+!3 = !{i64 2, i64 2685589004101179296, i64 3, i64 1, i64 4}
+!4 = !{i64 2, i64 2685589004101179296, i64 4, i64 1}
+
+; CHECK: [[PROFILE]] = distinct !{i64 2, i64 2685589004101179296, i64 11, i64 10, i64 7, i64 3, i64 10, i64 0}
+; CHECK: [[ENTRY_MD]] = !{i64 2, i64 2685589004101179296, i64 1, i64 1, i64 3, i64 5}
+; CHECK: [[FLOW_MD]] = !{i64 2, i64 2685589004101179296, i64 5, i64 0, i64 2, i64 4}
+; CHECK: [[LEFT_MD]] = !{i64 2, i64 2685589004101179296, i64 2, i64 1, i64 4}
+; CHECK: [[RIGHT_MD]] = !{i64 2, i64 2685589004101179296, i64 3, i64 1, i64 5}
+; CHECK: [[EXIT_MD]] = !{i64 2, i64 2685589004101179296, i64 4, i64 1}
diff --git a/llvm/unittests/IR/ProfDataUtilsTest.cpp b/llvm/unittests/IR/ProfDataUtilsTest.cpp
index c7fc0a020e6de1..bdf5bff698447e 100644
--- a/llvm/unittests/IR/ProfDataUtilsTest.cpp
+++ b/llvm/unittests/IR/ProfDataUtilsTest.cpp
@@ -15,7 +15,9 @@
#include "llvm/IR/Module.h"
#include "llvm/IR/Verifier.h"
#include "llvm/Support/SourceMgr.h"
+#include "llvm/Transforms/Utils/BasicBlockUtils.h"
#include "llvm/Transforms/Utils/Cloning.h"
+#include "llvm/Transforms/Utils/Local.h"
#include "gtest/gtest.h"
#include <initializer_list>
@@ -63,6 +65,228 @@ class WaveProfileTest : public testing::Test {
}
};
+TEST_F(WaveProfileTest, BatchSplitsAndRemovalsPreserveMeasuredZero) {
+ Function &F = *M->getFunction("diamond");
+ setBlockWaveCounts(F, {100, 0, 60, 100});
+ BasicBlock *Left = F.getEntryBlock().getNextNode();
+ BasicBlock *Right = Left->getNextNode();
+ BasicBlock *Exit = Right->getNextNode();
+ MDNode *Original = F.getMetadata(LLVMContext::MD_wave_profile);
+ BlockWaveCountUpdater WPU(F);
+ BasicBlock *SplitLeft = SplitBlockPredecessors(
+ Exit, {Left}, ".left", static_cast<DominatorTree *>(nullptr), nullptr,
+ nullptr, false, &WPU);
+ BasicBlock *SplitRight = SplitBlockPredecessors(
+ Exit, {Right}, ".right", static_cast<DominatorTree *>(nullptr), nullptr,
+ nullptr, false, &WPU);
+ EXPECT_EQ(F.getMetadata(LLVMContext::MD_wave_profile), Original);
+ WPU.apply();
+ SmallVector<uint64_t> Counts;
+ BitVector Valid;
+ uint64_t Entry;
+ ASSERT_TRUE(extractMappedBlockWaveCounts(F, Counts, Valid, Entry));
+ EXPECT_EQ(Entry, 100u);
+ EXPECT_EQ(Counts, (SmallVector<uint64_t>{100, 0, 60, 0, 0, 100}));
+ EXPECT_EQ(Valid, makeBitVector(6, {0, 1, 2, 5}));
+
+ BlockWaveCountUpdater Remove(F);
+ ASSERT_TRUE(
+ TryToSimplifyUncondBranchFromEmptyBlock(SplitLeft, nullptr, &Remove));
+ ASSERT_TRUE(
+ TryToSimplifyUncondBranchFromEmptyBlock(SplitRight, nullptr, &Remove));
+ Remove.apply();
+ ASSERT_TRUE(extractMappedBlockWaveCounts(F, Counts, Valid, Entry));
+ EXPECT_EQ(Counts, (SmallVector<uint64_t>{100, 0, 60, 100}));
+ EXPECT_EQ(Valid, makeBitVector(4, {0, 1, 2, 3}));
+ EXPECT_FALSE(verifyModule(*M, &errs()));
+}
+
+TEST_F(WaveProfileTest, BatchMergesKeepPredecessorIdentity) {
+ SMDiagnostic Error;
+ auto Chain = parseAssemblyString(R"(
+ define void @chain() {
+ entry:
+ br label %first
+ first:
+ br label %second
+ second:
+ ret void
+ })",
+ Error, Context);
+ ASSERT_TRUE(Chain);
+ Function &F = *Chain->getFunction("chain");
+ setBlockWaveCounts(F, {10, 20, 30});
+ BasicBlock *First = F.getEntryBlock().getNextNode();
+ BasicBlock *Second = First->getNextNode();
+ BlockWaveCountUpdater WPU(F);
+ ASSERT_TRUE(MergeBlockIntoPredecessor(First, nullptr, nullptr, nullptr,
+ nullptr, false, nullptr, &WPU));
+ ASSERT_TRUE(MergeBlockIntoPredecessor(Second, nullptr, nullptr, nullptr,
+ nullptr, false, nullptr, &WPU));
+ WPU.apply();
+ SmallVector<uint64_t> Counts;
+ BitVector Valid;
+ uint64_t Entry;
+ ASSERT_TRUE(extractMappedBlockWaveCounts(F, Counts, Valid, Entry));
+ EXPECT_EQ(Counts, (SmallVector<uint64_t>{10}));
+ EXPECT_EQ(Entry, 10u);
+ EXPECT_EQ(Valid, makeBitVector(1, {0}));
+ EXPECT_FALSE(verifyModule(*Chain, &errs()));
+}
+
+TEST_F(WaveProfileTest, BatchDoesNotHideEarlierUnhandledRedirect) {
+ Function &F = *M->getFunction("diamond");
+ setBlockWaveCounts(F, {100, 80, 60, 100});
+ BasicBlock *Left = F.getEntryBlock().getNextNode();
+ BasicBlock *Right = Left->getNextNode();
+ BlockWaveCountUpdater WPU(F);
+ ASSERT_TRUE(WPU.hasProfile());
+ Left->getTerminator()->setSuccessor(0, Right);
+ SplitBlockPredecessors(Right, {Left}, ".split",
+ static_cast<DominatorTree *>(nullptr), nullptr,
+ nullptr, false, &WPU);
+ WPU.apply();
+ SmallVector<uint64_t> Counts;
+ BitVector Valid;
+ uint64_t Entry;
+ ASSERT_TRUE(extractMappedBlockWaveCounts(F, Counts, Valid, Entry));
+ EXPECT_EQ(Entry, 100u);
+ EXPECT_EQ(Valid, makeBitVector(5, {0}));
+ EXPECT_FALSE(verifyModule(*M, &errs()));
+}
+
+TEST_F(WaveProfileTest, BatchRejectsLaterUnhandledConditionChange) {
+ Function &F = *M->getFunction("diamond");
+ setBlockWaveCounts(F, {100, 80, 60, 100});
+ BasicBlock *Left = F.getEntryBlock().getNextNode();
+ BasicBlock *Right = Left->getNextNode();
+ BasicBlock *Exit = Right->getNextNode();
+ BlockWaveCountUpdater WPU(F);
+ SplitBlockPredecessors(Exit, {Left}, ".split",
+ static_cast<DominatorTree *>(nullptr), nullptr,
+ nullptr, false, &WPU);
+ cast<CondBrInst>(F.getEntryBlock().getTerminator())
+ ->setCondition(ConstantInt::getTrue(Context));
+ WPU.apply();
+ SmallVector<uint64_t> Counts;
+ BitVector Valid;
+ uint64_t Entry;
+ ASSERT_TRUE(extractMappedBlockWaveCounts(F, Counts, Valid, Entry));
+ EXPECT_EQ(Entry, 100u);
+ EXPECT_EQ(Valid, makeBitVector(5, {4}));
+ EXPECT_FALSE(verifyModule(*M, &errs()));
+}
+
+TEST_F(WaveProfileTest, BatchPreservesInitialUnavailableCounts) {
+ Function &F = *M->getFunction("diamond");
+ setBlockWaveCounts(F, {100, 0, 60, 100}, makeBitVector(4, {0, 2, 3}));
+ BasicBlock *Left = F.getEntryBlock().getNextNode();
+ BasicBlock *Right = Left->getNextNode();
+ BasicBlock *Exit = Right->getNextNode();
+ BlockWaveCountUpdater WPU(F);
+ SplitBlockPredecessors(Exit, {Left}, ".split",
+ static_cast<DominatorTree *>(nullptr), nullptr,
+ nullptr, false, &WPU);
+ WPU.apply();
+ SmallVector<uint64_t> Counts;
+ BitVector Valid;
+ uint64_t Entry;
+ ASSERT_TRUE(extractMappedBlockWaveCounts(F, Counts, Valid, Entry));
+ EXPECT_EQ(Counts, (SmallVector<uint64_t>{100, 0, 60, 0, 100}));
+ EXPECT_EQ(Valid, makeBitVector(5, {0, 2, 4}));
+}
+
+TEST_F(WaveProfileTest, BatchDoesNotResurrectDuplicateIdentity) {
+ Function &F = *M->getFunction("diamond");
+ setBlockWaveCounts(F, {100, 80, 60, 100});
+ BasicBlock *Left = F.getEntryBlock().getNextNode();
+ BasicBlock *Right = Left->getNextNode();
+ Right->getTerminator()->setMetadata(
+ LLVMContext::MD_wave_profile_block,
+ Left->getTerminator()->getMetadata(LLVMContext::MD_wave_profile_block));
+ BlockWaveCountUpdater WPU(F);
+ ASSERT_TRUE(TryToSimplifyUncondBranchFromEmptyBlock(Right, nullptr, &WPU));
+ WPU.apply();
+ SmallVector<uint64_t> Counts;
+ BitVector Valid;
+ uint64_t Entry;
+ ASSERT_TRUE(extractMappedBlockWaveCounts(F, Counts, Valid, Entry));
+ EXPECT_EQ(Entry, 100u);
+ EXPECT_TRUE(Valid.none());
+}
+
+TEST_F(WaveProfileTest, BatchRetainsNormalizationWithoutOriginalEntry) {
+ Function &F = *M->getFunction("diamond");
+ setBlockWaveCounts(F, {100, 80, 60, 100});
+ F.getEntryBlock().eraseFromParent();
+ BasicBlock *Left = &F.getEntryBlock();
+ BasicBlock *Exit = &F.back();
+ BlockWaveCountUpdater WPU(F);
+ SplitBlockPredecessors(Exit, {Left}, ".split",
+ static_cast<DominatorTree *>(nullptr), nullptr,
+ nullptr, false, &WPU);
+ WPU.apply();
+ SmallVector<uint64_t> Counts;
+ BitVector Valid;
+ uint64_t Entry;
+ ASSERT_TRUE(extractMappedBlockWaveCounts(F, Counts, Valid, Entry));
+ EXPECT_EQ(Entry, 100u);
+ EXPECT_EQ(Counts, (SmallVector<uint64_t>{80, 60, 0, 100}));
+ EXPECT_EQ(Valid, makeBitVector(4, {0, 1, 3}));
+ EXPECT_FALSE(verifyModule(*M, &errs()));
+}
+
+TEST_F(WaveProfileTest, BatchLeavesMissingOrInvalidProfileAlone) {
+ Function &F = *M->getFunction("diamond");
+ for (bool Malformed : {false, true}) {
+ if (Malformed)
+ F.setMetadata(LLVMContext::MD_wave_profile, MDNode::get(Context, {}));
+ MDNode *Original = F.getMetadata(LLVMContext::MD_wave_profile);
+ BasicBlock *Left = F.getEntryBlock().getNextNode();
+ BlockWaveCountUpdater WPU(F);
+ SplitBlockPredecessors(&F.back(), {Left}, ".split",
+ static_cast<DominatorTree *>(nullptr), nullptr,
+ nullptr, false, &WPU);
+ WPU.apply();
+ EXPECT_EQ(F.getMetadata(LLVMContext::MD_wave_profile), Original);
+ }
+}
+
+TEST_F(WaveProfileTest, BatchRespectsNewProfileAfterSplitting) {
+ Function &F = *M->getFunction("diamond");
+ setBlockWaveCounts(F, {100, 80, 60, 100});
+ BasicBlock *Left = F.getEntryBlock().getNextNode();
+ BlockWaveCountUpdater WPU(F);
+ SplitBlockPredecessors(&F.back(), {Left}, ".split",
+ static_cast<DominatorTree *>(nullptr), nullptr,
+ nullptr, false, &WPU);
+ setBlockWaveCounts(F, {200, 160, 120, 160, 200});
+ MDNode *Replacement = F.getMetadata(LLVMContext::MD_wave_profile);
+ WPU.apply();
+ EXPECT_EQ(F.getMetadata(LLVMContext::MD_wave_profile), Replacement);
+ SmallVector<uint64_t> Counts;
+ ASSERT_TRUE(extractBlockWaveCounts(F, Counts));
+ EXPECT_EQ(Counts, (SmallVector<uint64_t>{200, 160, 120, 160, 200}));
+}
+
+TEST_F(WaveProfileTest, BatchDoesNotCertifyUnrecordedClone) {
+ Function &F = *M->getFunction("diamond");
+ setBlockWaveCounts(F, {100, 80, 60, 100});
+ BasicBlock *Left = F.getEntryBlock().getNextNode();
+ BlockWaveCountUpdater WPU(F);
+ SplitBlockPredecessors(&F.back(), {Left}, ".split",
+ static_cast<DominatorTree *>(nullptr), nullptr,
+ nullptr, false, &WPU);
+ ValueToValueMapTy VMap;
+ CloneBasicBlock(Left, VMap, ".clone", &F);
+ WPU.apply();
+ SmallVector<uint64_t> Counts;
+ BitVector Valid;
+ uint64_t Entry;
+ EXPECT_FALSE(extractMappedBlockWaveCounts(F, Counts, Valid, Entry));
+ EXPECT_FALSE(verifyModule(*M, &errs()));
+}
+
TEST_F(WaveProfileTest, RoundTripAndReplacement) {
Function &F = *M->getFunction("diamond");
F.setEntryCount(6400);
diff --git a/llvm/unittests/Transforms/Utils/CloningTest.cpp b/llvm/unittests/Transforms/Utils/CloningTest.cpp
index bc5ffd27a3af8a..279bf088a59b69 100644
--- a/llvm/unittests/Transforms/Utils/CloningTest.cpp
+++ b/llvm/unittests/Transforms/Utils/CloningTest.cpp
@@ -7,6 +7,7 @@
//===----------------------------------------------------------------------===//
#include "llvm/Transforms/Utils/Cloning.h"
+#include "llvm/ADT/BitVector.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/SmallPtrSet.h"
#include "llvm/Analysis/AliasAnalysis.h"
@@ -27,6 +28,7 @@
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/LLVMContext.h"
#include "llvm/IR/Module.h"
+#include "llvm/IR/ProfDataUtils.h"
#include "llvm/IR/Verifier.h"
#include "llvm/Support/SourceMgr.h"
#include "llvm/Transforms/Utils/ValueMapper.h"
@@ -388,6 +390,283 @@ static std::unique_ptr<Module> parseIR(LLVMContext &C, const char *IR) {
return Mod;
}
+// A profiled device helper can survive profile loading and be inlined into
+// several callsites later. Its aggregate wave visits do not describe any one
+// inlined copy, especially when waves visit both sides of its diamond.
+static const char *WaveInliningIR = R"(
+ define void @helper(i1 %c, ptr %p) {
+ entry:
+ br i1 %c, label %left, label %right
+ left:
+ store volatile i32 1, ptr %p
+ br label %exit
+ right:
+ store volatile i32 2, ptr %p
+ br label %exit
+ exit:
+ ret void
+ }
+ define void @caller(i1 %gate, i1 %c, ptr %p) {
+ entry:
+ br i1 %gate, label %calls, label %zero
+ calls:
+ call void @helper(i1 %c, ptr %p)
+ call void @helper(i1 %c, ptr %p)
+ br label %exit
+ zero:
+ br label %exit
+ exit:
+ ret void
+ }
+ define void @other(i1 %c, ptr %p) {
+ entry:
+ call void @helper(i1 %c, ptr %p)
+ ret void
+ }
+)";
+
+TEST(InlineFunction, CallerWaveCounts) {
+ for (bool Partial : {false, true}) {
+ LLVMContext Context;
+ auto M = parseIR(Context, WaveInliningIR);
+ ASSERT_TRUE(M);
+ Function &Caller = *M->getFunction("caller");
+ Function &Helper = *M->getFunction("helper");
+ Function &Other = *M->getFunction("other");
+ SmallVector<uint64_t> OriginalCounts{100, 60, 0, 100};
+ BitVector OriginalKnown(4, true);
+ if (Partial)
+ OriginalKnown.reset(2);
+ setBlockWaveCounts(Caller, OriginalCounts, OriginalKnown);
+ setBlockWaveCounts(Helper, {200, 180, 70, 200});
+ setBlockWaveCounts(Other, {5});
+ SmallVector<BasicBlock *> Originals;
+ for (BasicBlock &BB : Caller)
+ Originals.push_back(&BB);
+ SmallVector<CallBase *> Calls;
+ for (Instruction &I : instructions(Caller))
+ if (auto *CB = dyn_cast<CallBase>(&I))
+ Calls.push_back(CB);
+
+ for (CallBase *CB : Calls) {
+ InlineFunctionInfo IFI;
+ ASSERT_TRUE(InlineFunction(*CB, IFI).isSuccess());
+ ASSERT_FALSE(verifyModule(*M, &errs()));
+ SmallVector<uint64_t> Counts;
+ BitVector Known;
+ uint64_t Entry;
+ ASSERT_TRUE(extractMappedBlockWaveCounts(Caller, Counts, Known, Entry));
+ EXPECT_EQ(Entry, 100u);
+ for (auto [Index, BB] : enumerate(Caller)) {
+ auto It = llvm::find(Originals, &BB);
+ if (It == Originals.end()) {
+ EXPECT_FALSE(Known[Index]);
+ continue;
+ }
+ unsigned Original = It - Originals.begin();
+ // The independent zero block keeps its event. The call block and its
+ // predecessors/successors are conservatively unmeasured after
+ // splitting.
+ EXPECT_EQ(Known[Index], Original == 2 && OriginalKnown[Original]);
+ if (Known[Index])
+ EXPECT_EQ(Counts[Index], OriginalCounts[Original]);
+ }
+ }
+
+ InlineFunctionInfo IFI;
+ auto &Call = cast<CallBase>(Other.front().front());
+ ASSERT_TRUE(InlineFunction(Call, IFI).isSuccess());
+ SmallVector<uint64_t> Counts;
+ BitVector Known;
+ uint64_t Entry;
+ ASSERT_TRUE(extractMappedBlockWaveCounts(Other, Counts, Known, Entry));
+ EXPECT_EQ(Entry, 5u);
+ EXPECT_EQ(Known.count(), 0u);
+ ASSERT_TRUE(extractBlockWaveCounts(Helper, Counts));
+ EXPECT_EQ(Counts, (SmallVector<uint64_t>{200, 180, 70, 200}));
+ }
+}
+
+TEST(InlineFunction, UnprofiledCallerDoesNotInheritWaveCounts) {
+ LLVMContext Context;
+ auto M = parseIR(Context, WaveInliningIR);
+ ASSERT_TRUE(M);
+ Function &Helper = *M->getFunction("helper");
+ Function &Caller = *M->getFunction("other");
+ setBlockWaveCounts(Helper, {200, 180, 70, 200});
+ InlineFunctionInfo IFI;
+ ASSERT_TRUE(
+ InlineFunction(cast<CallBase>(Caller.front().front()), IFI).isSuccess());
+ EXPECT_FALSE(Caller.getMetadata(LLVMContext::MD_wave_profile));
+ for (BasicBlock &BB : Caller)
+ EXPECT_FALSE(
+ BB.getTerminator()->getMetadata(LLVMContext::MD_wave_profile_block));
+}
+
+TEST(InlineFunction, SingleBlockWaveCounts) {
+ LLVMContext Context;
+ auto M = parseIR(Context, R"(
+ define void @helper(ptr %p) {
+ store volatile i32 1, ptr %p
+ ret void
+ }
+ define void @caller(ptr %p) {
+ call void @helper(ptr %p)
+ ret void
+ }
+ )");
+ ASSERT_TRUE(M);
+ Function &Caller = *M->getFunction("caller");
+ setBlockWaveCounts(Caller, {10});
+ setBlockWaveCounts(*M->getFunction("helper"), {200});
+ InlineFunctionInfo IFI;
+ ASSERT_TRUE(
+ InlineFunction(cast<CallBase>(Caller.front().front()), IFI).isSuccess());
+ SmallVector<uint64_t> Counts;
+ ASSERT_TRUE(extractBlockWaveCounts(Caller, Counts));
+ EXPECT_EQ(Counts, (SmallVector<uint64_t>{10}));
+}
+
+TEST(InlineFunction, InvokeWaveCounts) {
+ LLVMContext Context;
+ auto M = parseIR(Context, R"(
+ declare i32 @personality(...)
+ declare void @may_throw()
+ define void @helper(i1 %c) {
+ entry:
+ br i1 %c, label %left, label %exit
+ left:
+ call void @may_throw()
+ br label %exit
+ exit:
+ ret void
+ }
+ define void @caller(i1 %c) personality ptr @personality {
+ entry:
+ invoke void @helper(i1 %c) to label %normal unwind label %unwind
+ normal:
+ ret void
+ unwind:
+ %lp = landingpad {ptr, i32} cleanup
+ ret void
+ }
+ )");
+ ASSERT_TRUE(M);
+ Function &Caller = *M->getFunction("caller");
+ setBlockWaveCounts(Caller, {100, 90, 20});
+ setBlockWaveCounts(*M->getFunction("helper"), {200, 140, 200});
+ InlineFunctionInfo IFI;
+ ASSERT_TRUE(
+ InlineFunction(cast<CallBase>(Caller.front().front()), IFI).isSuccess());
+ ASSERT_FALSE(verifyModule(*M, &errs()));
+ SmallVector<uint64_t> Counts;
+ BitVector Known;
+ uint64_t Entry;
+ ASSERT_TRUE(extractMappedBlockWaveCounts(Caller, Counts, Known, Entry));
+ EXPECT_EQ(Entry, 100u);
+ EXPECT_EQ(Known.count(), 0u);
+}
+
+TEST(InlineFunction, RecursiveWaveCounts) {
+ LLVMContext Context;
+ auto M = parseIR(Context, R"(
+ define void @recurse(i1 %c, ptr %p) {
+ entry:
+ br i1 %c, label %call, label %exit
+ call:
+ call void @recurse(i1 false, ptr %p)
+ br label %exit
+ exit:
+ store volatile i32 1, ptr %p
+ ret void
+ }
+ )");
+ ASSERT_TRUE(M);
+ Function &Caller = *M->getFunction("recurse");
+ setBlockWaveCounts(Caller, {100, 40, 100});
+ InlineFunctionInfo IFI;
+ auto &Call = cast<CallBase>(std::next(Caller.begin())->front());
+ ASSERT_TRUE(InlineFunction(Call, IFI).isSuccess());
+ ASSERT_FALSE(verifyModule(*M, &errs()));
+ EXPECT_FALSE(Caller.getMetadata(LLVMContext::MD_wave_profile));
+ for (BasicBlock &BB : Caller)
+ EXPECT_FALSE(
+ BB.getTerminator()->getMetadata(LLVMContext::MD_wave_profile_block));
+}
+
+TEST(InlineFunction, LoopCallerWaveCounts) {
+ LLVMContext Context;
+ auto M = parseIR(Context, R"(
+ define void @helper(i1 %c, ptr %p) {
+ entry:
+ br i1 %c, label %left, label %exit
+ left:
+ store volatile i32 1, ptr %p
+ br label %exit
+ exit:
+ ret void
+ }
+ define void @caller(i1 %c, ptr %p, i32 %n) {
+ entry:
+ br label %preheader
+ preheader:
+ br label %loop
+ loop:
+ %i = phi i32 [0, %preheader], [%next, %loop]
+ call void @helper(i1 %c, ptr %p)
+ %next = add i32 %i, 1
+ %again = icmp ult i32 %next, %n
+ br i1 %again, label %loop, label %exit
+ exit:
+ ret void
+ }
+ )");
+ ASSERT_TRUE(M);
+ Function &Caller = *M->getFunction("caller");
+ setBlockWaveCounts(Caller, {10, 10, 100, 10});
+ setBlockWaveCounts(*M->getFunction("helper"), {200, 170, 200});
+ SmallVector<BasicBlock *> Originals;
+ for (BasicBlock &BB : Caller)
+ Originals.push_back(&BB);
+ auto &Call = cast<CallBase>(*std::next(Originals[2]->begin()));
+ InlineFunctionInfo IFI;
+ ASSERT_TRUE(InlineFunction(Call, IFI).isSuccess());
+ ASSERT_FALSE(verifyModule(*M, &errs()));
+ SmallVector<uint64_t> Counts;
+ BitVector Known;
+ uint64_t Entry;
+ ASSERT_TRUE(extractMappedBlockWaveCounts(Caller, Counts, Known, Entry));
+ EXPECT_EQ(Entry, 10u);
+ EXPECT_EQ(Known.count(), 1u);
+ ASSERT_TRUE(Known[0]);
+ EXPECT_EQ(Counts[0], 10u);
+}
+
+TEST(InlineFunction, InvalidCallerWaveCountsStayInvalid) {
+ LLVMContext Context;
+ auto M = parseIR(Context, WaveInliningIR);
+ ASSERT_TRUE(M);
+ Function &Caller = *M->getFunction("caller");
+ Function &Helper = *M->getFunction("helper");
+ setBlockWaveCounts(Caller, {100, 60, 0, 100});
+ setBlockWaveCounts(Helper, {200, 180, 70, 200});
+ // Simulate a pre-existing foreign association in a block outside the call.
+ std::next(Caller.begin(), 2)
+ ->getTerminator()
+ ->setMetadata(LLVMContext::MD_wave_profile_block,
+ Helper.front().getTerminator()->getMetadata(
+ LLVMContext::MD_wave_profile_block));
+ SmallVector<uint64_t> Counts;
+ BitVector Known;
+ uint64_t Entry;
+ ASSERT_FALSE(extractMappedBlockWaveCounts(Caller, Counts, Known, Entry));
+ InlineFunctionInfo IFI;
+ auto &Call = cast<CallBase>(std::next(Caller.begin())->front());
+ ASSERT_TRUE(InlineFunction(Call, IFI).isSuccess());
+ ASSERT_FALSE(verifyModule(*M, &errs()));
+ EXPECT_FALSE(extractMappedBlockWaveCounts(Caller, Counts, Known, Entry));
+}
+
TEST(CloneLoop, CloneLoopNest) {
// Parse the module.
LLVMContext Context;
diff --git a/llvm/unittests/Transforms/Utils/LoopRotationUtilsTest.cpp b/llvm/unittests/Transforms/Utils/LoopRotationUtilsTest.cpp
index 70cdf0aba356cd..05ebd81140f5bb 100644
--- a/llvm/unittests/Transforms/Utils/LoopRotationUtilsTest.cpp
+++ b/llvm/unittests/Transforms/Utils/LoopRotationUtilsTest.cpp
@@ -17,6 +17,7 @@
#include "llvm/IR/Dominators.h"
#include "llvm/IR/LLVMContext.h"
#include "llvm/IR/Module.h"
+#include "llvm/IR/ProfDataUtils.h"
#include "llvm/Support/SourceMgr.h"
#include "gtest/gtest.h"
@@ -165,3 +166,70 @@ deopt.exit:
/// so we do change the IR.
EXPECT_TRUE(ret);
}
+
+TEST(LoopRotate, PreserveMappedWaveCounts) {
+ for (uint64_t HeaderCount : {30u, 31u}) {
+ SCOPED_TRACE(HeaderCount);
+ LLVMContext C;
+ std::unique_ptr<Module> M = parseIR(C, R"(
+define void @test(ptr %out, i32 %start) {
+entry:
+ br label %header
+header:
+ %i = phi i32 [ %start, %entry ], [ %inc, %body ]
+ %cond = icmp ult i32 %i, 4
+ br i1 %cond, label %body, label %exit
+exit:
+ ret void
+body:
+ store volatile i32 %i, ptr %out
+ %inc = add i32 %i, 1
+ br label %header
+}
+)");
+ ASSERT_TRUE(M);
+ Function &F = *M->getFunction("test");
+ if (HeaderCount == 30)
+ F.getArg(1)->replaceAllUsesWith(ConstantInt::get(Type::getInt32Ty(C), 0));
+ setBlockWaveCounts(F, {6, HeaderCount, 6, 24});
+
+ // An unused historical identity makes this a valid partial profile.
+ MDNode *Table = F.getMetadata(LLVMContext::MD_wave_profile);
+ SmallVector<Metadata *> Ops(Table->op_begin(), Table->op_end());
+ Ops.push_back(
+ ConstantAsMetadata::get(ConstantInt::get(Type::getInt64Ty(C), 0)));
+ F.setMetadata(LLVMContext::MD_wave_profile, MDNode::getDistinct(C, Ops));
+
+ SmallVector<uint64_t> Counts;
+ BitVector Valid;
+ EXPECT_FALSE(extractBlockWaveCounts(F, Counts, &Valid));
+
+ DominatorTree DT(F);
+ LoopInfo LI(DT);
+ AssumptionCache AC(F);
+ TargetTransformInfo TTI(M->getDataLayout());
+ TargetLibraryInfoImpl TLII(M->getTargetTriple());
+ TargetLibraryInfo TLI(TLII);
+ ScalarEvolution SE(F, TLI, AC, DT, LI);
+ SimplifyQuery SQ(M->getDataLayout());
+ ASSERT_TRUE(LoopRotation(*LI.begin(), &LI, &TTI, &AC, &DT, &SE, nullptr, SQ,
+ true, -1, false));
+
+ uint64_t EntryCount = 0;
+ ASSERT_TRUE(extractMappedBlockWaveCounts(F, Counts, Valid, EntryCount));
+ EXPECT_EQ(EntryCount, 6u);
+ EXPECT_GE(F.getMetadata(LLVMContext::MD_wave_profile)->getNumOperands(),
+ 7u);
+ for (auto [Index, BB] : enumerate(F)) {
+ if (BB.getName() == "entry" || BB.getName() == "exit") {
+ EXPECT_TRUE(Valid[Index]);
+ EXPECT_EQ(Counts[Index], 6u);
+ } else if (BB.getName() == "body") {
+ EXPECT_TRUE(Valid[Index]);
+ EXPECT_EQ(Counts[Index], 24u);
+ } else {
+ EXPECT_FALSE(Valid[Index]);
+ }
+ }
+ }
+}
More information about the llvm-branch-commits
mailing list