[llvm] 7b08b43 - [SPIR-V] add convergence region analysis (#78456)
via llvm-commits
llvm-commits at lists.llvm.org
Fri Feb 2 09:22:17 PST 2024
Author: Nathan Gauër
Date: 2024-02-02T18:22:14+01:00
New Revision: 7b08b4360b488b35428c97132b3f9d2a777bd770
URL: https://github.com/llvm/llvm-project/commit/7b08b4360b488b35428c97132b3f9d2a777bd770
DIFF: https://github.com/llvm/llvm-project/commit/7b08b4360b488b35428c97132b3f9d2a777bd770.diff
LOG: [SPIR-V] add convergence region analysis (#78456)
This new analysis returns a hierarchical view of the convergence regions
in the given function.
This will allow our passes to query which basic block belongs to which
convergence region, and structurize the code in consequence.
Definition
----------
A convergence region is a CFG with:
- a single entry node.
- one or multiple exit nodes (different from LLVM's regions).
- one back-edge
- zero or more subregions.
Excluding sub-regions nodes, the nodes of a region can only reference a
single convergence token. A subregion uses a different convergence
token.
Algorithm
---------
This algorithm assumes all loops are in the Simplify form.
Create an initial convergence region for the whole function.
- the convergence token is the function entry token.
- the entry is the function entrypoint.
- Exits are all the basic blocks terminating with a return instruction.
Take the function CFG, and process it in DAG order (ignoring
back-edges). If a basic block is a loop header:
- Create a new region.
- The parent region is the parent's loop region if any, otherwise, the
top level region.
- The region blocks are all the blocks belonging to this loop.
- For each loop exit: - visit the rest of the CFG in DAG order (ignore
back-edges). - if the region's convergence token is found, add all the
blocks dominated by the exit from which the token is reachable to the
region.
- continue the algorithm with the loop headers successors.
Added:
llvm/lib/Target/SPIRV/Analysis/CMakeLists.txt
llvm/lib/Target/SPIRV/Analysis/SPIRVConvergenceRegionAnalysis.cpp
llvm/lib/Target/SPIRV/Analysis/SPIRVConvergenceRegionAnalysis.h
llvm/unittests/Target/SPIRV/CMakeLists.txt
llvm/unittests/Target/SPIRV/SPIRVConvergenceRegionAnalysisTests.cpp
Modified:
llvm/lib/Target/SPIRV/CMakeLists.txt
llvm/lib/Target/SPIRV/SPIRV.h
llvm/lib/Target/SPIRV/SPIRVTargetMachine.cpp
Removed:
################################################################################
diff --git a/llvm/lib/Target/SPIRV/Analysis/CMakeLists.txt b/llvm/lib/Target/SPIRV/Analysis/CMakeLists.txt
new file mode 100644
index 0000000000000..132d8ff838353
--- /dev/null
+++ b/llvm/lib/Target/SPIRV/Analysis/CMakeLists.txt
@@ -0,0 +1,10 @@
+add_llvm_component_library(LLVMSPIRVAnalysis
+ SPIRVConvergenceRegionAnalysis.cpp
+
+ LINK_COMPONENTS
+ Core
+ Support
+
+ ADD_TO_COMPONENT
+ SPIRV
+ )
diff --git a/llvm/lib/Target/SPIRV/Analysis/SPIRVConvergenceRegionAnalysis.cpp b/llvm/lib/Target/SPIRV/Analysis/SPIRVConvergenceRegionAnalysis.cpp
new file mode 100644
index 0000000000000..7f5f7d0b1e4dc
--- /dev/null
+++ b/llvm/lib/Target/SPIRV/Analysis/SPIRVConvergenceRegionAnalysis.cpp
@@ -0,0 +1,350 @@
+//===- ConvergenceRegionAnalysis.h -----------------------------*- C++ -*--===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// The analysis determines the convergence region for each basic block of
+// the module, and provides a tree-like structure describing the region
+// hierarchy.
+//
+//===----------------------------------------------------------------------===//
+
+#include "SPIRVConvergenceRegionAnalysis.h"
+#include "llvm/Analysis/LoopInfo.h"
+#include "llvm/IR/Dominators.h"
+#include "llvm/IR/IntrinsicInst.h"
+#include "llvm/InitializePasses.h"
+#include "llvm/Transforms/Utils/LoopSimplify.h"
+#include <optional>
+#include <queue>
+
+#define DEBUG_TYPE "spirv-convergence-region-analysis"
+
+using namespace llvm;
+
+namespace llvm {
+void initializeSPIRVConvergenceRegionAnalysisWrapperPassPass(PassRegistry &);
+} // namespace llvm
+
+INITIALIZE_PASS_BEGIN(SPIRVConvergenceRegionAnalysisWrapperPass,
+ "convergence-region",
+ "SPIRV convergence regions analysis", true, true)
+INITIALIZE_PASS_DEPENDENCY(LoopSimplify)
+INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
+INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass)
+INITIALIZE_PASS_END(SPIRVConvergenceRegionAnalysisWrapperPass,
+ "convergence-region", "SPIRV convergence regions analysis",
+ true, true)
+
+namespace llvm {
+namespace SPIRV {
+namespace {
+
+template <typename BasicBlockType, typename IntrinsicInstType>
+std::optional<IntrinsicInstType *>
+getConvergenceTokenInternal(BasicBlockType *BB) {
+ static_assert(std::is_const_v<IntrinsicInstType> ==
+ std::is_const_v<BasicBlockType>,
+ "Constness must match between input and output.");
+ static_assert(std::is_same_v<BasicBlock, std::remove_const_t<BasicBlockType>>,
+ "Input must be a basic block.");
+ static_assert(
+ std::is_same_v<IntrinsicInst, std::remove_const_t<IntrinsicInstType>>,
+ "Output type must be an intrinsic instruction.");
+
+ for (auto &I : *BB) {
+ if (auto *II = dyn_cast<IntrinsicInst>(&I)) {
+ switch (II->getIntrinsicID()) {
+ case Intrinsic::experimental_convergence_entry:
+ case Intrinsic::experimental_convergence_loop:
+ return II;
+ case Intrinsic::experimental_convergence_anchor: {
+ auto Bundle = II->getOperandBundle(LLVMContext::OB_convergencectrl);
+ assert(Bundle->Inputs.size() == 1 &&
+ Bundle->Inputs[0]->getType()->isTokenTy());
+ auto TII = dyn_cast<IntrinsicInst>(Bundle->Inputs[0].get());
+ assert(TII != nullptr);
+ return TII;
+ }
+ }
+ }
+
+ if (auto *CI = dyn_cast<CallInst>(&I)) {
+ auto OB = CI->getOperandBundle(LLVMContext::OB_convergencectrl);
+ if (!OB.has_value())
+ continue;
+ return dyn_cast<IntrinsicInst>(OB.value().Inputs[0]);
+ }
+ }
+
+ return std::nullopt;
+}
+
+// Given a ConvergenceRegion tree with |Start| as its root, finds the smallest
+// region |Entry| belongs to. If |Entry| does not belong to the region defined
+// by |Start|, this function returns |nullptr|.
+ConvergenceRegion *findParentRegion(ConvergenceRegion *Start,
+ BasicBlock *Entry) {
+ ConvergenceRegion *Candidate = nullptr;
+ ConvergenceRegion *NextCandidate = Start;
+
+ while (Candidate != NextCandidate && NextCandidate != nullptr) {
+ Candidate = NextCandidate;
+ NextCandidate = nullptr;
+
+ // End of the search, we can return.
+ if (Candidate->Children.size() == 0)
+ return Candidate;
+
+ for (auto *Child : Candidate->Children) {
+ if (Child->Blocks.count(Entry) != 0) {
+ NextCandidate = Child;
+ break;
+ }
+ }
+ }
+
+ return Candidate;
+}
+
+} // anonymous namespace
+
+std::optional<IntrinsicInst *> getConvergenceToken(BasicBlock *BB) {
+ return getConvergenceTokenInternal<BasicBlock, IntrinsicInst>(BB);
+}
+
+std::optional<const IntrinsicInst *> getConvergenceToken(const BasicBlock *BB) {
+ return getConvergenceTokenInternal<const BasicBlock, const IntrinsicInst>(BB);
+}
+
+ConvergenceRegion::ConvergenceRegion(DominatorTree &DT, LoopInfo &LI,
+ Function &F)
+ : DT(DT), LI(LI), Parent(nullptr) {
+ Entry = &F.getEntryBlock();
+ ConvergenceToken = getConvergenceToken(Entry);
+ for (auto &B : F) {
+ Blocks.insert(&B);
+ if (isa<ReturnInst>(B.getTerminator()))
+ Exits.insert(&B);
+ }
+}
+
+ConvergenceRegion::ConvergenceRegion(
+ DominatorTree &DT, LoopInfo &LI,
+ std::optional<IntrinsicInst *> ConvergenceToken, BasicBlock *Entry,
+ SmallPtrSet<BasicBlock *, 8> &&Blocks, SmallPtrSet<BasicBlock *, 2> &&Exits)
+ : DT(DT), LI(LI), ConvergenceToken(ConvergenceToken), Entry(Entry),
+ Exits(std::move(Exits)), Blocks(std::move(Blocks)) {
+ for (auto *BB : this->Exits)
+ assert(this->Blocks.count(BB) != 0);
+ assert(this->Blocks.count(this->Entry) != 0);
+}
+
+void ConvergenceRegion::releaseMemory() {
+ // Parent memory is owned by the parent.
+ Parent = nullptr;
+ for (auto *Child : Children) {
+ Child->releaseMemory();
+ delete Child;
+ }
+ Children.resize(0);
+}
+
+void ConvergenceRegion::dump(const unsigned IndentSize) const {
+ const std::string Indent(IndentSize, '\t');
+ dbgs() << Indent << this << ": {\n";
+ dbgs() << Indent << " Parent: " << Parent << "\n";
+
+ if (ConvergenceToken.value_or(nullptr)) {
+ dbgs() << Indent
+ << " ConvergenceToken: " << ConvergenceToken.value()->getName()
+ << "\n";
+ }
+
+ if (Entry->getName() != "")
+ dbgs() << Indent << " Entry: " << Entry->getName() << "\n";
+ else
+ dbgs() << Indent << " Entry: " << Entry << "\n";
+
+ dbgs() << Indent << " Exits: { ";
+ for (const auto &Exit : Exits) {
+ if (Exit->getName() != "")
+ dbgs() << Exit->getName() << ", ";
+ else
+ dbgs() << Exit << ", ";
+ }
+ dbgs() << " }\n";
+
+ dbgs() << Indent << " Blocks: { ";
+ for (const auto &Block : Blocks) {
+ if (Block->getName() != "")
+ dbgs() << Block->getName() << ", ";
+ else
+ dbgs() << Block << ", ";
+ }
+ dbgs() << " }\n";
+
+ dbgs() << Indent << " Children: {\n";
+ for (const auto Child : Children)
+ Child->dump(IndentSize + 2);
+ dbgs() << Indent << " }\n";
+
+ dbgs() << Indent << "}\n";
+}
+
+class ConvergenceRegionAnalyzer {
+
+public:
+ ConvergenceRegionAnalyzer(Function &F, DominatorTree &DT, LoopInfo &LI)
+ : DT(DT), LI(LI), F(F) {}
+
+private:
+ bool isBackEdge(const BasicBlock *From, const BasicBlock *To) const {
+ assert(From != To && "From == To. This is awkward.");
+
+ // We only handle loop in the simplified form. This means:
+ // - a single back-edge, a single latch.
+ // - meaning the back-edge target can only be the loop header.
+ // - meaning the From can only be the loop latch.
+ if (!LI.isLoopHeader(To))
+ return false;
+
+ auto *L = LI.getLoopFor(To);
+ if (L->contains(From) && L->isLoopLatch(From))
+ return true;
+
+ return false;
+ }
+
+ std::unordered_set<BasicBlock *>
+ findPathsToMatch(LoopInfo &LI, BasicBlock *From,
+ std::function<bool(const BasicBlock *)> isMatch) const {
+ std::unordered_set<BasicBlock *> Output;
+
+ if (isMatch(From))
+ Output.insert(From);
+
+ auto *Terminator = From->getTerminator();
+ for (unsigned i = 0; i < Terminator->getNumSuccessors(); ++i) {
+ auto *To = Terminator->getSuccessor(i);
+ if (isBackEdge(From, To))
+ continue;
+
+ auto ChildSet = findPathsToMatch(LI, To, isMatch);
+ if (ChildSet.size() == 0)
+ continue;
+
+ Output.insert(ChildSet.begin(), ChildSet.end());
+ Output.insert(From);
+ if (LI.isLoopHeader(From)) {
+ auto *L = LI.getLoopFor(From);
+ for (auto *BB : L->getBlocks()) {
+ Output.insert(BB);
+ }
+ }
+ }
+
+ return Output;
+ }
+
+ SmallPtrSet<BasicBlock *, 2>
+ findExitNodes(const SmallPtrSetImpl<BasicBlock *> &RegionBlocks) {
+ SmallPtrSet<BasicBlock *, 2> Exits;
+
+ for (auto *B : RegionBlocks) {
+ auto *Terminator = B->getTerminator();
+ for (unsigned i = 0; i < Terminator->getNumSuccessors(); ++i) {
+ auto *Child = Terminator->getSuccessor(i);
+ if (RegionBlocks.count(Child) == 0)
+ Exits.insert(B);
+ }
+ }
+
+ return Exits;
+ }
+
+public:
+ ConvergenceRegionInfo analyze() {
+ ConvergenceRegion *TopLevelRegion = new ConvergenceRegion(DT, LI, F);
+ std::queue<Loop *> ToProcess;
+ for (auto *L : LI.getLoopsInPreorder())
+ ToProcess.push(L);
+
+ while (ToProcess.size() != 0) {
+ auto *L = ToProcess.front();
+ ToProcess.pop();
+ assert(L->isLoopSimplifyForm());
+
+ auto CT = getConvergenceToken(L->getHeader());
+ SmallPtrSet<BasicBlock *, 8> RegionBlocks(L->block_begin(),
+ L->block_end());
+ SmallVector<BasicBlock *> LoopExits;
+ L->getExitingBlocks(LoopExits);
+ if (CT.has_value()) {
+ for (auto *Exit : LoopExits) {
+ auto N = findPathsToMatch(LI, Exit, [&CT](const BasicBlock *block) {
+ auto Token = getConvergenceToken(block);
+ if (Token == std::nullopt)
+ return false;
+ return Token.value() == CT.value();
+ });
+ RegionBlocks.insert(N.begin(), N.end());
+ }
+ }
+
+ auto RegionExits = findExitNodes(RegionBlocks);
+ ConvergenceRegion *Region = new ConvergenceRegion(
+ DT, LI, CT, L->getHeader(), std::move(RegionBlocks),
+ std::move(RegionExits));
+ Region->Parent = findParentRegion(TopLevelRegion, Region->Entry);
+ assert(Region->Parent != nullptr && "This is impossible.");
+ Region->Parent->Children.push_back(Region);
+ }
+
+ return ConvergenceRegionInfo(TopLevelRegion);
+ }
+
+private:
+ DominatorTree &DT;
+ LoopInfo &LI;
+ Function &F;
+};
+
+ConvergenceRegionInfo getConvergenceRegions(Function &F, DominatorTree &DT,
+ LoopInfo &LI) {
+ ConvergenceRegionAnalyzer Analyzer(F, DT, LI);
+ return Analyzer.analyze();
+}
+
+} // namespace SPIRV
+
+char SPIRVConvergenceRegionAnalysisWrapperPass::ID = 0;
+
+SPIRVConvergenceRegionAnalysisWrapperPass::
+ SPIRVConvergenceRegionAnalysisWrapperPass()
+ : FunctionPass(ID) {}
+
+bool SPIRVConvergenceRegionAnalysisWrapperPass::runOnFunction(Function &F) {
+ DominatorTree &DT = getAnalysis<DominatorTreeWrapperPass>().getDomTree();
+ LoopInfo &LI = getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
+
+ CRI = SPIRV::getConvergenceRegions(F, DT, LI);
+ // Nothing was modified.
+ return false;
+}
+
+SPIRVConvergenceRegionAnalysis::Result
+SPIRVConvergenceRegionAnalysis::run(Function &F, FunctionAnalysisManager &AM) {
+ Result CRI;
+ auto &DT = AM.getResult<DominatorTreeAnalysis>(F);
+ auto &LI = AM.getResult<LoopAnalysis>(F);
+ CRI = SPIRV::getConvergenceRegions(F, DT, LI);
+ return CRI;
+}
+
+AnalysisKey SPIRVConvergenceRegionAnalysis::Key;
+
+} // namespace llvm
diff --git a/llvm/lib/Target/SPIRV/Analysis/SPIRVConvergenceRegionAnalysis.h b/llvm/lib/Target/SPIRV/Analysis/SPIRVConvergenceRegionAnalysis.h
new file mode 100644
index 0000000000000..f9e30e4effa1d
--- /dev/null
+++ b/llvm/lib/Target/SPIRV/Analysis/SPIRVConvergenceRegionAnalysis.h
@@ -0,0 +1,176 @@
+//===- SPIRVConvergenceRegionAnalysis.h ------------------------*- C++ -*--===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// The analysis determines the convergence region for each basic block of
+// the module, and provides a tree-like structure describing the region
+// hierarchy.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_TARGET_SPIRV_SPIRVCONVERGENCEREGIONANALYSIS_H
+#define LLVM_LIB_TARGET_SPIRV_SPIRVCONVERGENCEREGIONANALYSIS_H
+
+#include "llvm/ADT/SmallPtrSet.h"
+#include "llvm/Analysis/CFG.h"
+#include "llvm/Analysis/LoopInfo.h"
+#include "llvm/IR/Dominators.h"
+#include "llvm/IR/IntrinsicInst.h"
+#include <iostream>
+#include <optional>
+#include <unordered_set>
+
+namespace llvm {
+class SPIRVSubtarget;
+class MachineFunction;
+class MachineModuleInfo;
+
+namespace SPIRV {
+
+// Returns the first convergence intrinsic found in |BB|, |nullopt| otherwise.
+std::optional<IntrinsicInst *> getConvergenceToken(BasicBlock *BB);
+std::optional<const IntrinsicInst *> getConvergenceToken(const BasicBlock *BB);
+
+// Describes a hierarchy of convergence regions.
+// A convergence region defines a CFG for which the execution flow can diverge
+// starting from the entry block, but should reconverge back before the end of
+// the exit blocks.
+class ConvergenceRegion {
+ DominatorTree &DT;
+ LoopInfo &LI;
+
+public:
+ // The parent region of this region, if any.
+ ConvergenceRegion *Parent = nullptr;
+ // The sub-regions contained in this region, if any.
+ SmallVector<ConvergenceRegion *> Children = {};
+ // The convergence instruction linked to this region, if any.
+ std::optional<IntrinsicInst *> ConvergenceToken = std::nullopt;
+ // The only block with a predecessor outside of this region.
+ BasicBlock *Entry = nullptr;
+ // All the blocks with an edge leaving this convergence region.
+ SmallPtrSet<BasicBlock *, 2> Exits = {};
+ // All the blocks that belongs to this region, including its subregions'.
+ SmallPtrSet<BasicBlock *, 8> Blocks = {};
+
+ // Creates a single convergence region encapsulating the whole function |F|.
+ ConvergenceRegion(DominatorTree &DT, LoopInfo &LI, Function &F);
+
+ // Creates a single convergence region defined by entry and exits nodes, a
+ // list of blocks, and possibly a convergence token.
+ ConvergenceRegion(DominatorTree &DT, LoopInfo &LI,
+ std::optional<IntrinsicInst *> ConvergenceToken,
+ BasicBlock *Entry, SmallPtrSet<BasicBlock *, 8> &&Blocks,
+ SmallPtrSet<BasicBlock *, 2> &&Exits);
+
+ ConvergenceRegion(ConvergenceRegion &&CR)
+ : DT(CR.DT), LI(CR.LI), Parent(std::move(CR.Parent)),
+ Children(std::move(CR.Children)),
+ ConvergenceToken(std::move(CR.ConvergenceToken)),
+ Entry(std::move(CR.Entry)), Exits(std::move(CR.Exits)),
+ Blocks(std::move(CR.Blocks)) {}
+
+ ConvergenceRegion(const ConvergenceRegion &other) = delete;
+
+ // Returns true if the given basic block belongs to this region, or to one of
+ // its subregion.
+ bool contains(const BasicBlock *BB) const { return Blocks.count(BB) != 0; }
+
+ void releaseMemory();
+
+ // Write to the debug output this region's hierarchy.
+ // |IndentSize| defines the number of tabs to print before any new line.
+ void dump(const unsigned IndentSize = 0) const;
+};
+
+// Holds a ConvergenceRegion hierarchy.
+class ConvergenceRegionInfo {
+ // The convergence region this structure holds.
+ ConvergenceRegion *TopLevelRegion;
+
+public:
+ ConvergenceRegionInfo() : TopLevelRegion(nullptr) {}
+
+ // Creates a new ConvergenceRegionInfo. Ownership of the TopLevelRegion is
+ // passed to this object.
+ ConvergenceRegionInfo(ConvergenceRegion *TopLevelRegion)
+ : TopLevelRegion(TopLevelRegion) {}
+
+ ~ConvergenceRegionInfo() { releaseMemory(); }
+
+ ConvergenceRegionInfo(ConvergenceRegionInfo &&LHS)
+ : TopLevelRegion(LHS.TopLevelRegion) {
+ if (TopLevelRegion != LHS.TopLevelRegion) {
+ releaseMemory();
+ TopLevelRegion = LHS.TopLevelRegion;
+ }
+ LHS.TopLevelRegion = nullptr;
+ }
+
+ ConvergenceRegionInfo &operator=(ConvergenceRegionInfo &&LHS) {
+ if (TopLevelRegion != LHS.TopLevelRegion) {
+ releaseMemory();
+ TopLevelRegion = LHS.TopLevelRegion;
+ }
+ LHS.TopLevelRegion = nullptr;
+ return *this;
+ }
+
+ void releaseMemory() {
+ if (TopLevelRegion == nullptr)
+ return;
+
+ TopLevelRegion->releaseMemory();
+ delete TopLevelRegion;
+ TopLevelRegion = nullptr;
+ }
+
+ const ConvergenceRegion *getTopLevelRegion() const { return TopLevelRegion; }
+};
+
+} // namespace SPIRV
+
+// Wrapper around the function above to use it with the legacy pass manager.
+class SPIRVConvergenceRegionAnalysisWrapperPass : public FunctionPass {
+ SPIRV::ConvergenceRegionInfo CRI;
+
+public:
+ static char ID;
+
+ SPIRVConvergenceRegionAnalysisWrapperPass();
+
+ void getAnalysisUsage(AnalysisUsage &AU) const override {
+ AU.setPreservesAll();
+ AU.addRequired<LoopInfoWrapperPass>();
+ AU.addRequired<DominatorTreeWrapperPass>();
+ };
+
+ bool runOnFunction(Function &F) override;
+
+ SPIRV::ConvergenceRegionInfo &getRegionInfo() { return CRI; }
+ const SPIRV::ConvergenceRegionInfo &getRegionInfo() const { return CRI; }
+};
+
+// Wrapper around the function above to use it with the new pass manager.
+class SPIRVConvergenceRegionAnalysis
+ : public AnalysisInfoMixin<SPIRVConvergenceRegionAnalysis> {
+ friend AnalysisInfoMixin<SPIRVConvergenceRegionAnalysis>;
+ static AnalysisKey Key;
+
+public:
+ using Result = SPIRV::ConvergenceRegionInfo;
+
+ Result run(Function &F, FunctionAnalysisManager &AM);
+};
+
+namespace SPIRV {
+ConvergenceRegionInfo getConvergenceRegions(Function &F, DominatorTree &DT,
+ LoopInfo &LI);
+} // namespace SPIRV
+
+} // namespace llvm
+#endif // LLVM_LIB_TARGET_SPIRV_SPIRVCONVERGENCEREGIONANALYSIS_H
diff --git a/llvm/lib/Target/SPIRV/CMakeLists.txt b/llvm/lib/Target/SPIRV/CMakeLists.txt
index d9e24375dcb24..d1ada45d17a5b 100644
--- a/llvm/lib/Target/SPIRV/CMakeLists.txt
+++ b/llvm/lib/Target/SPIRV/CMakeLists.txt
@@ -45,6 +45,7 @@ add_llvm_target(SPIRVCodeGen
Core
Demangle
GlobalISel
+ SPIRVAnalysis
MC
SPIRVDesc
SPIRVInfo
@@ -60,3 +61,4 @@ add_llvm_target(SPIRVCodeGen
add_subdirectory(MCTargetDesc)
add_subdirectory(TargetInfo)
+add_subdirectory(Analysis)
diff --git a/llvm/lib/Target/SPIRV/SPIRV.h b/llvm/lib/Target/SPIRV/SPIRV.h
index b947062d79ea8..9460b0808cae8 100644
--- a/llvm/lib/Target/SPIRV/SPIRV.h
+++ b/llvm/lib/Target/SPIRV/SPIRV.h
@@ -30,6 +30,7 @@ createSPIRVInstructionSelector(const SPIRVTargetMachine &TM,
const RegisterBankInfo &RBI);
void initializeSPIRVModuleAnalysisPass(PassRegistry &);
+void initializeSPIRVConvergenceRegionAnalysisWrapperPassPass(PassRegistry &);
void initializeSPIRVPreLegalizerPass(PassRegistry &);
void initializeSPIRVEmitIntrinsicsPass(PassRegistry &);
} // namespace llvm
diff --git a/llvm/lib/Target/SPIRV/SPIRVTargetMachine.cpp b/llvm/lib/Target/SPIRV/SPIRVTargetMachine.cpp
index 3485e367dfc0f..e1b7bdd3140db 100644
--- a/llvm/lib/Target/SPIRV/SPIRVTargetMachine.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVTargetMachine.cpp
@@ -43,6 +43,7 @@ extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeSPIRVTarget() {
PassRegistry &PR = *PassRegistry::getPassRegistry();
initializeGlobalISel(PR);
initializeSPIRVModuleAnalysisPass(PR);
+ initializeSPIRVConvergenceRegionAnalysisWrapperPassPass(PR);
}
static std::string computeDataLayout(const Triple &TT) {
diff --git a/llvm/unittests/Target/SPIRV/CMakeLists.txt b/llvm/unittests/Target/SPIRV/CMakeLists.txt
new file mode 100644
index 0000000000000..326a74b0cbe50
--- /dev/null
+++ b/llvm/unittests/Target/SPIRV/CMakeLists.txt
@@ -0,0 +1,17 @@
+include_directories(
+ ${LLVM_MAIN_SRC_DIR}/lib/Target/SPIRV
+ ${LLVM_BINARY_DIR}/lib/Target/SPIRV
+ )
+
+set(LLVM_LINK_COMPONENTS
+ AsmParser
+ Core
+ SPIRVCodeGen
+ SPIRVAnalysis
+ Support
+ )
+
+add_llvm_target_unittest(SPIRVTests
+ SPIRVConvergenceRegionAnalysisTests.cpp
+ )
+
diff --git a/llvm/unittests/Target/SPIRV/SPIRVConvergenceRegionAnalysisTests.cpp b/llvm/unittests/Target/SPIRV/SPIRVConvergenceRegionAnalysisTests.cpp
new file mode 100644
index 0000000000000..e04fc85df4f93
--- /dev/null
+++ b/llvm/unittests/Target/SPIRV/SPIRVConvergenceRegionAnalysisTests.cpp
@@ -0,0 +1,1099 @@
+//===- SPIRVConvergenceRegionAnalysisTests.cpp ----------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "Analysis/SPIRVConvergenceRegionAnalysis.h"
+#include "llvm/Analysis/DominanceFrontier.h"
+#include "llvm/Analysis/PostDominators.h"
+#include "llvm/AsmParser/Parser.h"
+#include "llvm/IR/Instructions.h"
+#include "llvm/IR/LLVMContext.h"
+#include "llvm/IR/LegacyPassManager.h"
+#include "llvm/IR/Type.h"
+#include "llvm/IR/TypedPointerType.h"
+#include "llvm/Support/SourceMgr.h"
+
+#include "gmock/gmock.h"
+#include "gtest/gtest.h"
+#include <queue>
+
+using ::testing::Contains;
+using ::testing::Pair;
+
+using namespace llvm;
+using namespace llvm::SPIRV;
+
+template <typename T> struct IsA {
+ friend bool operator==(const Value *V, const IsA &) { return isa<T>(V); }
+};
+
+class SPIRVConvergenceRegionAnalysisTest : public testing::Test {
+protected:
+ void SetUp() override {
+ // Required for tests.
+ FAM.registerPass([&] { return PassInstrumentationAnalysis(); });
+ MAM.registerPass([&] { return PassInstrumentationAnalysis(); });
+
+ // Required for ConvergenceRegionAnalysis.
+ FAM.registerPass([&] { return DominatorTreeAnalysis(); });
+ FAM.registerPass([&] { return LoopAnalysis(); });
+
+ FAM.registerPass([&] { return SPIRVConvergenceRegionAnalysis(); });
+ }
+
+ void TearDown() override { M.reset(); }
+
+ SPIRVConvergenceRegionAnalysis::Result &runAnalysis(StringRef Assembly) {
+ assert(M == nullptr &&
+ "Calling runAnalysis multiple times is unsafe. See getAnalysis().");
+
+ SMDiagnostic Error;
+ M = parseAssemblyString(Assembly, Error, Context);
+ assert(M && "Bad assembly. Bad test?");
+ auto *F = getFunction();
+
+ ModulePassManager MPM;
+ MPM.run(*M, MAM);
+ return FAM.getResult<SPIRVConvergenceRegionAnalysis>(*F);
+ }
+
+ SPIRVConvergenceRegionAnalysis::Result &getAnalysis() {
+ assert(M != nullptr && "Has runAnalysis been called before?");
+ return FAM.getResult<SPIRVConvergenceRegionAnalysis>(*getFunction());
+ }
+
+ Function *getFunction() const {
+ assert(M != nullptr && "Has runAnalysis been called before?");
+ return M->getFunction("main");
+ }
+
+ const BasicBlock *getBlock(StringRef Name) {
+ assert(M != nullptr && "Has runAnalysis been called before?");
+
+ auto *F = getFunction();
+ for (BasicBlock &BB : *F) {
+ if (BB.getName() == Name)
+ return &BB;
+ }
+
+ ADD_FAILURE() << "Error: Could not locate requested block. Bad test?";
+ return nullptr;
+ }
+
+ const ConvergenceRegion *getRegionWithEntry(StringRef Name) {
+ assert(M != nullptr && "Has runAnalysis been called before?");
+
+ std::queue<const ConvergenceRegion *> ToProcess;
+ ToProcess.push(getAnalysis().getTopLevelRegion());
+
+ while (ToProcess.size() != 0) {
+ auto *R = ToProcess.front();
+ ToProcess.pop();
+ for (auto *Child : R->Children)
+ ToProcess.push(Child);
+
+ if (R->Entry->getName() == Name)
+ return R;
+ }
+
+ ADD_FAILURE() << "Error: Could not locate requested region. Bad test?";
+ return nullptr;
+ }
+
+ void checkRegionBlocks(const ConvergenceRegion *R,
+ std::initializer_list<const char *> InRegion,
+ std::initializer_list<const char *> NotInRegion) {
+ for (const char *Name : InRegion) {
+ EXPECT_TRUE(R->contains(getBlock(Name)))
+ << "error: " << Name << " not in region " << R->Entry->getName();
+ }
+
+ for (const char *Name : NotInRegion) {
+ EXPECT_FALSE(R->contains(getBlock(Name)))
+ << "error: " << Name << " in region " << R->Entry->getName();
+ }
+ }
+
+protected:
+ LLVMContext Context;
+ FunctionAnalysisManager FAM;
+ ModuleAnalysisManager MAM;
+ std::unique_ptr<Module> M;
+};
+
+MATCHER_P(ContainsBasicBlock, label, "") {
+ for (const auto *bb : arg)
+ if (bb->getName() == label)
+ return true;
+ return false;
+}
+
+TEST_F(SPIRVConvergenceRegionAnalysisTest, DefaultRegion) {
+ StringRef Assembly = R"(
+ define void @main() convergent "hlsl.numthreads"="4,8,16" "hlsl.shader"="compute" {
+ ret void
+ }
+ )";
+
+ const auto *CR = runAnalysis(Assembly).getTopLevelRegion();
+
+ EXPECT_EQ(CR->Parent, nullptr);
+ EXPECT_EQ(CR->ConvergenceToken, std::nullopt);
+ EXPECT_EQ(CR->Children.size(), 0u);
+}
+
+TEST_F(SPIRVConvergenceRegionAnalysisTest, DefaultRegionWithToken) {
+ StringRef Assembly = R"(
+ define void @main() convergent "hlsl.numthreads"="4,8,16" "hlsl.shader"="compute" {
+ %t1 = call token @llvm.experimental.convergence.entry()
+ ret void
+ }
+
+ declare token @llvm.experimental.convergence.entry()
+ )";
+
+ const auto *CR = runAnalysis(Assembly).getTopLevelRegion();
+
+ EXPECT_EQ(CR->Parent, nullptr);
+ EXPECT_EQ(CR->Children.size(), 0u);
+ EXPECT_TRUE(CR->ConvergenceToken.has_value());
+ EXPECT_EQ(CR->ConvergenceToken.value()->getIntrinsicID(),
+ Intrinsic::experimental_convergence_entry);
+}
+
+TEST_F(SPIRVConvergenceRegionAnalysisTest, SingleLoopOneRegion) {
+ StringRef Assembly = R"(
+ define void @main() convergent "hlsl.numthreads"="4,8,16" "hlsl.shader"="compute" {
+ %t1 = call token @llvm.experimental.convergence.entry()
+ %1 = icmp ne i32 0, 0
+ br label %l1
+
+ l1:
+ %tl1 = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token %t1) ]
+ br i1 %1, label %l1_body, label %l1_end
+
+ l1_body:
+ br label %l1_continue
+
+ l1_continue:
+ br label %l1
+
+ l1_end:
+ br label %end
+
+ end:
+ ret void
+ }
+
+ declare token @llvm.experimental.convergence.entry()
+ declare token @llvm.experimental.convergence.control()
+ declare token @llvm.experimental.convergence.loop()
+ )";
+
+ const auto *CR = runAnalysis(Assembly).getTopLevelRegion();
+
+ EXPECT_EQ(CR->Parent, nullptr);
+ EXPECT_EQ(CR->ConvergenceToken.value()->getName(), "t1");
+ EXPECT_TRUE(CR->ConvergenceToken.has_value());
+ EXPECT_EQ(CR->ConvergenceToken.value()->getIntrinsicID(),
+ Intrinsic::experimental_convergence_entry);
+ EXPECT_EQ(CR->Children.size(), 1u);
+}
+
+TEST_F(SPIRVConvergenceRegionAnalysisTest,
+ SingleLoopLoopRegionParentsIsTopLevelRegion) {
+ StringRef Assembly = R"(
+ define void @main() convergent "hlsl.numthreads"="4,8,16" "hlsl.shader"="compute" {
+ %t1 = call token @llvm.experimental.convergence.entry()
+ %1 = icmp ne i32 0, 0
+ br label %l1
+
+ l1:
+ %tl1 = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token %t1) ]
+ br i1 %1, label %l1_body, label %l1_end
+
+ l1_body:
+ br label %l1_continue
+
+ l1_continue:
+ br label %l1
+
+ l1_end:
+ br label %end
+
+ end:
+ ret void
+ }
+
+ declare token @llvm.experimental.convergence.entry()
+ declare token @llvm.experimental.convergence.control()
+ declare token @llvm.experimental.convergence.loop()
+ )";
+
+ const auto *CR = runAnalysis(Assembly).getTopLevelRegion();
+
+ EXPECT_EQ(CR->Parent, nullptr);
+ EXPECT_EQ(CR->ConvergenceToken.value()->getName(), "t1");
+ EXPECT_EQ(CR->Children[0]->Parent, CR);
+ EXPECT_EQ(CR->Children[0]->ConvergenceToken.value()->getName(), "tl1");
+}
+
+TEST_F(SPIRVConvergenceRegionAnalysisTest, SingleLoopExits) {
+ StringRef Assembly = R"(
+ define void @main() convergent "hlsl.numthreads"="4,8,16" "hlsl.shader"="compute" {
+ %t1 = call token @llvm.experimental.convergence.entry()
+ %1 = icmp ne i32 0, 0
+ br label %l1
+
+ l1:
+ %tl1 = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token %t1) ]
+ br i1 %1, label %l1_body, label %l1_end
+
+ l1_body:
+ br label %l1_continue
+
+ l1_continue:
+ br label %l1
+
+ l1_end:
+ br label %end
+
+ end:
+ ret void
+ }
+
+ declare token @llvm.experimental.convergence.entry()
+ declare token @llvm.experimental.convergence.control()
+ declare token @llvm.experimental.convergence.loop()
+ )";
+
+ const auto *CR = runAnalysis(Assembly).getTopLevelRegion();
+ const auto *L = CR->Children[0];
+
+ EXPECT_EQ(L->Exits.size(), 1ul);
+ EXPECT_THAT(L->Exits, ContainsBasicBlock("l1"));
+}
+
+TEST_F(SPIRVConvergenceRegionAnalysisTest, SingleLoopWithBreakExits) {
+ StringRef Assembly = R"(
+ define void @main() convergent "hlsl.numthreads"="4,8,16" "hlsl.shader"="compute" {
+ %t1 = call token @llvm.experimental.convergence.entry()
+ %1 = icmp ne i32 0, 0
+ br label %l1_header
+
+ l1_header:
+ %tl1 = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token %t1) ]
+ br i1 %1, label %l1_body, label %end.loopexit
+
+ l1_body:
+ %2 = icmp ne i32 0, 0
+ br i1 %2, label %l1_condition_true, label %l1_condition_false
+
+ l1_condition_true:
+ %call = call spir_func i32 @_Z3absi(i32 0) [ "convergencectrl"(token %tl1) ]
+ br label %end
+
+ l1_condition_false:
+ br label %l1_continue
+
+ l1_continue:
+ br label %l1_header
+
+ end.loopexit:
+ br label %end
+
+ end:
+ ret void
+ }
+
+ declare token @llvm.experimental.convergence.entry()
+ declare token @llvm.experimental.convergence.control()
+ declare token @llvm.experimental.convergence.loop()
+
+ ; This intrinsic is not convergent. This is only because the backend doesn't
+ ; support convergent operations yet.
+ declare spir_func i32 @_Z3absi(i32) convergent
+ )";
+
+ const auto *CR = runAnalysis(Assembly).getTopLevelRegion();
+ const auto *L = CR->Children[0];
+
+ EXPECT_EQ(L->Exits.size(), 2ul);
+ EXPECT_THAT(L->Exits, ContainsBasicBlock("l1_header"));
+ EXPECT_THAT(L->Exits, ContainsBasicBlock("l1_condition_true"));
+
+ EXPECT_TRUE(CR->contains(getBlock("l1_header")));
+ EXPECT_TRUE(CR->contains(getBlock("l1_condition_true")));
+}
+
+TEST_F(SPIRVConvergenceRegionAnalysisTest, SingleLoopWithBreakRegionBlocks) {
+ StringRef Assembly = R"(
+ define void @main() convergent "hlsl.numthreads"="4,8,16" "hlsl.shader"="compute" {
+ %t1 = call token @llvm.experimental.convergence.entry()
+ %1 = icmp ne i32 0, 0
+ br label %l1_header
+
+ l1_header:
+ %tl1 = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token %t1) ]
+ br i1 %1, label %l1_body, label %end.loopexit
+
+ l1_body:
+ %2 = icmp ne i32 0, 0
+ br i1 %2, label %l1_condition_true, label %l1_condition_false
+
+ l1_condition_true:
+ %call = call spir_func i32 @_Z3absi(i32 0) [ "convergencectrl"(token %tl1) ]
+ br label %end
+
+ l1_condition_false:
+ br label %l1_continue
+
+ l1_continue:
+ br label %l1_header
+
+ end.loopexit:
+ br label %end
+
+ end:
+ ret void
+ }
+
+ declare token @llvm.experimental.convergence.entry()
+ declare token @llvm.experimental.convergence.control()
+ declare token @llvm.experimental.convergence.loop()
+
+ ; This intrinsic is not convergent. This is only because the backend doesn't
+ ; support convergent operations yet.
+ declare spir_func i32 @_Z3absi(i32) convergent
+ )";
+
+ const auto *CR = runAnalysis(Assembly).getTopLevelRegion();
+ const auto *L = CR->Children[0];
+
+ EXPECT_TRUE(CR->contains(getBlock("l1_header")));
+ EXPECT_TRUE(L->contains(getBlock("l1_header")));
+
+ EXPECT_TRUE(CR->contains(getBlock("l1_body")));
+ EXPECT_TRUE(L->contains(getBlock("l1_body")));
+
+ EXPECT_TRUE(CR->contains(getBlock("l1_condition_true")));
+ EXPECT_TRUE(L->contains(getBlock("l1_condition_true")));
+
+ EXPECT_TRUE(CR->contains(getBlock("l1_condition_false")));
+ EXPECT_TRUE(L->contains(getBlock("l1_condition_false")));
+
+ EXPECT_TRUE(CR->contains(getBlock("l1_continue")));
+ EXPECT_TRUE(L->contains(getBlock("l1_continue")));
+
+ EXPECT_TRUE(CR->contains(getBlock("end.loopexit")));
+ EXPECT_FALSE(L->contains(getBlock("end.loopexit")));
+
+ EXPECT_TRUE(CR->contains(getBlock("end")));
+ EXPECT_FALSE(L->contains(getBlock("end")));
+}
+
+// Exact same test as before, except the 'if() break' condition in the loop is
+// not marked with any convergence intrinsic. In such case, it is valid to
+// consider it outside of the loop.
+TEST_F(SPIRVConvergenceRegionAnalysisTest,
+ SingleLoopWithBreakNoConvergenceControl) {
+ StringRef Assembly = R"(
+ define void @main() convergent "hlsl.numthreads"="4,8,16" "hlsl.shader"="compute" {
+ %t1 = call token @llvm.experimental.convergence.entry()
+ %1 = icmp ne i32 0, 0
+ br label %l1_header
+
+ l1_header:
+ %tl1 = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token %t1) ]
+ br i1 %1, label %l1_body, label %end.loopexit
+
+ l1_body:
+ %2 = icmp ne i32 0, 0
+ br i1 %2, label %l1_condition_true, label %l1_condition_false
+
+ l1_condition_true:
+ br label %end
+
+ l1_condition_false:
+ br label %l1_continue
+
+ l1_continue:
+ br label %l1_header
+
+ end.loopexit:
+ br label %end
+
+ end:
+ ret void
+ }
+
+ declare token @llvm.experimental.convergence.entry()
+ declare token @llvm.experimental.convergence.control()
+ declare token @llvm.experimental.convergence.loop()
+ )";
+
+ runAnalysis(Assembly);
+ const auto *L = getRegionWithEntry("l1_header");
+
+ EXPECT_EQ(L->Entry->getName(), "l1_header");
+ EXPECT_EQ(L->Exits.size(), 2ul);
+ EXPECT_THAT(L->Exits, ContainsBasicBlock("l1_header"));
+ EXPECT_THAT(L->Exits, ContainsBasicBlock("l1_body"));
+
+ EXPECT_TRUE(L->contains(getBlock("l1_header")));
+ EXPECT_TRUE(L->contains(getBlock("l1_body")));
+ EXPECT_FALSE(L->contains(getBlock("l1_condition_true")));
+ EXPECT_TRUE(L->contains(getBlock("l1_condition_false")));
+ EXPECT_TRUE(L->contains(getBlock("l1_continue")));
+ EXPECT_FALSE(L->contains(getBlock("end.loopexit")));
+ EXPECT_FALSE(L->contains(getBlock("end")));
+}
+
+TEST_F(SPIRVConvergenceRegionAnalysisTest, TwoLoopsWithControl) {
+ StringRef Assembly = R"(
+ define void @main() convergent "hlsl.numthreads"="4,8,16" "hlsl.shader"="compute" {
+ %t1 = call token @llvm.experimental.convergence.entry()
+ %1 = icmp ne i32 0, 0
+ br label %l1_header
+
+ l1_header:
+ %tl1 = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token %t1) ]
+ br i1 %1, label %l1_body, label %l1_exit
+
+ l1_body:
+ br i1 %1, label %l1_condition_true, label %l1_condition_false
+
+ l1_condition_true:
+ br label %mid
+
+ l1_condition_false:
+ br label %l1_continue
+
+ l1_continue:
+ br label %l1_header
+
+ l1_exit:
+ br label %mid
+
+ mid:
+ br label %l2_header
+
+ l2_header:
+ %tl2 = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token %t1) ]
+ br i1 %1, label %l2_body, label %l2_exit
+
+ l2_body:
+ br i1 %1, label %l2_condition_true, label %l2_condition_false
+
+ l2_condition_true:
+ br label %end
+
+ l2_condition_false:
+ br label %l2_continue
+
+ l2_continue:
+ br label %l2_header
+
+ l2_exit:
+ br label %end
+
+ end:
+ ret void
+ }
+
+ declare token @llvm.experimental.convergence.entry()
+ declare token @llvm.experimental.convergence.control()
+ declare token @llvm.experimental.convergence.loop()
+ )";
+
+ runAnalysis(Assembly);
+
+ {
+ const auto *L = getRegionWithEntry("l1_header");
+
+ EXPECT_EQ(L->Entry->getName(), "l1_header");
+ EXPECT_EQ(L->Exits.size(), 2ul);
+ EXPECT_THAT(L->Exits, ContainsBasicBlock("l1_header"));
+ EXPECT_THAT(L->Exits, ContainsBasicBlock("l1_body"));
+
+ checkRegionBlocks(
+ L, {"l1_header", "l1_body", "l1_condition_false", "l1_continue"},
+ {"", "l2_header", "l2_body", "l2_condition_true", "l2_condition_false",
+ "l2_continue", "l2_exit", "l1_condition_true", "l1_exit", "end"});
+ }
+ {
+ const auto *L = getRegionWithEntry("l2_header");
+
+ EXPECT_EQ(L->Entry->getName(), "l2_header");
+ EXPECT_EQ(L->Exits.size(), 2ul);
+ EXPECT_THAT(L->Exits, ContainsBasicBlock("l2_header"));
+ EXPECT_THAT(L->Exits, ContainsBasicBlock("l2_body"));
+
+ checkRegionBlocks(
+ L, {"l2_header", "l2_body", "l2_condition_false", "l2_continue"},
+ {"", "l1_header", "l1_body", "l1_condition_true", "l1_condition_false",
+ "l1_continue", "l1_exit", "l2_condition_true", "l2_exit", "end"});
+ }
+}
+
+// Both branches in the loop condition break. This means the loop continue
+// targets are unreachable, meaning no reachable back-edge. This should
+// transform the loop condition into a simple condition, meaning we have a
+// single convergence region.
+TEST_F(SPIRVConvergenceRegionAnalysisTest, LoopBothBranchExits) {
+ StringRef Assembly = R"(
+ define void @main() convergent "hlsl.numthreads"="4,8,16" "hlsl.shader"="compute" {
+ %t1 = call token @llvm.experimental.convergence.entry()
+ %1 = icmp ne i32 0, 0
+ br label %l1_header
+
+ l1_header:
+ %tl1 = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token %t1) ]
+ br i1 %1, label %l1_body, label %l1_exit
+
+ l1_body:
+ br i1 %1, label %l1_condition_true, label %l1_condition_false
+
+ l1_condition_true:
+ %call_true = call spir_func i32 @_Z3absi(i32 0) [ "convergencectrl"(token %tl1) ]
+ br label %end
+
+ l1_condition_false:
+ %call_false = call spir_func i32 @_Z3absi(i32 0) [ "convergencectrl"(token %tl1) ]
+ br label %end
+
+ l1_continue:
+ br label %l1_header
+
+ l1_exit:
+ br label %end
+
+ end:
+ ret void
+ }
+
+ declare token @llvm.experimental.convergence.entry()
+ declare token @llvm.experimental.convergence.control()
+ declare token @llvm.experimental.convergence.loop()
+
+ ; This intrinsic is not convergent. This is only because the backend doesn't
+ ; support convergent operations yet.
+ declare spir_func i32 @_Z3absi(i32) convergent
+ )";
+
+ ;
+ const auto *R = runAnalysis(Assembly).getTopLevelRegion();
+
+ ASSERT_EQ(R->Children.size(), 0ul);
+ EXPECT_EQ(R->Exits.size(), 1ul);
+ EXPECT_THAT(R->Exits, ContainsBasicBlock("end"));
+}
+
+TEST_F(SPIRVConvergenceRegionAnalysisTest, InnerLoopBreaks) {
+ StringRef Assembly = R"(
+ define void @main() convergent "hlsl.numthreads"="4,8,16" "hlsl.shader"="compute" {
+ %t1 = call token @llvm.experimental.convergence.entry()
+ %1 = icmp ne i32 0, 0
+ br label %l1_header
+
+ l1_header:
+ %tl1 = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token %t1) ]
+ br i1 %1, label %l1_body, label %l1_exit
+
+ l1_body:
+ br label %l2_header
+
+ l2_header:
+ %tl2 = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token %tl1) ]
+ br i1 %1, label %l2_body, label %l2_exit
+
+ l2_body:
+ br i1 %1, label %l2_condition_true, label %l2_condition_false
+
+ l2_condition_true:
+ %call_true = call spir_func i32 @_Z3absi(i32 0) [ "convergencectrl"(token %tl1) ]
+ br label %end
+
+ l2_condition_false:
+ br label %l2_continue
+
+ l2_continue:
+ br label %l2_header
+
+ l2_exit:
+ br label %l1_continue
+
+ l1_continue:
+ br label %l1_header
+
+ l1_exit:
+ br label %end
+
+ end:
+ ret void
+ }
+
+ declare token @llvm.experimental.convergence.entry()
+ declare token @llvm.experimental.convergence.control()
+ declare token @llvm.experimental.convergence.loop()
+
+ ; This intrinsic is not convergent. This is only because the backend doesn't
+ ; support convergent operations yet.
+ declare spir_func i32 @_Z3absi(i32) convergent
+ )";
+
+ const auto *R = runAnalysis(Assembly).getTopLevelRegion();
+ const auto *L1 = getRegionWithEntry("l1_header");
+ const auto *L2 = getRegionWithEntry("l2_header");
+
+ EXPECT_EQ(R->Children.size(), 1ul);
+ EXPECT_EQ(L1->Children.size(), 1ul);
+ EXPECT_EQ(L1->Parent, R);
+ EXPECT_EQ(L2->Parent, L1);
+
+ EXPECT_EQ(R->Entry->getName(), "");
+ EXPECT_EQ(R->Exits.size(), 1ul);
+ EXPECT_THAT(R->Exits, ContainsBasicBlock("end"));
+
+ EXPECT_EQ(L1->Entry->getName(), "l1_header");
+ EXPECT_EQ(L1->Exits.size(), 2ul);
+ EXPECT_THAT(L1->Exits, ContainsBasicBlock("l1_header"));
+ EXPECT_THAT(L1->Exits, ContainsBasicBlock("l2_condition_true"));
+
+ checkRegionBlocks(L1,
+ {"l1_header", "l1_body", "l2_header", "l2_body",
+ "l2_condition_false", "l2_condition_true", "l2_continue",
+ "l2_exit", "l1_continue"},
+ {"", "l1_exit", "end"});
+
+ EXPECT_EQ(L2->Entry->getName(), "l2_header");
+ EXPECT_EQ(L2->Exits.size(), 2ul);
+ EXPECT_THAT(L2->Exits, ContainsBasicBlock("l2_header"));
+ EXPECT_THAT(L2->Exits, ContainsBasicBlock("l2_body"));
+ checkRegionBlocks(
+ L2, {"l2_header", "l2_body", "l2_condition_false", "l2_continue"},
+ {"", "l1_header", "l1_body", "l2_exit", "l1_continue",
+ "l2_condition_true", "l1_exit", "end"});
+}
+
+TEST_F(SPIRVConvergenceRegionAnalysisTest, SingleLoopMultipleExits) {
+ StringRef Assembly = R"(
+ define void @main() convergent "hlsl.numthreads"="4,8,16" "hlsl.shader"="compute" {
+ %t1 = call token @llvm.experimental.convergence.entry()
+ %cond = icmp ne i32 0, 0
+ br label %l1
+
+ l1:
+ %tl1 = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token %t1) ]
+ br i1 %cond, label %l1_body, label %l1_exit
+
+ l1_body:
+ switch i32 0, label %sw.default.exit [
+ i32 0, label %sw.bb
+ i32 1, label %sw.bb1
+ i32 2, label %sw.bb2
+ ]
+
+ sw.default.exit:
+ br label %sw.default
+
+ sw.default:
+ br label %l1_end
+
+ sw.bb:
+ br label %l1_end
+
+ sw.bb1:
+ br label %l1_continue
+
+ sw.bb2:
+ br label %sw.default
+
+ l1_continue:
+ br label %l1
+
+ l1_exit:
+ br label %l1_end
+
+ l1_end:
+ br label %end
+
+ end:
+ ret void
+ }
+
+ declare token @llvm.experimental.convergence.entry()
+ declare token @llvm.experimental.convergence.control()
+ declare token @llvm.experimental.convergence.loop()
+ )";
+
+ runAnalysis(Assembly).getTopLevelRegion();
+ const auto *L = getRegionWithEntry("l1");
+ ASSERT_NE(L, nullptr);
+
+ EXPECT_EQ(L->Entry, getBlock("l1"));
+ EXPECT_EQ(L->Exits.size(), 2ul);
+ EXPECT_THAT(L->Exits, ContainsBasicBlock("l1"));
+ EXPECT_THAT(L->Exits, ContainsBasicBlock("l1_body"));
+
+ checkRegionBlocks(L, {"l1", "l1_body", "l1_continue", "sw.bb1"},
+ {"", "sw.default.exit", "sw.default", "l1_end", "end",
+ "sw.bb", "sw.bb2", "l1_exit"});
+}
+
+TEST_F(SPIRVConvergenceRegionAnalysisTest,
+ SingleLoopMultipleExitsWithPartialConvergence) {
+ StringRef Assembly = R"(
+ define void @main() convergent "hlsl.numthreads"="4,8,16" "hlsl.shader"="compute" {
+ %t1 = call token @llvm.experimental.convergence.entry()
+ %cond = icmp ne i32 0, 0
+ br label %l1
+
+ l1:
+ %tl1 = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token %t1) ]
+ br i1 %cond, label %l1_body, label %l1_exit
+
+ l1_body:
+ switch i32 0, label %sw.default.exit [
+ i32 0, label %sw.bb
+ i32 1, label %sw.bb1
+ i32 2, label %sw.bb2
+ ]
+
+ sw.default.exit:
+ br label %sw.default
+
+ sw.default:
+ %call = call spir_func i32 @_Z3absi(i32 0) [ "convergencectrl"(token %tl1) ]
+ br label %l1_end
+
+ sw.bb:
+ br label %l1_end
+
+ sw.bb1:
+ br label %l1_continue
+
+ sw.bb2:
+ br label %sw.default
+
+ l1_continue:
+ br label %l1
+
+ l1_exit:
+ br label %l1_end
+
+ l1_end:
+ br label %end
+
+ end:
+ ret void
+ }
+
+ declare token @llvm.experimental.convergence.entry()
+ declare token @llvm.experimental.convergence.control()
+ declare token @llvm.experimental.convergence.loop()
+
+ ; This intrinsic is not convergent. This is only because the backend doesn't
+ ; support convergent operations yet.
+ declare spir_func i32 @_Z3absi(i32) convergent
+ )";
+
+ runAnalysis(Assembly).getTopLevelRegion();
+ const auto *L = getRegionWithEntry("l1");
+ ASSERT_NE(L, nullptr);
+
+ EXPECT_EQ(L->Entry, getBlock("l1"));
+ EXPECT_EQ(L->Exits.size(), 3ul);
+ EXPECT_THAT(L->Exits, ContainsBasicBlock("l1"));
+ EXPECT_THAT(L->Exits, ContainsBasicBlock("l1_body"));
+ EXPECT_THAT(L->Exits, ContainsBasicBlock("sw.default"));
+
+ checkRegionBlocks(L,
+ {"l1", "l1_body", "l1_continue", "sw.bb1",
+ "sw.default.exit", "sw.bb2", "sw.default"},
+ {"", "l1_end", "end", "sw.bb", "l1_exit"});
+}
+
+TEST_F(SPIRVConvergenceRegionAnalysisTest,
+ SingleLoopWithDeepConvergenceBranch) {
+ StringRef Assembly = R"(
+ define void @main() convergent "hlsl.numthreads"="4,8,16" "hlsl.shader"="compute" {
+ %t1 = call token @llvm.experimental.convergence.entry()
+ %1 = icmp ne i32 0, 0
+ br label %l1_header
+
+ l1_header:
+ %tl1 = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token %t1) ]
+ br i1 %1, label %l1_body, label %l1_end
+
+ l1_body:
+ %2 = icmp ne i32 0, 0
+ br i1 %2, label %l1_condition_true, label %l1_condition_false
+
+ l1_condition_true:
+ br label %a
+
+ a:
+ br label %b
+
+ b:
+ br label %c
+
+ c:
+ %call = call spir_func i32 @_Z3absi(i32 0) [ "convergencectrl"(token %tl1) ]
+ br label %end
+
+ l1_condition_false:
+ br label %l1_continue
+
+ l1_continue:
+ br label %l1_header
+
+ l1_end:
+ br label %end
+
+ end:
+ ret void
+ }
+
+ declare token @llvm.experimental.convergence.entry()
+ declare token @llvm.experimental.convergence.control()
+ declare token @llvm.experimental.convergence.loop()
+
+ ; This intrinsic is not convergent. This is only because the backend doesn't
+ ; support convergent operations yet.
+ declare spir_func i32 @_Z3absi(i32) convergent
+ )";
+
+ runAnalysis(Assembly).getTopLevelRegion();
+ const auto *L = getRegionWithEntry("l1_header");
+ ASSERT_NE(L, nullptr);
+
+ EXPECT_EQ(L->Entry, getBlock("l1_header"));
+ EXPECT_EQ(L->Exits.size(), 2ul);
+ EXPECT_THAT(L->Exits, ContainsBasicBlock("l1_header"));
+ EXPECT_THAT(L->Exits, ContainsBasicBlock("c"));
+
+ checkRegionBlocks(L,
+ {"l1_header", "l1_body", "l1_continue",
+ "l1_condition_false", "l1_condition_true", "a", "b", "c"},
+ {"", "l1_end", "end"});
+}
+
+TEST_F(SPIRVConvergenceRegionAnalysisTest,
+ SingleLoopWithDeepConvergenceLateBranch) {
+ StringRef Assembly = R"(
+ define void @main() convergent "hlsl.numthreads"="4,8,16" "hlsl.shader"="compute" {
+ %t1 = call token @llvm.experimental.convergence.entry()
+ %1 = icmp ne i32 0, 0
+ br label %l1_header
+
+ l1_header:
+ %tl1 = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token %t1) ]
+ br i1 %1, label %l1_body, label %l1_end
+
+ l1_body:
+ %2 = icmp ne i32 0, 0
+ br i1 %2, label %l1_condition_true, label %l1_condition_false
+
+ l1_condition_true:
+ br label %a
+
+ a:
+ br label %b
+
+ b:
+ br i1 %2, label %c, label %d
+
+ c:
+ %call = call spir_func i32 @_Z3absi(i32 0) [ "convergencectrl"(token %tl1) ]
+ br label %end
+
+ d:
+ br label %end
+
+ l1_condition_false:
+ br label %l1_continue
+
+ l1_continue:
+ br label %l1_header
+
+ l1_end:
+ br label %end
+
+ end:
+ ret void
+ }
+
+ declare token @llvm.experimental.convergence.entry()
+ declare token @llvm.experimental.convergence.control()
+ declare token @llvm.experimental.convergence.loop()
+
+ ; This intrinsic is not convergent. This is only because the backend doesn't
+ ; support convergent operations yet.
+ declare spir_func i32 @_Z3absi(i32) convergent
+ )";
+
+ runAnalysis(Assembly).getTopLevelRegion();
+ const auto *L = getRegionWithEntry("l1_header");
+ ASSERT_NE(L, nullptr);
+
+ EXPECT_EQ(L->Entry, getBlock("l1_header"));
+ EXPECT_EQ(L->Exits.size(), 3ul);
+ EXPECT_THAT(L->Exits, ContainsBasicBlock("l1_header"));
+ EXPECT_THAT(L->Exits, ContainsBasicBlock("b"));
+ EXPECT_THAT(L->Exits, ContainsBasicBlock("c"));
+
+ checkRegionBlocks(L,
+ {"l1_header", "l1_body", "l1_continue",
+ "l1_condition_false", "l1_condition_true", "a", "b", "c"},
+ {"", "l1_end", "end", "d"});
+}
+
+TEST_F(SPIRVConvergenceRegionAnalysisTest,
+ SingleLoopWithNoConvergenceIntrinsics) {
+ StringRef Assembly = R"(
+ define void @main() "hlsl.numthreads"="4,8,16" "hlsl.shader"="compute" {
+ %1 = icmp ne i32 0, 0
+ br label %l1_header
+
+ l1_header:
+ br i1 %1, label %l1_body, label %l1_end
+
+ l1_body:
+ %2 = icmp ne i32 0, 0
+ br i1 %2, label %l1_condition_true, label %l1_condition_false
+
+ l1_condition_true:
+ br label %a
+
+ a:
+ br label %end
+
+ l1_condition_false:
+ br label %l1_continue
+
+ l1_continue:
+ br label %l1_header
+
+ l1_end:
+ br label %end
+
+ end:
+ ret void
+ }
+ )";
+
+ runAnalysis(Assembly).getTopLevelRegion();
+ const auto *L = getRegionWithEntry("l1_header");
+ ASSERT_NE(L, nullptr);
+
+ EXPECT_EQ(L->Entry, getBlock("l1_header"));
+ EXPECT_EQ(L->Exits.size(), 2ul);
+ EXPECT_THAT(L->Exits, ContainsBasicBlock("l1_header"));
+ EXPECT_THAT(L->Exits, ContainsBasicBlock("l1_body"));
+
+ checkRegionBlocks(
+ L, {"l1_header", "l1_body", "l1_continue", "l1_condition_false"},
+ {"", "l1_end", "end", "l1_condition_true", "a"});
+}
+
+TEST_F(SPIRVConvergenceRegionAnalysisTest, SimpleFunction) {
+ StringRef Assembly = R"(
+ define void @main() "hlsl.numthreads"="4,8,16" "hlsl.shader"="compute" {
+ ret void
+ }
+ )";
+
+ const auto *R = runAnalysis(Assembly).getTopLevelRegion();
+ ASSERT_NE(R, nullptr);
+
+ EXPECT_EQ(R->Entry, getBlock(""));
+ EXPECT_EQ(R->Exits.size(), 1ul);
+ EXPECT_THAT(R->Exits, ContainsBasicBlock(""));
+ EXPECT_TRUE(R->contains(getBlock("")));
+}
+
+TEST_F(SPIRVConvergenceRegionAnalysisTest, NestedLoopInBreak) {
+ StringRef Assembly = R"(
+ define void @main() convergent "hlsl.numthreads"="4,8,16" "hlsl.shader"="compute" {
+ %t1 = call token @llvm.experimental.convergence.entry()
+ %1 = icmp ne i32 0, 0
+ br label %l1
+
+ l1:
+ %tl1 = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token %t1) ]
+ br i1 %1, label %l1_body, label %l1_to_end
+
+ l1_body:
+ br i1 %1, label %cond_inner, label %l1_continue
+
+ cond_inner:
+ br label %l2
+
+ l2:
+ %tl2 = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token %tl1) ]
+ br i1 %1, label %l2_body, label %l2_end
+
+ l2_body:
+ %call = call spir_func i32 @_Z3absi(i32 0) [ "convergencectrl"(token %tl2) ]
+ br label %l2_continue
+
+ l2_continue:
+ br label %l2
+
+ l2_end:
+ br label %l2_exit
+
+ l2_exit:
+ %call2 = call spir_func i32 @_Z3absi(i32 0) [ "convergencectrl"(token %tl1) ]
+ br label %l1_end
+
+ l1_continue:
+ br label %l1
+
+ l1_to_end:
+ br label %l1_end
+
+ l1_end:
+ br label %end
+
+ end:
+ ret void
+ }
+
+ declare token @llvm.experimental.convergence.entry()
+ declare token @llvm.experimental.convergence.control()
+ declare token @llvm.experimental.convergence.loop()
+ declare spir_func i32 @_Z3absi(i32) convergent
+ )";
+
+ const auto *R = runAnalysis(Assembly).getTopLevelRegion();
+ ASSERT_NE(R, nullptr);
+
+ EXPECT_EQ(R->Children.size(), 1ul);
+
+ const auto *L1 = R->Children[0];
+ EXPECT_EQ(L1->Children.size(), 1ul);
+ EXPECT_EQ(L1->Entry->getName(), "l1");
+ EXPECT_EQ(L1->Exits.size(), 2ul);
+ EXPECT_THAT(L1->Exits, ContainsBasicBlock("l1"));
+ EXPECT_THAT(L1->Exits, ContainsBasicBlock("l2_exit"));
+ checkRegionBlocks(L1,
+ {"l1", "l1_body", "l1_continue", "cond_inner", "l2",
+ "l2_body", "l2_end", "l2_continue", "l2_exit"},
+ {"", "l1_to_end", "l1_end", "end"});
+
+ const auto *L2 = L1->Children[0];
+ EXPECT_EQ(L2->Children.size(), 0ul);
+ EXPECT_EQ(L2->Entry->getName(), "l2");
+ EXPECT_EQ(L2->Exits.size(), 1ul);
+ EXPECT_THAT(L2->Exits, ContainsBasicBlock("l2"));
+ checkRegionBlocks(L2, {"l2", "l2_body", "l2_continue"},
+ {"", "l1_to_end", "l1_end", "end", "l1", "l1_body",
+ "l1_continue", "cond_inner", "l2_end", "l2_exit"});
+}
More information about the llvm-commits
mailing list